@prodero/curator 0.0.0-stage → 0.2.0

package/dist/core/memory-pressure.js

@@ -0,0 +1,212 @@
+/** Bounded, observational evidence. Nothing here chooses or mutates knowledge. */
+import fs from 'node:fs/promises';
+import { z } from 'zod';
+import { checkedDataPath, ensureDataDirectory, writeFileAtomic, sha256, parseCheckpointData } from './storage.js';
+import { withSyncLock } from './sync-transaction.js';
+export const MEMORY_PRESSURE_LIMITS = { sessions: 32, syncs: 32, records: 128, relationships: 128, bytes: 1048576 };
+export const MEMORY_PRESSURE_PATH = 'local/memory-pressure.json';
+const reasons = ['selected', 'marginal-value', 'context-budget', 'required-budget', 'item-limit', 'rank-window', 'inactive', 'superseded', 'scope-mismatch', 'no-match'];
+const count = z.number().finite().nonnegative();
+const id = z.string().max(256);
+const recordSchema = z.object({
+    localityKey: z.string().regex(/^[a-f0-9]{64}$/), id, layer: z.enum(['local', 'versioned']), title: z.string().max(160), scope: z.string().max(32), scopeRef: z.string().max(256).nullable(), scopeKind: z.enum(['path', 'topic']).optional(),
+    category: z.string().max(32), status: z.string().max(32), estimatedTokens: count, related: z.array(id).max(8),
+}).strict();
+export function pressureRecord(item) {
+    return { localityKey: sha256(JSON.stringify([item.scope, item.scopeRef, item.scopeKind ?? null, item.category])), id: item.id, layer: item.layer, title: item.title.slice(0, 160), scope: item.scope, scopeRef: item.scopeRef?.slice(0, 256) ?? null, ...(item.scopeKind ? { scopeKind: item.scopeKind } : {}), category: item.category, status: item.status, estimatedTokens: item.estimatedTokens,
+        related: [...new Set([...item.related, ...item.supersedes, ...item.relations.map(r => r.target)])].slice(0, 8) };
+}
+const selectionSchema = recordSchema.extend({ bucket: z.enum(['required', 'contextual', 'filtered']), reason: z.enum(reasons), score: z.number().finite().nullable(), rankScore: z.number().finite().nullable(), signals: z.array(z.object({ kind: z.string().max(32), score: z.number().finite() }).strict()).max(12) }).strict();
+const totalsSchema = z.object({ records: count, tokens: count }).strict();
+const retrievalSchema = z.object({
+    queryHash: z.string().regex(/^[a-f0-9]{64}$/), purpose: z.string().max(16),
+    budgets: z.object({ contextBudgetTokens: count, requiredContextBudgetTokens: count, maxRequiredItems: count, maxContextItems: count }).strict(),
+    totals: z.record(z.enum(reasons), totalsSchema), requiredTruncated: z.boolean(), requiredOmitted: count,
+    records: z.array(selectionSchema).max(MEMORY_PRESSURE_LIMITS.records), detailsDropped: count,
+}).strict();
+/** Called at existing selection/filter branches; never evaluates relevance itself. */
+export class RetrievalEvidenceCollector {
+    samples = {};
+    totals = {};
+    requiredOmitted = 0;
+    observe(item, bucket, reason, score = null, rankScore = null, signals = []) {
+        const total = this.totals[reason] ??= { records: 0, tokens: 0 };
+        total.records++;
+        total.tokens += item.estimatedTokens;
+        if (bucket === 'required' && reason !== 'selected')
+            this.requiredOmitted++;
+        const sample = this.samples[reason] ??= [];
+        if (sample.length < MEMORY_PRESSURE_LIMITS.records)
+            sample.push({ ...pressureRecord(item), bucket, reason, score, rankScore, signals: signals.slice(0, 12).map(({ kind, score }) => ({ kind, score })) });
+    }
+    finish(query, purpose, budgets) {
+        // Retain selected and budget-pressure details before lower-ranked/filter samples.
+        const records = ['selected', 'required-budget', 'context-budget', 'item-limit', 'marginal-value', 'rank-window', 'scope-mismatch', 'inactive', 'superseded', 'no-match'].flatMap(r => this.samples[r] ?? []).slice(0, MEMORY_PRESSURE_LIMITS.records);
+        return { queryHash: sha256(query.trim()), purpose, budgets, totals: this.totals, requiredTruncated: this.requiredOmitted > 0, requiredOmitted: this.requiredOmitted, records, detailsDropped: Object.values(this.totals).reduce((n, t) => n + (t?.records ?? 0), 0) - records.length };
+    }
+}
+const categories = ['state_transition', 'refinement', 'duplicate', 'supporting', 'potential_conflict', 'unresolved', 'independent'];
+const relationshipSchema = z.object({ source: id, target: recordSchema, classification: z.enum(categories) }).strict();
+const syncSchema = z.object({ checkpoint: id, sessionIds: z.array(id).max(128), sessionIdsDropped: count,
+    changes: z.array(recordSchema.extend({ actions: z.array(z.enum(['created', 'updated', 'merged', 'superseded'])).max(4) }).strict()).max(MEMORY_PRESSURE_LIMITS.records),
+    totals: z.object({ created: count, updated: count, merged: count, superseded: count }).strict(), changesDropped: count,
+    reconciliation: z.array(relationshipSchema).max(MEMORY_PRESSURE_LIMITS.relationships), classifications: z.record(z.enum(categories), count), relationshipsDropped: count,
+}).strict();
+export class SyncEvidenceCollector {
+    relationships = [];
+    classifications = {};
+    observe(source, target, classification) {
+        this.classifications[classification] = (this.classifications[classification] ?? 0) + 1;
+        if (this.relationships.length < MEMORY_PRESSURE_LIMITS.relationships)
+            this.relationships.push({ source, target, classification });
+    }
+    finish(records, ids) {
+        const sets = { created: new Set(ids.created), updated: new Set(ids.updated), merged: new Set(ids.consolidated), superseded: new Set(ids.superseded) };
+        const changes = records.map(record => ({ ...record, actions: Object.keys(sets).filter(key => sets[key].has(record.id)) })).filter(r => r.actions.length);
+        return { changes: changes.slice(0, MEMORY_PRESSURE_LIMITS.records), totals: { created: sets.created.size, updated: sets.updated.size, merged: sets.merged.size, superseded: sets.superseded.size }, changesDropped: Math.max(0, changes.length - MEMORY_PRESSURE_LIMITS.records),
+            reconciliation: this.relationships, classifications: this.classifications, relationshipsDropped: Object.values(this.classifications).reduce((n, c) => n + (c ?? 0), 0) - this.relationships.length };
+    }
+}
+const sessionSchema = retrievalSchema.extend({ sessionId: id }).strict();
+const storeSchema = z.object({ version: z.literal(1), sessionFloor: id.nullable().default(null), syncFloor: id.nullable().default(null), sessions: z.array(sessionSchema).max(MEMORY_PRESSURE_LIMITS.sessions), syncs: z.array(syncSchema).max(MEMORY_PRESSURE_LIMITS.syncs) }).strict();
+const empty = () => ({ version: 1, sessionFloor: null, syncFloor: null, sessions: [], syncs: [] });
+async function readStore(root) {
+    try {
+        const target = await checkedDataPath(root, MEMORY_PRESSURE_PATH);
+        if ((await fs.stat(target)).size > MEMORY_PRESSURE_LIMITS.bytes)
+            return { data: empty(), status: 'unavailable' };
+        const parsed = storeSchema.safeParse(JSON.parse(await fs.readFile(target, 'utf8')));
+        return parsed.success ? { data: parsed.data, status: 'available' } : { data: empty(), status: 'unavailable' };
+    }
+    catch (error) {
+        return { data: empty(), status: error.code === 'ENOENT' ? 'missing' : 'unavailable' };
+    }
+}
+function evictOldest(state, kind) {
+    const removed = kind === 'sessions' ? state.sessions.pop()?.sessionId : state.syncs.pop()?.checkpoint;
+    const field = kind === 'sessions' ? 'sessionFloor' : 'syncFloor';
+    if (removed && (!state[field] || removed.localeCompare(state[field]) > 0))
+        state[field] = removed;
+}
+async function update(root, mutate) {
+    try {
+        const current = await readStore(root);
+        if (current.status === 'unavailable')
+            return 'unavailable';
+        const before = JSON.stringify(current.data);
+        mutate(current.data);
+        current.data.sessions.sort((a, b) => b.sessionId.localeCompare(a.sessionId));
+        while (current.data.sessions.length > MEMORY_PRESSURE_LIMITS.sessions)
+            evictOldest(current.data, 'sessions');
+        current.data.syncs.sort((a, b) => b.checkpoint.localeCompare(a.checkpoint));
+        while (current.data.syncs.length > MEMORY_PRESSURE_LIMITS.syncs)
+            evictOldest(current.data, 'syncs');
+        let text = JSON.stringify(storeSchema.parse(current.data));
+        while (Buffer.byteLength(text) > MEMORY_PRESSURE_LIMITS.bytes) {
+            if (current.data.sessions.length >= current.data.syncs.length)
+                evictOldest(current.data, 'sessions');
+            else
+                evictOldest(current.data, 'syncs');
+            text = JSON.stringify(current.data);
+        }
+        if (text !== before) {
+            await ensureDataDirectory(root, 'local');
+            await writeFileAtomic(await checkedDataPath(root, MEMORY_PRESSURE_PATH), text);
+        }
+        return 'recorded';
+    }
+    catch {
+        return 'unavailable';
+    }
+}
+export async function recordRetrievalEvidence(root, sessionId, evidence) {
+    try {
+        return await withSyncLock(root, () => update(root, state => { if (state.sessionFloor && sessionId.localeCompare(state.sessionFloor) <= 0)
+            return; state.sessions = state.sessions.filter(s => s.sessionId !== sessionId); state.sessions.push({ ...evidence, sessionId }); }), { waitMs: 5000 });
+    }
+    catch {
+        return 'unavailable';
+    }
+}
+/** Caller holds the existing Sync lock. Only publish after its checkpoint committed. */
+export async function recordSyncEvidenceLocked(root, evidence) {
+    return update(root, state => { if (state.syncFloor && evidence.checkpoint.localeCompare(state.syncFloor) <= 0)
+        return; state.syncs = state.syncs.filter(s => s.checkpoint !== evidence.checkpoint); state.syncs.push(evidence); });
+}
+/** Read-only recent evidence, never a maintenance recommendation. No knowledge bodies read. */
+export async function getMemoryPressureSnapshot(root) {
+    const { data, status } = await readStore(root);
+    const checkpoints = [];
+    let checkpointCoverage = 'available';
+    try {
+        const directory = await checkedDataPath(root, 'checkpoints');
+        const names = (await fs.readdir(directory)).filter(n => /^chk_[a-zA-Z0-9_-]+\.json$/.test(n)).sort().reverse().slice(0, MEMORY_PRESSURE_LIMITS.syncs);
+        for (const name of names)
+            checkpoints.push(parseCheckpointData(await fs.readFile(await checkedDataPath(root, 'checkpoints', name), 'utf8'), name));
+    }
+    catch {
+        checkpointCoverage = 'unavailable';
+    }
+    const committed = new Set(checkpoints.map(c => c.id));
+    const syncs = data.syncs.filter(s => committed.has(s.checkpoint));
+    const records = new Map();
+    const groups = new Map();
+    const group = (r) => {
+        const key = r.localityKey;
+        let g = groups.get(key);
+        if (!g) {
+            g = { scope: r.scope, scopeRef: r.scopeRef, scopeKind: r.scopeKind, category: r.category, records: new Set(), sessions: new Set(), selected: new Set(), budgetOmitted: new Set(), budgetSessions: new Set(), created: new Set(), updated: new Set(), superseded: new Set(), classifications: {} };
+            groups.set(key, g);
+        }
+        g.records.add(`${r.layer}:${r.id}`);
+        return g;
+    };
+    for (const session of data.sessions)
+        for (const r of session.records) {
+            if (r.bucket === 'filtered')
+                continue;
+            const key = `${r.layer}:${r.id}`;
+            let evidence = records.get(key);
+            if (!evidence) {
+                evidence = { record: r, consideredSessions: new Set(), selectedSessions: new Set(), budgetOmissionSessions: new Set() };
+                records.set(key, evidence);
+            }
+            evidence.consideredSessions.add(session.sessionId);
+            const g = group(r);
+            g.sessions.add(session.sessionId);
+            if (r.reason === 'selected') {
+                evidence.selectedSessions.add(session.sessionId);
+                g.selected.add(key);
+            }
+            if (r.reason === 'context-budget' || r.reason === 'required-budget') {
+                evidence.budgetOmissionSessions.add(session.sessionId);
+                g.budgetOmitted.add(key);
+                g.budgetSessions.add(session.sessionId);
+            }
+        }
+    const classifications = {};
+    for (const sync of syncs) {
+        for (const [kind, n] of Object.entries(sync.classifications))
+            classifications[kind] = (classifications[kind] ?? 0) + (n ?? 0);
+        for (const change of sync.changes) {
+            const g = group(change);
+            for (const action of change.actions)
+                if (action !== 'merged')
+                    g[action].add(`${change.layer}:${change.id}`);
+        }
+        for (const rel of sync.reconciliation) {
+            const g = group(rel.target);
+            g.classifications[rel.classification] = (g.classifications[rel.classification] ?? 0) + 1;
+        }
+    }
+    return { version: 1, status, checkpointCoverage, limits: MEMORY_PRESSURE_LIMITS, sessions: data.sessions, syncs,
+        coverage: { sessionFloor: data.sessionFloor, syncFloor: data.syncFloor, sessions: data.sessions.length, checkpoints: checkpoints.length, checkpointsWithoutDetails: checkpoints.filter(c => !syncs.some(s => s.checkpoint === c.id)).map(c => c.id), retrievalDetailsDropped: data.sessions.reduce((n, s) => n + s.detailsDropped, 0) },
+        totals: { relevantConsiderations: data.sessions.reduce((n, s) => n + Object.entries(s.totals).filter(([r]) => ['selected', 'marginal-value', 'context-budget', 'required-budget', 'item-limit', 'rank-window'].includes(r)).reduce((n, [, v]) => n + (v?.records ?? 0), 0), 0),
+            selectedKnowledgeTokens: data.sessions.reduce((n, s) => n + (s.totals.selected?.tokens ?? 0), 0),
+            budgetOmittedKnowledgeTokens: data.sessions.reduce((n, s) => n + (s.totals['context-budget']?.tokens ?? 0) + (s.totals['required-budget']?.tokens ?? 0), 0),
+            repeatedBudgetOmittedRecords: [...records.values()].filter(r => r.budgetOmissionSessions.size > 1).length,
+            requiredTruncatedSessions: data.sessions.filter(s => s.requiredTruncated).length, newKnowledge: checkpoints.reduce((n, c) => n + (c.createdMemory?.length ?? 0), 0), updatedKnowledge: checkpoints.reduce((n, c) => n + (c.updatedMemory?.length ?? 0), 0), supersededKnowledge: checkpoints.reduce((n, c) => n + (c.supersededMemory?.length ?? 0), 0), classifications },
+        records: [...records.values()].map(r => ({ ...r.record, consideredSessions: r.consideredSessions.size, selectedSessions: r.selectedSessions.size, budgetOmissionSessions: r.budgetOmissionSessions.size })),
+        groups: [...groups.entries()].map(([key, g]) => ({ key, scope: g.scope, scopeRef: g.scopeRef, scopeKind: g.scopeKind, category: g.category, records: [...g.records], retrievalSessions: g.sessions.size, selectedRecords: g.selected.size, budgetOmittedRecords: g.budgetOmitted.size, budgetOmissionSessions: g.budgetSessions.size, newRecords: g.created.size, updatedRecords: g.updated.size, supersededRecords: g.superseded.size, classifications: g.classifications })), };
+}
+//# sourceMappingURL=memory-pressure.js.map

package/dist/core/memory-sync-curator.js

@@ -0,0 +1,82 @@
+/** Sync orchestration adapter. Questions, answers and jobs belong to Curator. */
+import { createCurationReview, resumeCurationReview, latestCurationReview, loadCurationReview, assertCurationHead, CurationReviewError } from './curation-review.js';
+import { planKnowledgeCuration } from './knowledge-curation.js';
+import { graphicalReviewPayload } from './curation-graphical.js';
+import { prepareSemanticCuration, inspectSemanticCuration, curationAnswers } from './curation-semantic-jobs.js';
+import { prepareMemoryTransformation, validateMemoryTransformationProposal } from './memory-transformation-proposal.js';
+import { previewMemoryTransformation } from './memory-transformation.js';
+import { scopeEffect, semanticHash } from './curation-semantic-contract.js';
+import { deterministicBaseline } from './curation-revision.js';
+/** Explicit expiry refresh carries staged semantics and non-approval decisions
+ * into a fresh revision. It neither reopens a question nor records an apply. */
+export async function refreshSyncQuestions(root, sourceIds, input) {
+    const review = await loadCurationReview(root, input.reviewId, { allowExpired: true }), revision = review.snapshot.revision;
+    await assertCurationHead(root, review);
+    if (revision.id !== input.reviewRevisionId || review.snapshot.planHash !== input.planHash || review.state.revision !== input.decisionRevision || semanticHash(await curationAnswers(root, revision.lineageId)) !== input.answerLogHash)
+        throw new CurationReviewError('review_changed', 'This saved question changed. Inspect current progress.');
+    const payload = await graphicalReviewPayload(root, review), baseline = deterministicBaseline(revision);
+    if (payload.projection.lifecycle !== 'stale')
+        throw new CurationReviewError('review_changed', 'This review has not expired. Inspect its current questions.');
+    if ((await planKnowledgeCuration(root, { memorySyncSources: sourceIds })).planId !== baseline.planId)
+        throw new CurationReviewError('semantic_stale', 'Knowledge or evidence changed. Re-analyze before refreshing this question.');
+    const next = await createCurationReview(root, { parent: review, operationId: `question_refresh_${semanticHash([revision.id, input.decisionRevision, input.answerLogHash])}`, applied: [] }, sourceIds);
+    return graphicalReviewPayload(root, next);
+}
+import { sameApplicability } from './applicability.js';
+export async function prepareSyncCuration(root, sourceIds, reviewId, retry = false) {
+    const previous = reviewId ? undefined : await latestCurationReview(root, (await planKnowledgeCuration(root, { memorySyncSources: sourceIds })).planId);
+    const review = reviewId ? await resumeCurationReview(root, reviewId) : previous ?? await createCurationReview(root, undefined, sourceIds);
+    const payload = await graphicalReviewPayload(root, review), g = payload.graphical;
+    if (g.activeQuestionCount || g.deferredQuestionIds.length)
+        return { reviewId: review.snapshot.reviewId, payload };
+    // Replaying Prepare after Retry must return its live attempt, not the old initial request.
+    if (g.continuationJobId)
+        return { reviewId: review.snapshot.reviewId, payload, job: await inspectSemanticCuration(root, g.continuationJobId) };
+    const failed = retry ? g.jobs.find(j => !j.suspended && ['rejected', 'stale'].includes(j.status)) : undefined;
+    const job = await prepareSemanticCuration(root, { reviewId: review.snapshot.reviewId, reviewRevisionId: review.snapshot.revision.id, componentId: failed?.componentId ?? review.snapshot.revision.dependencies.groups[0].component, requestId: failed ? `sync_retry_${failed.id}` : `sync_${review.snapshot.reviewId}`, reason: failed ? (failed.status === 'rejected' ? 'validation_correction' : 'resume') : 'initial', ...(failed ? failed.status === 'rejected' ? { correctionForJobId: failed.id } : { resumeForJobId: failed.id } : {}), limits: { maxItems: 24 } });
+    return { reviewId: review.snapshot.reviewId, payload, job };
+}
+export async function inspectSyncCuration(root, reviewId) {
+    return graphicalReviewPayload(root, await resumeCurationReview(root, reviewId));
+}
+/** Reuse the existing Sync validator/writer; Curator staging is never authority.
+ * Protected, shared or scope-changing outputs stay current. */
+export async function previewSyncCuration(root, candidateId, payload) {
+    const typed = payload.review.snapshot.revision.typedPlan;
+    if (typed.kind !== 'semantic-v2')
+        return undefined;
+    const analysis = typed.analyses.at(-1);
+    const proposal = analysis.proposal;
+    if (proposal.issues.length || proposal.additionalEvidenceRequired.length || proposal.limitations.length || proposal.sources.some(s => s.disposition === 'unresolved'))
+        return undefined;
+    const prepared = await prepareMemoryTransformation(root, candidateId), b = prepared.bundle;
+    if (!prepared.canPropose)
+        throw new Error('Complete transformation evidence is required');
+    if (analysis.bundle.expectedSources.length !== b.sourceIds.length || analysis.bundle.expectedSources.some(s => b.revisions.active[s.id] !== s.revision))
+        throw new Error('Sync analysis sources changed');
+    const eligible = proposal.outputs.filter(o => o.sourceIds.every(id => {
+        const s = b.activeSources.find(s => s.id === id), first = b.activeSources.find(s => s.id === o.sourceIds[0]);
+        if (!s || !first || s.protected || s.layer !== 'local' || s.knowledge.status !== 'active' || s.knowledge.contextPolicy !== 'relevant' || s.knowledge.authority !== 'inferred')
+            return false;
+        const legacyRepresentable = s.knowledge.applicability?.kind !== 'legacy' || s.knowledge.scope === 'global' || s.knowledge.scope === 'module' && s.knowledge.scopeKind === 'path';
+        return legacyRepresentable && sameApplicability(s.knowledge, first.knowledge) && scopeEffect(s.knowledge, o.applicability) === 'unchanged';
+    }));
+    // A source with several outputs is replaced only when every output is eligible.
+    let allowed = eligible;
+    while (true) {
+        const next = allowed.filter(o => o.sourceIds.every(id => proposal.sources.find(s => s.sourceId === id).outputKeys.every(k => allowed.some(a => a.key === k))));
+        if (next.length === allowed.length)
+            break;
+        allowed = next;
+    }
+    const records = allowed.map(o => {
+        const source = b.activeSources.find(s => s.id === o.sourceIds[0]).knowledge;
+        return { key: o.key.replace(/^out_/, 'draft_'), category: source.category, priority: source.priority, title: o.title, body: o.body, sourceIds: o.sourceIds, applicability: o.applicability, scope: source.scope === 'global' ? 'global' : source.scopeKind === 'path' ? 'path' : 'project', ...(source.scopeKind === 'path' ? { path: source.scopeRef } : {}) };
+    });
+    const transformed = { ...prepared.binding, records, dispositions: b.sourceIds.map(sourceId => {
+            const keys = allowed.filter(o => o.sourceIds.includes(sourceId)).map(o => o.key.replace(/^out_/, 'draft_'));
+            return { sourceId, action: keys.length ? 'consolidate' : 'keep', proposedKeys: keys, rationale: proposal.sources.find(s => s.sourceId === sourceId).rationale, coverage: 'Validated Curator clause coverage; protected and shared sources remain current.', omittedAsRedundant: [], unresolvedInformation: [] };
+        }), summary: `${proposal.rationale}\nCurator review: ${payload.review.snapshot.reviewId}; revision: ${payload.review.snapshot.revision.id}; answer events: ${analysis.bundle.answers.map(a => a.id).join(', ') || 'none'}.`, warnings: [], additionalEvidenceRequired: [] };
+    return previewMemoryTransformation(root, await validateMemoryTransformationProposal(root, transformed));
+}
+//# sourceMappingURL=memory-sync-curator.js.map