@prodero/curator 0.0.0-stage → 0.1.0

package/dist/core/curation-semantic-jobs.js

@@ -0,0 +1,280 @@
+/** Bounded host-agent adapter. Writes only local job/answer/review state.
+ * No provider, sampling, grant, canonical writer, promotion or Apply imports. */
+import { assertSafeCurationData } from './curation-evidence.js';
+import fs from 'node:fs/promises';
+import { z } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+import { checkedDataPath, ensureDataDirectory, writeFileAtomic } from './storage.js';
+import { withSyncLock, ownsSyncLock } from './sync-transaction.js';
+import { loadMemoryCatalog } from './memory-index.js';
+import { planKnowledgeCuration } from './knowledge-curation.js';
+import { loadCurationReview, assertCurationHead, publishSemanticReview, CurationReviewError } from './curation-review.js';
+import { deterministicBaseline, revisionIssues } from './curation-revision.js';
+import { SEMANTIC_CONTRACT, SEMANTIC_VALIDATOR, SEMANTIC_INSTRUCTIONS, semanticProposalSchema, semanticLimitsSchema, semanticHash as hash, semanticClauses, measuredTokens, validateSemanticProposal, materializeSemanticPlan, SemanticValidationError } from './curation-semantic-contract.js';
+export const SEMANTIC_RETENTION = { maxJobs: 64, maxBytes: 2 * 1024 * 1024, maxAnswers: 128 };
+export const jobStatuses = ['prepared', 'awaiting_reasoning', 'submitted', 'validated', 'rejected', 'stale', 'failed'];
+const adapterSchema = z.object({ host: z.string().max(120).optional(), model: z.string().max(120).optional() }).strict();
+export const semanticPrepareSchema = z.object({ reviewId: z.string().regex(/^review_[a-zA-Z0-9_]+$/), reviewRevisionId: z.string().regex(/^revision_[a-f0-9]{64}$/), componentId: z.string().regex(/^component_[a-f0-9]{24}$/), requestId: z.string().min(1).max(100), reason: z.enum(['initial', 'new_evidence', 'answer', 'validation_correction', 'resume']).default('initial'), correctionForJobId: z.string().regex(/^semantic_job_[a-f0-9]{40}$/).optional(), limits: semanticLimitsSchema.partial().optional(), adapter: adapterSchema.optional() }).strict();
+const dir = 'local/curation-reasoning';
+const reviewLock = (root, operation, options) => ownsSyncLock(root) ? operation() : withSyncLock(root, operation, options);
+const last = (job) => job.events.at(-1);
+const reject = (code, message) => { throw new CurationReviewError(code, message); };
+async function jobPath(root, id) { if (!/^semantic_job_[a-f0-9]{40}$/.test(id))
+    return reject('semantic_invalid', 'Invalid reasoning job.'); return checkedDataPath(root, dir, `${id}.json`); }
+export async function semanticJobs(root) { const names = await fs.readdir(await checkedDataPath(root, dir)).catch((e) => { if (e.code === 'ENOENT')
+    return []; throw e; }); return Promise.all(names.filter(n => /^semantic_job_[a-f0-9]{40}\.json$/.test(n)).sort().map(n => loadSemanticJob(root, n.slice(0, -5)))); }
+export async function loadSemanticJob(root, id) {
+    const file = await jobPath(root, id);
+    if ((await fs.stat(file)).size > SEMANTIC_RETENTION.maxBytes)
+        return reject('semantic_limit', 'Reasoning history reached its limit.');
+    const job = JSON.parse(await fs.readFile(file, 'utf8'));
+    assertSafeCurationData(job);
+    if (job.version !== 1 || job.input?.jobId !== id || job.inputSeal !== hash(job.input) || job.input.contractVersion !== SEMANTIC_CONTRACT || job.input.validatorVersion !== SEMANTIC_VALIDATOR || job.input.bundleHash !== hash(job.input.bundle) || job.input.projectIdentity !== await fs.realpath(root))
+        return reject('semantic_invalid', 'Reasoning job integrity failed.');
+    let previous = job.inputSeal;
+    for (const [i, event] of job.events.entries()) {
+        const { hash: eventHash, ...body } = event;
+        if (event.sequence !== i + 1 || event.previousHash !== previous || eventHash !== hash(body) || !jobStatuses.includes(event.status))
+            return reject('semantic_invalid', 'Reasoning history integrity failed.');
+        previous = eventHash;
+    }
+    if (!job.events.length)
+        return reject('semantic_invalid', 'Reasoning history is empty.');
+    return job;
+}
+async function save(root, job) { assertSafeCurationData(job); const data = JSON.stringify(job); if (Buffer.byteLength(data) > SEMANTIC_RETENTION.maxBytes)
+    return reject('semantic_limit', 'Reasoning history reached its limit.'); await ensureDataDirectory(root, dir); await writeFileAtomic(await jobPath(root, job.input.jobId), data); }
+async function event(root, job, status, data = {}) { const body = { sequence: job.events.length + 1, at: new Date().toISOString(), status, previousHash: last(job)?.hash ?? job.inputSeal, ...data }; job.events.push({ ...body, hash: hash(body) }); await save(root, job); }
+async function noPendingTransaction(root) { if (await fs.stat(await checkedDataPath(root, '.transactions/pending.json')).then(() => true, () => false))
+    return reject('recovery_required', 'Finish the pending transaction before semantic analysis.'); }
+async function current(root, reviewId, revisionId, decisionRevision) {
+    await noPendingTransaction(root);
+    const review = await loadCurationReview(root, reviewId);
+    if (review.snapshot.revision?.id !== revisionId || decisionRevision !== undefined && review.state.revision !== decisionRevision)
+        return reject('semantic_stale', 'Reasoning input is obsolete.');
+    await assertCurationHead(root, review);
+    const baseline = deterministicBaseline(review.snapshot.revision);
+    const now = await planKnowledgeCuration(root, { memorySyncSources: baseline.memorySyncSources });
+    if (now.planId !== baseline.planId)
+        return reject('semantic_stale', 'Knowledge or evidence changed.');
+    return review;
+}
+async function answerPath(root, lineageId) { if (!/^review_[a-zA-Z0-9_]+$/.test(lineageId))
+    return reject('semantic_invalid', 'Invalid answer lineage.'); return checkedDataPath(root, dir, `${lineageId}.answers.json`); }
+export async function curationAnswers(root, lineageId) {
+    const file = await answerPath(root, lineageId);
+    let data;
+    try {
+        if ((await fs.stat(file)).size > SEMANTIC_RETENTION.maxBytes)
+            return reject('semantic_limit', 'Answer history reached its limit.');
+        data = JSON.parse(await fs.readFile(file, 'utf8'));
+    }
+    catch (e) {
+        if (e.code === 'ENOENT')
+            return [];
+        throw e;
+    }
+    // Historical sealed logs fail closed; never rewrite their contents during replay.
+    assertSafeCurationData(data);
+    if (data.version !== 1 || data.projectIdentity !== await fs.realpath(root) || data.hash !== hash(data.events) || !Array.isArray(data.events) || data.events.length > SEMANTIC_RETENTION.maxAnswers)
+        return reject('semantic_invalid', 'Answer history integrity failed.');
+    return data.events;
+}
+/** Internal input foundation for a future trusted App/form adapter. Not an MCP
+ * tool; an agent cannot claim App origin via the reasoning contract. */
+export async function appendCurationAnswer(root, input) {
+    input = z.object({ reviewId: z.string(), reviewRevisionId: z.string(), questionId: z.string(), questionVersion: z.number().int().positive(), requestId: z.string().min(1).max(100), origin: z.enum(['app', 'form', 'agent_relay']), choiceId: z.string().max(100).optional(), rawText: z.string().max(8192).optional(), replacesAnswerId: z.string().optional(), decisionRevision: z.number().int().nonnegative().optional(), answerLogHash: z.string().optional() }).strict().parse(input);
+    assertSafeCurationData(input);
+    return reviewLock(root, async () => {
+        const review = await current(root, input.reviewId, input.reviewRevisionId, input.decisionRevision), revision = review.snapshot.revision, issue = revisionIssues(revision).find(i => i.question.id === input.questionId && i.question.version === input.questionVersion);
+        if (!issue)
+            return reject('semantic_invalid', 'Unknown question version.');
+        if (!!input.choiceId === !!input.rawText || input.rawText && (!issue.question.allowFreeText || Buffer.byteLength(input.rawText) > 8192))
+            return reject('semantic_invalid', 'Invalid answer.');
+        const choice = input.choiceId ? issue.question.choices.find(c => c.id === input.choiceId) : undefined;
+        if (input.choiceId && !choice)
+            return reject('semantic_invalid', 'Unknown answer choice.');
+        const events = await curationAnswers(root, revision.lineageId), prior = events.find(e => e.requestId === input.requestId);
+        if (prior) {
+            if (prior.questionId !== input.questionId || prior.choice?.id !== input.choiceId || prior.rawText !== input.rawText)
+                return reject('review_changed', 'Answer request changed.');
+            return prior;
+        }
+        if (input.answerLogHash !== undefined && input.answerLogHash !== hash(events))
+            return reject('review_changed', 'Another answer was saved. Reload this review.');
+        if (events.length >= SEMANTIC_RETENTION.maxAnswers)
+            return reject('semantic_limit', 'Answer history reached its limit.');
+        if (events.some(e => e.questionId === input.questionId && !events.some(a => a.replacesAnswerId === e.id)) && !input.replacesAnswerId)
+            return reject('review_changed', 'Replacing an answer requires its exact prior event.');
+        if (input.replacesAnswerId && !events.some(e => e.id === input.replacesAnswerId && e.questionId === input.questionId && !events.some(a => a.replacesAnswerId === e.id)))
+            return reject('semantic_invalid', 'Unknown previous answer.');
+        const answer = { id: `answer_${hash([revision.id, input.requestId]).slice(0, 32)}`, reviewId: input.reviewId, reviewRevisionId: revision.id, lineageId: revision.lineageId, questionId: issue.question.id, questionVersion: issue.question.version, issueId: issue.id, basisHash: issue.basisHash, sourceIds: issue.sourceIds, origin: input.origin, requestId: input.requestId, at: new Date().toISOString(), ...(choice ? { choice } : {}), ...(input.rawText ? { rawText: input.rawText } : {}), ...(input.replacesAnswerId ? { replacesAnswerId: input.replacesAnswerId } : {}) };
+        assertSafeCurationData(answer);
+        events.push(answer);
+        await ensureDataDirectory(root, dir);
+        await writeFileAtomic(await answerPath(root, revision.lineageId), JSON.stringify({ version: 1, projectIdentity: await fs.realpath(root), events, hash: hash(events) }));
+        return answer;
+    }, { recoverLifecycle: false, waitMs: 2000 });
+}
+async function makeBundle(root, review, componentId, limits) {
+    const revision = review.snapshot.revision, base = deterministicBaseline(revision), groups = revision.dependencies.groups.filter(g => g.component === componentId);
+    if (!groups.length)
+        return reject('semantic_invalid', 'Unknown dependency component.');
+    const ids = [...new Set(groups.flatMap(g => [...g.proposalIds, ...g.policyDependencies]))].sort();
+    const catalog = await loadMemoryCatalog(root, { persistCache: false }), sources = [], missing = [];
+    for (const id of ids) {
+        const summary = catalog.summaries.find(s => s.id === id);
+        if (!summary) {
+            missing.push(`missing-source:${id}`);
+            continue;
+        }
+        const record = await catalog.load(id);
+        sources.push({ id, revision: catalog.originalHashes.get(id), record, layer: summary.relativePath.startsWith('local/') ? 'local' : 'versioned', clauses: semanticClauses(id, catalog.originalHashes.get(id), record.content), editable: base.proposals.some(p => p.id === id) });
+    }
+    const expectedSources = sources.map(s => ({ id: s.id, revision: s.revision }));
+    if (expectedSources.length > 32)
+        return reject('semantic_limit', 'Component exceeds the maximum accountable source count.');
+    const evidence = [...new Map(base.proposals.filter(p => ids.includes(p.id)).flatMap(p => (p.investigation?.evidence ?? []).map(e => { const id = `evidence_${hash([e.path, e.revision, e.line]).slice(0, 24)}`; return [id, { id, revision: e.revision, excerpt: e.excerpt, kind: e.kind }]; }))).values()];
+    const allAnswers = await curationAnswers(root, revision.lineageId), issues = revisionIssues(revision);
+    const answers = allAnswers.filter(a => !allAnswers.some(b => b.replacesAnswerId === a.id) && issues.some(i => i.id === a.issueId && i.basisHash === a.basisHash && i.question.id === a.questionId) && a.sourceIds.some(id => ids.includes(id)));
+    const bundle = { version: 1, id: '', contract: SEMANTIC_CONTRACT, reviewId: review.snapshot.reviewId, reviewRevisionId: revision.id, lineageId: revision.lineageId, componentId, baseline: revision.baseline, expectedSources, sources, evidence, answers, answerLogHash: hash(allAnswers), dependencies: { ...revision.dependencies, groups }, signals: base.proposals.filter(p => ids.includes(p.id)).map(p => ({ sourceId: p.id, attention: p.attention, changes: p.changes })), missingFacts: base.proposals.filter(p => ids.includes(p.id) && p.attention.length && ((p.issueBasis?.length ?? 0) > 1 || p.before.status === 'disputed' || p.before.relations?.some(r => r.type === 'contradicts'))).map(p => ({ sourceIds: (p.issueBasis ?? [{ id: p.id }]).map(s => s.id).filter(id => ids.includes(id)), reason: 'Existing conflicting policy requires a project fact; neither source is authoritative merely because it is newer.' })), completeness: { complete: !missing.length && !base.evidenceTruncated, reasons: [...missing, ...(base.evidenceTruncated ? ['evidence-incomplete'] : [])] }, limits };
+    const mark = (reason) => { bundle.completeness.complete = false; if (!bundle.completeness.reasons.includes(reason))
+        bundle.completeness.reasons.push(reason); };
+    if (bundle.sources.length > limits.maxItems) {
+        bundle.sources = bundle.sources.slice(0, limits.maxItems);
+        mark('source-limit');
+    }
+    if (bundle.evidence.length > limits.maxEvidence) {
+        bundle.evidence = bundle.evidence.slice(0, limits.maxEvidence);
+        mark('evidence-limit');
+    }
+    while (Buffer.byteLength(JSON.stringify(bundle)) > limits.maxBytes - 128 || measuredTokens(bundle) > limits.maxTokens - 32) {
+        mark('bundle-limit');
+        if (bundle.evidence.length)
+            bundle.evidence.pop();
+        else if (bundle.sources.length)
+            bundle.sources.pop();
+        else
+            return reject('semantic_limit', 'The minimum component manifest exceeds the reasoning budget.');
+    }
+    bundle.id = `bundle_${hash(bundle)}`;
+    return bundle;
+}
+function response(job) { const { bundle, ...input } = job.input; const history = job.events.map(({ proposal, ...event }) => event); return { jobId: job.input.jobId, status: last(job).status, job: { ...job, input, events: history }, task: { instructions: SEMANTIC_INSTRUCTIONS, proposalSchema: zodToJsonSchema(semanticProposalSchema, { $refStrategy: 'none' }), data: bundle }, acceptance: 'staged_candidate_only', validation: last(job).reason, resultReviewId: last(job).resultReviewId }; }
+export async function prepareSemanticCuration(root, raw) {
+    const input = semanticPrepareSchema.parse(raw);
+    return reviewLock(root, async () => {
+        const review = await current(root, input.reviewId, input.reviewRevisionId), limits = semanticLimitsSchema.parse(input.limits ?? {}), bundle = await makeBundle(root, review, input.componentId, limits), history = await semanticJobs(root);
+        const semanticInputHash = hash([bundle.expectedSources, bundle.sources, bundle.evidence, bundle.answers, bundle.signals, bundle.missingFacts, bundle.completeness, SEMANTIC_CONTRACT, SEMANTIC_VALIDATOR]);
+        const priorRequest = history.find(j => j.input.requestId === input.requestId && j.input.lineageId === bundle.lineageId);
+        if (priorRequest) {
+            if (priorRequest.input.semanticInputHash !== semanticInputHash || priorRequest.input.reviewRevisionId !== input.reviewRevisionId)
+                return reject('review_changed', 'Reasoning request changed.');
+            return response(priorRequest);
+        }
+        const matches = history.filter(j => j.input.semanticInputHash === semanticInputHash && j.input.lineageId === bundle.lineageId).sort((a, b) => b.input.createdAt.localeCompare(a.input.createdAt));
+        const same = matches.find(j => j.input.reviewRevisionId === input.reviewRevisionId && j.input.decisionRevision === review.state.revision) ?? matches[0];
+        let correction = 0;
+        if (input.reason === 'validation_correction') {
+            const previous = history.find(j => j.input.jobId === input.correctionForJobId);
+            if (!previous || last(previous).status !== 'rejected' || previous.input.semanticInputHash !== semanticInputHash || previous.input.correction >= previous.input.bundle.limits.maxCorrections)
+                return reject('semantic_limit', 'No correction retry remains.');
+            correction = previous.input.correction + 1;
+            const retry = history.find(j => j.input.previousJobId === previous.input.jobId);
+            if (retry)
+                return response(retry);
+        }
+        else if (same && !(input.reason === 'resume' && ['prepared', 'awaiting_reasoning', 'submitted', 'stale'].includes(last(same).status) && (same.input.reviewRevisionId !== input.reviewRevisionId || same.input.decisionRevision !== review.state.revision)))
+            return response(same);
+        if (history.length >= SEMANTIC_RETENTION.maxJobs || correction === 0 && history.filter(j => j.input.lineageId === bundle.lineageId && j.input.correction === 0).length >= Math.min(limits.maxPasses, ...history.filter(j => j.input.lineageId === bundle.lineageId).map(j => j.input.bundle.limits.maxPasses)))
+            return reject('semantic_limit', 'Reasoning budget reached; pending history was preserved.');
+        const createdAt = new Date().toISOString(), inputHash = hash([bundle.id, review.state.revision, bundle.answerLogHash, SEMANTIC_VALIDATOR, correction]), jobId = `semantic_job_${hash([bundle.lineageId, inputHash]).slice(0, 40)}`;
+        const body = { jobId, projectIdentity: review.snapshot.projectIdentity, reviewId: input.reviewId, reviewRevisionId: input.reviewRevisionId, lineageId: bundle.lineageId, decisionRevision: review.state.revision, bundleId: bundle.id, bundleHash: hash(bundle), bundle, contractVersion: SEMANTIC_CONTRACT, validatorVersion: SEMANTIC_VALIDATOR, requestId: input.requestId, reason: input.reason, inputHash, semanticInputHash, createdAt, expiresAt: new Date(Date.now() + limits.ttlMs).toISOString(), correction, ...(input.correctionForJobId ? { previousJobId: input.correctionForJobId } : {}), ...(input.adapter ? { adapter: input.adapter } : {}) };
+        const job = { version: 1, input: body, inputSeal: hash(body), events: [] };
+        await event(root, job, 'prepared');
+        await event(root, job, 'awaiting_reasoning');
+        return response(job);
+    }, { recoverLifecycle: false, waitMs: 2000 });
+}
+export async function inspectSemanticCuration(root, jobId) { return response(await loadSemanticJob(root, jobId)); }
+export async function submitSemanticCuration(root, input) {
+    return reviewLock(root, async () => {
+        const job = await loadSemanticJob(root, input.jobId), outputHash = hash(input.proposal), status = last(job).status;
+        if (input.inputHash !== job.input.inputHash)
+            return reject('semantic_invalid', 'Reasoning input hash changed.');
+        const submission = job.events.find(e => e.status === 'submitted');
+        if (submission && submission.outputHash !== outputHash)
+            return reject('review_changed', 'This job already has a different submission.');
+        if (['validated', 'rejected', 'stale', 'failed'].includes(status)) {
+            if (last(job).outputHash !== outputHash)
+                return reject('review_changed', 'This job already has a different terminal result.');
+            return response(job);
+        }
+        if (submission) {
+            const resultId = `review_${hash(['semantic', job.input.jobId]).slice(0, 40)}`;
+            const file = await checkedDataPath(root, 'local/curation-reviews', `${resultId}.json`);
+            if (await fs.stat(file).then(() => true, () => false)) {
+                const recovered = await loadCurationReview(root, resultId, { allowExpired: true }), typed = recovered.snapshot.revision?.typedPlan;
+                if (typed?.kind !== 'semantic-v2' || typed.analyses.at(-1)?.jobId !== job.input.jobId || typed.analyses.at(-1)?.outputHash !== outputHash)
+                    return reject('semantic_invalid', 'Published semantic revision changed.');
+                await event(root, job, 'validated', { outputHash, resultReviewId: resultId, resultRevisionId: recovered.snapshot.revision.id, reason: 'recovered_staged_revision' });
+                return { ...response(job), review: recovered };
+            }
+        }
+        let review;
+        try {
+            review = await current(root, job.input.reviewId, job.input.reviewRevisionId, job.input.decisionRevision);
+            if (Date.now() > Date.parse(job.input.expiresAt) || hash(await curationAnswers(root, job.input.lineageId)) !== job.input.bundle.answerLogHash)
+                throw new Error('obsolete');
+        }
+        catch {
+            await event(root, job, 'stale', { reason: 'obsolete_revision_or_evidence', outputHash });
+            return response(job);
+        }
+        // Size-check before persisting an untrusted output. Store only its hash if over budget.
+        if (Buffer.byteLength((JSON.stringify(input.proposal) ?? 'null')) > job.input.bundle.limits.maxOutputBytes || measuredTokens(input.proposal) > job.input.bundle.limits.maxOutputTokens) {
+            await event(root, job, 'rejected', { reason: 'output_limit', outputHash });
+            return response(job);
+        }
+        if (!submission)
+            await event(root, job, 'submitted', { outputHash, proposal: input.proposal });
+        try {
+            const proposal = validateSemanticProposal(job.input.bundle, input.proposal), accepted = { jobId: job.input.jobId, inputHash: job.input.inputHash, outputHash, decisionRevision: job.input.decisionRevision, correction: job.input.correction, bundle: job.input.bundle, proposal }, previous = review.snapshot.revision.typedPlan;
+            const analyses = [...(previous.kind === 'semantic-v2' ? previous.analyses : []), accepted], basePlan = deterministicBaseline(review.snapshot.revision), plan = materializeSemanticPlan(basePlan, analyses);
+            const next = await publishSemanticReview(root, review, { kind: 'semantic-v2', validator: SEMANTIC_VALIDATOR, basePlan, analyses, plan }, job.input.jobId);
+            await event(root, job, 'validated', { outputHash, resultReviewId: next.snapshot.reviewId, resultRevisionId: next.snapshot.revision.id, reason: proposal.sources.every(s => s.disposition !== 'proposed') ? 'terminal_no_rewrite' : 'structurally_validated_candidate' });
+            return { ...response(job), review: next };
+        }
+        catch (error) {
+            if (error instanceof SemanticValidationError || error instanceof z.ZodError) {
+                await event(root, job, 'rejected', { reason: error instanceof SemanticValidationError ? error.reason : 'malformed_output', outputHash });
+                return response(job);
+            }
+            if (error instanceof CurationReviewError && ['review_retention_full', 'review_too_large', 'semantic_limit'].includes(error.code)) {
+                await event(root, job, 'failed', { reason: 'staging_capacity', outputHash });
+                return response(job);
+            }
+            throw error;
+        }
+    }, { recoverLifecycle: false, waitMs: 2000 });
+}
+export function semanticJobStatus(job, review) { const status = last(job).status; return ['prepared', 'awaiting_reasoning', 'submitted'].includes(status) && (job.input.decisionRevision !== review.state.revision || Date.now() > Date.parse(job.input.expiresAt)) ? 'stale' : status; }
+/** Unchanged supporting records must not keep a decided component's job active. */
+export function semanticJobSuspended(job, review) {
+    const sourceIds = new Set(job.input.bundle.dependencies.groups.flatMap(g => g.proposalIds));
+    const affected = review.snapshot.units.filter(u => u.proposalIds.some(id => sourceIds.has(id))).map(u => review.state.items[u.id]);
+    const work = affected.filter(item => item.outcome !== 'unchanged');
+    return (work.length ? work : affected).every(item => item.execution === 'applied' || ['deferred', 'keep_current'].includes(item.disposition ?? ''));
+}
+export async function semanticReviewStatus(root, review) {
+    const history = (await semanticJobs(root)).filter(j => j.input.reviewRevisionId === review.snapshot.revision?.id);
+    const active = history.filter(j => !history.some(next => next.input.previousJobId === j.input.jobId) && !semanticJobSuspended(j, review));
+    const pending = active.filter(j => ['prepared', 'awaiting_reasoning', 'submitted'].includes(semanticJobStatus(j, review))), failed = active.filter(j => ['failed', 'rejected', 'stale'].includes(semanticJobStatus(j, review)));
+    const revision = review.snapshot.revision, issues = revision ? revisionIssues(revision) : [], answers = revision ? await curationAnswers(root, revision.lineageId) : [];
+    const bound = new Set(revision?.answerBindings.map(a => a.answerEventId) ?? []);
+    const awaiting = answers.filter(a => review.snapshot.units.some(u => u.proposalIds.some(id => a.sourceIds.includes(id)) && !['deferred', 'keep_current'].includes(review.state.items[u.id]?.disposition ?? '')) && !bound.has(a.id) && !answers.some(b => b.replacesAnswerId === a.id) && issues.some(i => i.id === a.issueId && i.basisHash === a.basisHash && i.question.id === a.questionId));
+    const answerGroups = revision?.dependencies.groups.filter(g => awaiting.some(a => a.sourceIds.some(id => g.proposalIds.includes(id)))).map(g => g.id) ?? [];
+    return { pending: pending.length + awaiting.filter(a => !pending.some(j => j.input.bundle.answers.some(bound => bound.id === a.id))).length, answeredIssueIds: awaiting.map(a => a.issueId), failed: failed.length, jobIds: history.map(j => j.input.jobId), blockedGroupIds: [...new Set([...answerGroups, ...[...pending, ...failed].flatMap(j => j.input.bundle.dependencies.groups.map(g => g.id))])] };
+}
+export async function semanticReferencedReviews(root) { return new Set((await semanticJobs(root)).map(j => j.input.reviewId)); }
+//# sourceMappingURL=curation-semantic-jobs.js.map

package/dist/core/diagnostics.js

@@ -0,0 +1,17 @@
+import { hasSecret } from "./secrets.js";
+/** Public diagnostics never include parser source excerpts or rejected schema values. */
+export function publicDiagnostic(error) {
+    const message = error instanceof Error ? error.message : typeof error === "string" ? error : "Operation failed";
+    if (message.startsWith("PRODERO_") && !hasSecret(message))
+        return message;
+    if (error instanceof SyntaxError || /Unexpected token|Unexpected end of JSON|not valid JSON|JSON at position/.test(message)) {
+        return "PRODERO_INVALID_JSON: Invalid JSON input or stored data. Inspect the selected input/configuration or preserved transaction journal; restore valid data before retrying.";
+    }
+    if (/Input validation error|Output validation error|"code":\s*"(?:invalid_|too_|unrecognized_)/.test(message)) {
+        return "PRODERO_INVALID_ARGUMENT: Tool arguments or metadata do not match the schema. Check the tool's field types and allowed values before retrying.";
+    }
+    if (hasSecret(message))
+        return "PRODERO_UNSAFE_DIAGNOSTIC: Rejected input contains a potential credential. Remove secrets and retry; the rejected value is omitted.";
+    return message;
+}
+//# sourceMappingURL=diagnostics.js.map

package/dist/core/format.js

@@ -0,0 +1,114 @@
+export function parseFrontmatter(content, filename = "canonical Markdown") {
+    content = content.replace(/\r\n/g, "\n");
+    if (!content.startsWith("---\n"))
+        return { metadata: {}, body: content };
+    const end = content.indexOf("\n---\n", 4);
+    if (end < 0)
+        throw new Error(`Invalid canonical frontmatter: ${filename} has no closing delimiter.`);
+    const metadata = {};
+    for (const line of content.slice(4, end).split("\n")) {
+        const match = line.match(/^([a-zA-Z0-9_]+):\s*(.*)$/);
+        if (!line.trim())
+            continue;
+        if (!match)
+            throw new Error(`Invalid canonical frontmatter: ${filename} has a malformed metadata line.`);
+        const [, key, raw] = match;
+        if (Object.hasOwn(metadata, key))
+            throw new Error(`Invalid canonical frontmatter: ${filename} has duplicate ${key}.`);
+        if (raw === "null" || raw === "")
+            metadata[key] = null;
+        else if (raw === "true" || raw === "false")
+            metadata[key] = raw === "true";
+        else if (raw.startsWith("[") || raw.startsWith("{") || raw.startsWith('"')) {
+            try {
+                metadata[key] = JSON.parse(raw);
+            }
+            catch {
+                throw new Error(`Invalid canonical frontmatter: ${filename} has malformed ${key}.`);
+            }
+        }
+        else if (/^-?\d+$/.test(raw))
+            metadata[key] = Number(raw);
+        else
+            metadata[key] = raw;
+    }
+    return { metadata, body: content.slice(end + 5).replace(/^\r?\n/, "") };
+}
+export function frontmatter(metadata, body) {
+    const lines = Object.entries(metadata).map(([key, value]) => {
+        if (value === null || value === undefined)
+            return `${key}: null`;
+        if (typeof value === "number" || typeof value === "boolean")
+            return `${key}: ${value}`;
+        if (Array.isArray(value) || typeof value === "object")
+            return `${key}: ${JSON.stringify(value)}`;
+        const stringValue = String(value);
+        return `${key}: ${/[\n:[\]{}#,]|^\s|\s$|^"|^(?:true|false|null|-?\d+)$/.test(stringValue) ? JSON.stringify(stringValue) : stringValue}`;
+    });
+    return `---\n${lines.join("\n")}\n---\n\n${body.trim()}\n`;
+}
+export function sectionBullets(body, headings) {
+    const result = Object.fromEntries(headings.map((heading) => [heading, []]));
+    let current = null;
+    let bullet = null;
+    const flush = () => {
+        if (current && bullet) {
+            const value = bullet.join("\n").trim();
+            if (value)
+                result[current].push(value);
+        }
+        bullet = null;
+    };
+    for (const line of body.split(/\r?\n/)) {
+        if (bullet && line.startsWith("  ")) {
+            bullet.push(line.slice(2));
+            continue;
+        }
+        const heading = line.match(/^##\s+(.+?)\s*$/)?.[1];
+        if (heading) {
+            flush();
+            current = headings.includes(heading) ? heading : null;
+            continue;
+        }
+        const item = line.match(/^\s*[-*]\s+(.+)$/)?.[1];
+        if (item && current) {
+            flush();
+            bullet = [item];
+        }
+        else if (bullet && /^\s{2,}\S/.test(line))
+            bullet.push(line.trim());
+    }
+    flush();
+    return result;
+}
+export function parseObservationAnnotations(raw) {
+    if (!raw.startsWith("["))
+        return { metadata: {}, content: raw.trim() };
+    let end = -1;
+    let quoted = false;
+    for (let i = 1; i < raw.length; i += 1) {
+        if (raw[i] === "\\" && quoted) {
+            i += 1;
+            continue;
+        }
+        if (raw[i] === '"')
+            quoted = !quoted;
+        if (raw[i] === "]" && !quoted) {
+            end = i;
+            break;
+        }
+    }
+    if (end < 0)
+        return { metadata: {}, content: raw.trim() };
+    const metadata = {};
+    for (const match of raw.slice(1, end).matchAll(/([a-z_]+)=("(?:\\.|[^"\\])*"|[^\s;]+)/g)) {
+        try {
+            metadata[match[1]] = match[2].startsWith('"') ? JSON.parse(match[2]) : match[2];
+        }
+        catch {
+            metadata[match[1]] = match[2].replace(/^"|"$/g, "");
+        }
+    }
+    return { metadata, content: raw.slice(end + 1).trim() };
+}
+//# sourceMappingURL=format.js.map

package/dist/core/ids.js

@@ -0,0 +1,6 @@
+import crypto from "node:crypto";
+export function sortableId(prefix, now = new Date()) {
+    const stamp = now.toISOString().replace(/[-:.TZ]/g, "").slice(0, 17);
+    return `${prefix}_${stamp}_${crypto.randomUUID().replace(/-/g, "").slice(0, 12)}`;
+}
+//# sourceMappingURL=ids.js.map

package/dist/core/imports.js

@@ -0,0 +1,169 @@
+import { applicabilitySchema, withApplicability, sameApplicability, applicabilityKey, inferApplicability } from "./applicability.js";
+import { assertCompatibleProject } from "./storage.js";
+import fs from "node:fs/promises";
+import { sortableId } from "./ids.js";
+import { z } from "zod";
+import { listKnowledge, renderKnowledge } from "./memory.js";
+import { listProjectSkills, renderSkill, skillInputSchema } from "./skills.js";
+import { readConfig, sha256, listCheckpoints } from "./storage.js";
+import { checkedPath, hasSecret, readSource, safeRelative, snapshotImportState } from "./plugin-files.js";
+import { commitSyncTransaction, recoverSyncTransaction, withSyncLock } from "./sync-transaction.js";
+import { buildIndex } from "./index.js";
+import { estimateTokens, knowledgeText, MAX_CANONICAL_ITEM_TOKENS } from "./telemetry.js";
+import { assertEnglishProse } from "./language.js";
+const hash = z.string().regex(/^[a-f0-9]{64}$/);
+const text = z.string().trim().min(1);
+export const importEvidenceSchema = z.object({ path: text.max(400), sha256: hash, startLine: z.number().int().positive(), endLine: z.number().int().positive() }).strict();
+export const importItemSchema = z.object({
+    applicability: applicabilitySchema.optional(), scopeKind: z.enum(["path", "topic"]).optional(),
+    key: text.max(100), destination: z.enum(["project_instruction", "knowledge", "skill"]),
+    title: text.max(200).regex(/^[^\r\n]+$/), content: text.max(8000),
+    scope: z.enum(["global", "module", "task"]), scopeRef: text.max(300).nullable(),
+    category: z.enum(["principle", "invariant", "decision", "constraint", "lesson", "state", "instruction"]),
+    evidence: z.array(importEvidenceSchema).min(1).max(12), reason: text.max(1000),
+    persistEvidence: z.boolean().optional(),
+    skill: skillInputSchema.strict().nullable(), activateSkill: z.boolean(),
+}).strict();
+export const importBatchSchema = z.object({
+    version: z.literal(1), planId: hash, approvedPlanId: hash,
+    stateFingerprint: hash, items: z.array(importItemSchema).max(100),
+}).strict();
+export async function readImportState(root) {
+    if (!(await readConfig(root)))
+        throw new Error("Initialize Prodero before importing knowledge");
+    const files = await snapshotImportState(root);
+    return { knowledge: await listKnowledge(root), skills: await listProjectSkills(root), fingerprint: sha256(JSON.stringify(files)), files, importedSources: (await listCheckpoints(root)).flatMap(c => c.importReceipt ? [...c.importReceipt.created, ...c.importReceipt.duplicates].flatMap(i => i.source ?? []) : []), importedKeys: (await listCheckpoints(root)).flatMap(c => c.importReceipt ? [...c.importReceipt.created, ...c.importReceipt.duplicates].map(i => i.key) : []) };
+}
+const normalized = (content) => content.trim().replace(/\s+/g, " ").toLowerCase();
+/** Read-only preflight is shared by proposal review and the durable apply boundary. */
+export async function validateImportItems(root, raw) {
+    await assertCompatibleProject(root);
+    const items = z.array(importItemSchema).max(100).parse(raw);
+    if (new Set(items.map(i => i.key)).size !== items.length)
+        throw new Error("Duplicate import item key");
+    if (hasSecret(JSON.stringify(items)))
+        throw new Error("Potential secret in import proposal");
+    // Validate every proposal and evidence before taking the writer lock or touching durable state.
+    for (const item of items) {
+        assertEnglishProse([item.title, item.content, item.reason, item.skill?.title, item.skill?.description, item.skill?.scope, ...(item.skill?.triggers ?? []), item.skill?.content], "Imported Prodero knowledge");
+        if (item.destination !== "skill" && estimateTokens(knowledgeText(item)) > MAX_CANONICAL_ITEM_TOKENS)
+            throw new Error("Knowledge exceeds canonical token budget");
+        if ((item.scope === "global") !== (item.scopeRef === null))
+            throw new Error("Invalid scope boundary");
+        if (item.destination === "project_instruction" && item.category !== "instruction")
+            throw new Error("Project rules require instruction category");
+        if (item.destination === "skill" ? !item.skill : item.skill !== null || item.activateSkill)
+            throw new Error("Invalid skill destination");
+        if (item.skill && (item.title !== item.skill.title || item.content !== item.skill.content || item.skill.relatedKnowledge.length))
+            throw new Error("Skill proposal must match its reviewed content and use evidence references");
+        if (item.skill && estimateTokens(renderSkill({ ...item.skill, id: "validation", status: "candidate", origin: "legacy", createdAt: "", updatedAt: "", sourceSessions: [], derivedFrom: [], proposedReason: item.reason, reviewedAt: null, reviewReason: null, path: "skills/validation.md" })) > 1500)
+            throw new Error("Skill exceeds context budget; distill the procedure");
+        for (const reference of item.skill?.references ?? [])
+            safeRelative(reference);
+        for (const evidence of item.evidence) {
+            safeRelative(evidence.path);
+            const source = await readSource(root, evidence.path);
+            if (source.sha256 !== evidence.sha256 || evidence.endLine < evidence.startLine || evidence.endLine > source.content.split("\n").length)
+                throw new Error("Source evidence changed or line range invalid");
+        }
+    }
+}
+/** No plugin-supplied write paths, overwrites or model execution. Exact approved input only. */
+export async function applyImport(root, plugin, raw) {
+    const batch = importBatchSchema.parse(raw);
+    if (batch.approvedPlanId !== batch.planId)
+        throw new Error("Exact plan approval required");
+    await validateImportItems(root, batch.items);
+    const batchHash = sha256(JSON.stringify(batch));
+    return withSyncLock(root, async () => {
+        await recoverSyncTransaction(root);
+        const checkpoints = await listCheckpoints(root);
+        const previousImport = checkpoints.find(c => c.importReceipt?.plugin === plugin.id && c.importReceipt?.planId === batch.planId);
+        if (previousImport?.importReceipt) {
+            const receipt = previousImport.importReceipt;
+            if (receipt.batchHash !== batchHash)
+                throw new Error("Applied plan differs from requested batch");
+            await checkedPath(root, ".prodero/index.md");
+            await buildIndex(root);
+            return receipt;
+        }
+        // Reuse Core's chronological ID contract; the receipt is the stable replay key.
+        const checkpoint = sortableId("chk");
+        const state = await readImportState(root);
+        if (state.fingerprint !== batch.stateFingerprint)
+            throw new Error("Prodero state changed since audit; create a fresh reviewed plan");
+        const writes = [];
+        const now = new Date().toISOString();
+        const receipt = { version: 1, plugin: plugin.id, pluginVersion: plugin.version, planId: batch.planId, batchHash, checkpoint, created: [], duplicates: [], before: state.files };
+        const knowledge = [...state.knowledge], skills = [...state.skills];
+        for (const item of batch.items) {
+            const applicability = item.applicability ?? (item.scopeRef ? { kind: "scoped", ...(item.scopeKind === "path" || /[/.]/.test(item.scopeRef) ? { paths: [item.scopeRef] } : { subsystems: [item.scopeRef] }) } : inferApplicability(item.content));
+            const signature = item.destination === "skill" ? `skill:${item.skill.scope}:${normalized(item.content)}` : `${item.destination}:${applicabilityKey({ ...item, applicability })}:${item.category}:${normalized(item.content)}`;
+            const id = `${item.destination === "skill" ? "skill" : "mem"}_import_${sha256(signature).slice(0, 32)}`;
+            const provenance = item.persistEvidence !== false ? item.evidence.map(e => `${e.path}#L${e.startLine}-L${e.endLine}@${e.sha256}`) : [];
+            const derivedFrom = [...provenance, `plugin:${plugin.id}@${plugin.version}/plan:${batch.planId}`];
+            let target, after;
+            if (item.destination === "skill") {
+                const prior = skills.find(s => s.status !== "superseded" && normalized(s.content) === normalized(item.content) && s.scope === item.skill.scope);
+                if (prior) {
+                    const p = `.prodero/${prior.path}`;
+                    receipt.duplicates.push({ key: item.key, path: p, sha256: sha256(await fs.readFile(await checkedPath(root, p), "utf8").catch(() => writes.find(w => `.prodero/${w.path}` === p).after)), destination: item.destination, source: item.evidence });
+                    continue;
+                }
+                const skill = { ...item.skill, id, status: item.activateSkill ? "active" : "candidate", origin: "legacy", createdAt: now, updatedAt: now, sourceSessions: [], derivedFrom, proposedReason: item.reason, reviewedAt: item.activateSkill ? now : null, reviewReason: item.activateSkill ? `Approved plan ${batch.planId}` : null, path: `skills/${id}.md` };
+                after = renderSkill(skill);
+                if (estimateTokens(after) > 1500)
+                    throw new Error("Skill with provenance exceeds context budget");
+                target = skill.path;
+                skills.push(skill);
+            }
+            else {
+                const prior = knowledge.find(k => k.status === "active" && sameApplicability(k, { ...item, applicability }) && k.category === item.category && normalized(k.content) === normalized(item.content));
+                if (prior) {
+                    const p = `.prodero/memory/${prior.id}.md`;
+                    receipt.duplicates.push({ key: item.key, path: p, sha256: state.files[p] ?? sha256(writes.find(w => `.prodero/${w.path}` === p).after), destination: item.destination, source: item.evidence });
+                    continue;
+                }
+                const knowledgeItem = withApplicability({ version: 3, contextPolicy: "relevant", authority: "imported", id, scope: item.scope, scopeRef: item.scopeRef, category: item.category, priority: "normal", status: "active", title: item.title, content: item.content, createdAt: now, updatedAt: now, sourceSessions: [], derivedFrom, supersedes: [], related: [], evidence: provenance.map(ref => ({ kind: "source_file", ref })) }, applicability);
+                if (estimateTokens(knowledgeText(knowledgeItem)) > MAX_CANONICAL_ITEM_TOKENS)
+                    throw new Error("Knowledge exceeds canonical token budget");
+                after = renderKnowledge(knowledgeItem);
+                target = `memory/${id}.md`;
+                knowledge.push(knowledgeItem);
+            }
+            writes.push({ path: target, before: null, after });
+            receipt.created.push({ key: item.key, path: `.prodero/${target}`, sha256: sha256(after), destination: item.destination, source: item.evidence });
+        }
+        await checkedPath(root, ".prodero/index.md");
+        const record = { id: checkpoint, createdAt: now, previousCheckpoint: checkpoints.at(-1)?.id ?? null, sessions: [], consolidation: "not-performed", createdMemory: receipt.created.filter(i => i.destination !== "skill").map(i => i.path.split("/").at(-1).slice(0, -3)), createdSkills: receipt.created.filter(i => i.destination === "skill").map(i => i.path.split("/").at(-1).slice(0, -3)), updatedMemory: [], supersededMemory: [], duplicateMemory: [], conflicts: [], summary: { sessions: 0, candidates: batch.items.length, created: receipt.created.length, consolidated: 0, duplicates: receipt.duplicates.length, superseded: 0, conflicts: 0 }, importReceipt: receipt };
+        try {
+            await commitSyncTransaction(root, [...writes, { path: `checkpoints/${checkpoint}.json`, before: null, after: JSON.stringify(record, null, 2) + "\n" }]);
+            await buildIndex(root);
+        }
+        catch {
+            throw new Error("Import interrupted and may be partially applied. Core retains its recovery journal/checkpoint; retry this exact approved plan after resolving the filesystem or concurrent-write error.");
+        }
+        return receipt;
+    });
+}
+export async function verifyImport(root, receipt) {
+    await assertCompatibleProject(root);
+    const issues = [];
+    if (!/^chk_[0-9]{17}_[a-f0-9]{12}$/.test(receipt.checkpoint))
+        throw new Error("Invalid receipt checkpoint");
+    const record = JSON.parse(await fs.readFile(await checkedPath(root, `.prodero/checkpoints/${receipt.checkpoint}.json`), "utf8"));
+    if (JSON.stringify(record.importReceipt) !== JSON.stringify(receipt))
+        throw new Error("Receipt does not match durable checkpoint");
+    const state = await readImportState(root);
+    for (const item of [...receipt.created, ...receipt.duplicates])
+        if (state.files[item.path] !== item.sha256)
+            issues.push(`Imported item missing or changed: ${item.key}`);
+    const allowed = new Set([...receipt.created.map(i => i.path), `.prodero/checkpoints/${receipt.checkpoint}.json`, ".prodero/index.md"]);
+    for (const p of new Set([...Object.keys(receipt.before), ...Object.keys(state.files)]))
+        if (!allowed.has(p) && receipt.before[p] !== state.files[p])
+            issues.push(`Other canonical state changed: ${p}`);
+    if (new Set(receipt.created.map(i => i.path)).size !== receipt.created.length)
+        issues.push("Duplicate created destination");
+    return { valid: issues.length === 0, issues };
+}
+//# sourceMappingURL=imports.js.map