@prodero/curator 0.0.0-stage → 0.2.0

package/dist/core/consolidation.js

@@ -0,0 +1,315 @@
+import { parseObservationAnnotations as annotations } from "./format.js";
+import { validateApplicability, withApplicability, sameApplicability } from "./applicability.js";
+import { inferredKnowledge } from './knowledge-authority.js';
+import { inheritKnowledgePolicy } from "./knowledge-policy.js";
+import { SyncEvidenceCollector, pressureRecord } from "./memory-pressure.js";
+import { deterministicClassification, reconciliationSignal } from "./reconciliation.js";
+import { assertCompatibleProject } from "./storage.js";
+import { assertNoSecrets } from "./secrets.js";
+import { exactKnowledge, semanticRisk, evidenceSentences } from "./semantic-safety.js";
+import { createSkillCandidate } from "./skills.js";
+import { sortableId } from "./ids.js";
+import { tokens } from "./matching.js";
+import { loadMemoryCatalog, MAX_RELATED_MEMORY_CANDIDATES } from "./memory-index.js";
+import { renderKnowledge, parseKnowledge } from "./memory.js";
+import { safeProjectPath, fileHash } from "./sync-transaction.js";
+import { estimateTokens, knowledgeText } from "./telemetry.js";
+const SECTION_DEFAULTS = {
+    Decisions: { category: "decision", priority: "important", scope: "global" },
+    Lessons: { category: "lesson", priority: "normal", scope: "task" },
+    Discoveries: { category: "state", priority: "normal", scope: "task" },
+    "Project State Changes": { category: "state", priority: "normal", scope: "global" },
+    "Global Instruction Candidates": { category: "instruction", priority: "important", scope: "global" },
+    "Skill Candidates": { category: "instruction", priority: "contextual", scope: "task" },
+};
+function similarity(a, b) {
+    const left = tokens(a);
+    const right = tokens(b);
+    if (!left.size || !right.size)
+        return { jaccard: 0, coverage: 0 };
+    const intersection = [...left].filter((token) => right.has(token)).length;
+    return { jaccard: intersection / (left.size + right.size - intersection), coverage: intersection / Math.min(left.size, right.size) };
+}
+function asScope(value, fallback) { return value === "module" || value === "task" || value === "global" ? value : fallback; }
+function asPriority(value, fallback) { return value === "critical" || value === "important" || value === "normal" || value === "contextual" ? value : fallback; }
+function asCategory(value, fallback) {
+    return value === "principle" || value === "invariant" || value === "decision" || value === "constraint" || value === "lesson" || value === "state" || value === "instruction" ? value : fallback;
+}
+function titleFrom(content) {
+    const clean = content.replace(/[`*_]/g, "").replace(/\s+/g, " ").trim();
+    const sentence = clean.match(/^(.{1,100}?[.!?])(?:\s|$)/)?.[1] ?? clean;
+    return sentence.length > 100 ? `${sentence.slice(0, 97).trim()}…` : sentence;
+}
+export function sessionCandidates(session) {
+    assertNoSecrets(session);
+    const candidates = [];
+    for (const [section, items] of Object.entries(session.sections)) {
+        const defaults = SECTION_DEFAULTS[section];
+        if (!defaults)
+            continue;
+        items.forEach((raw, index) => {
+            const parsed = annotations(raw);
+            if (!parsed.content)
+                return;
+            const explicitFromText = [...parsed.content.matchAll(/\b(?:replaces|supersedes)\s+(mem_[a-zA-Z0-9_]+)/gi)].map((match) => match[1]);
+            let triggers = [];
+            try {
+                const value = JSON.parse(parsed.metadata.triggers ?? "[]");
+                if (Array.isArray(value))
+                    triggers = value.filter((v) => typeof v === "string");
+            }
+            catch { /* Legacy observations have no structured triggers. */ }
+            let relations = [];
+            try {
+                const value = JSON.parse(parsed.metadata.relations ?? "[]");
+                if (Array.isArray(value))
+                    relations = value.filter((v) => Boolean(v && typeof v === "object" && typeof v.type === "string" && typeof v.target === "string"));
+            }
+            catch { /* Legacy observations have no structured relationships. */ }
+            candidates.push({
+                ...(parsed.metadata.applicability ? { applicability: validateApplicability(JSON.parse(parsed.metadata.applicability)) } : {}),
+                instructionCandidate: section === "Global Instruction Candidates",
+                skill: section === "Skill Candidates",
+                description: parsed.metadata.description,
+                triggers,
+                proposedReason: parsed.metadata.proposed_reason,
+                scope: asScope(parsed.metadata.scope, defaults.scope),
+                scopeRef: parsed.metadata.scope_ref ?? null,
+                ...(parsed.metadata.scope_kind ? { scopeKind: parsed.metadata.scope_kind } : {}),
+                category: asCategory(parsed.metadata.category, defaults.category),
+                priority: asPriority(parsed.metadata.priority, defaults.priority),
+                title: parsed.metadata.title ?? titleFrom(parsed.content),
+                content: parsed.content,
+                sessionId: session.id,
+                derivedFrom: `${session.id}#${parsed.metadata.id ?? `${section.replace(/\s+/g, "-").toLowerCase()}-${index + 1}`}`,
+                supersedes: unique([...(parsed.metadata.supersedes?.split(",").map((value) => value.trim()).filter(Boolean) ?? []), ...explicitFromText]),
+                relations,
+            });
+        });
+    }
+    return candidates.map(c => c.applicability && !c.skill && !c.instructionCandidate ? withApplicability(c, c.applicability) : c);
+}
+function unique(values) { return [...new Set(values)].sort(); }
+function uniqueRelations(values) {
+    return [...new Map(values.map((relation) => [`${relation.type}:${relation.target}`, relation])).values()]
+        .sort((a, b) => a.type.localeCompare(b.type) || a.target.localeCompare(b.target));
+}
+function explicitSuperseded(candidate, active) {
+    return active.filter((item) => candidate.supersedes.includes(item.id) || new RegExp(`\\b(?:replaces|supersedes)\\s+${item.id.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`, "i").test(candidate.content));
+}
+/** Core-owned signals; similarity and merge guards are never conflict verdicts. */
+export function candidateReviewSignals(candidate, related) {
+    const signals = [];
+    if (candidate.instructionCandidate)
+        signals.push({ kind: 'instruction-candidate', detail: 'Proposed instruction; not an active required policy.', severity: 'info', requiresReview: true });
+    if (candidate.skill)
+        signals.push({ kind: 'skill-candidate', detail: 'Proposed skill; activation requires separate review.', severity: 'info', requiresReview: true });
+    for (const old of related) {
+        const category = deterministicClassification(candidate, old);
+        // Retrieval can include loose neighbors; keep unrelated observations quiet.
+        if (category === 'unresolved' && similarity(candidate.content, old.content).coverage < 0.3)
+            continue;
+        signals.push(reconciliationSignal(category, old));
+    }
+    return signals;
+}
+function mergeEligible(candidate, old) {
+    if (semanticRisk(candidate.content, old.content))
+        return false;
+    const incoming = evidenceSentences(candidate.content);
+    const existing = evidenceSentences(old.content);
+    // Only an unchanged sentence prefix plus complementary complete sentences is safe without reasoning.
+    const smaller = incoming.length < existing.length ? incoming : existing;
+    const larger = incoming.length < existing.length ? existing : incoming;
+    return smaller.length > 0 && smaller.every((sentence, index) => sentence === larger[index])
+        && estimateTokens(`${old.title}\n${old.content}\n${candidate.content}`) <= 2_000;
+}
+function segments(value) {
+    return value.split(/(?:\r?\n){2,}|(?<=[.!?])\s+(?=[\p{Lu}\p{N}`])/u).map((part) => part.trim()).filter(Boolean);
+}
+function compactContribution(existing, incoming) {
+    const kept = segments(existing);
+    const seen = new Set(kept.map(exactKnowledge));
+    const added = [];
+    let removed = 0;
+    for (const segment of segments(incoming)) {
+        const key = exactKnowledge(segment);
+        if (!key || seen.has(key)) {
+            removed += 1;
+            continue;
+        }
+        seen.add(key);
+        kept.push(segment);
+        added.push(segment);
+    }
+    return { content: kept.join("\n\n"), added: added.join("\n\n"), removed };
+}
+async function deterministicConsolidate(root, sessions, now = new Date(), processedSources = new Set()) {
+    const candidates = sessions.flatMap(sessionCandidates).filter((candidate) => !processedSources.has(candidate.derivedFrom));
+    const catalog = await loadMemoryCatalog(root);
+    const writes = new Map();
+    const pressure = new SyncEvidenceCollector();
+    const changedRecords = new Map();
+    const stageKnowledge = async (item) => {
+        const relative = `local/memory/${item.id}.md`;
+        const after = renderKnowledge(item);
+        parseKnowledge(after, `${item.id}.md`);
+        const existing = writes.get(relative);
+        const before = existing ? existing.before : catalog.originalHashes.get(item.id) ?? await fileHash(await safeProjectPath(root, relative));
+        writes.set(relative, { path: relative, before, after });
+        catalog.stage(item, "local");
+        changedRecords.set(item.id, pressureRecord({ ...item, layer: "local", estimatedTokens: estimateTokens(knowledgeText(item)), relations: item.relations ?? [] }));
+    };
+    const result = {
+        candidates: 0, created: [], createdSkills: [], skillCandidateTokensAdded: 0, updated: [], consolidated: [], superseded: [], duplicates: [], conflicts: [], conflictCount: 0,
+        memoryFilesScanned: catalog.filesScanned, memoryBodiesIndexed: catalog.bodiesIndexed, memoryBodiesLoaded: 0, relatedCandidatesSelectedMax: 0,
+        observationsProcessed: 0, rawSessionTokens: 0, rawKnowledgeTokens: 0, rawSkillCandidateTokens: 0, canonicalTokensAdded: 0,
+        compactedSegments: 0, conservativeMergesSkipped: 0,
+    };
+    result.candidates = candidates.length;
+    result.observationsProcessed = candidates.length;
+    result.rawKnowledgeTokens = candidates.filter((candidate) => !candidate.skill).reduce((total, candidate) => total + estimateTokens(candidate.content), 0);
+    result.rawSkillCandidateTokens = candidates.filter((candidate) => candidate.skill).reduce((total, candidate) => total + estimateTokens(candidate.content), 0);
+    result.rawSessionTokens = candidates.reduce((total, candidate) => total + estimateTokens(candidate.content), 0);
+    for (const candidate of candidates) {
+        if (candidate.instructionCandidate) {
+            (result.instructionCandidates ??= []).push({ ...candidate, id: `candidate_${result.instructionCandidates?.length ?? 0}`, kind: "instruction_candidate", sources: [candidate.derivedFrom], evidence: [{ source: candidate.derivedFrom, quote: candidate.content }] });
+            continue;
+        }
+        if (candidate.skill) {
+            const saved = await createSkillCandidate(root, {
+                title: candidate.title,
+                description: candidate.description ?? candidate.content.replace(/\s+/g, " ").slice(0, 300),
+                content: candidate.content,
+                scope: candidate.scopeRef ?? "project",
+                triggers: candidate.triggers,
+            }, { sessionId: candidate.sessionId, derivedFrom: candidate.derivedFrom, proposedReason: candidate.proposedReason }, now, writes);
+            if (saved.created) {
+                result.createdSkills.push(saved.skill.id);
+                result.skillCandidateTokensAdded += estimateTokens(saved.skill.content);
+            }
+            continue;
+        }
+        const selected = catalog.selectRelated({
+            scope: candidate.scope,
+            scopeRef: candidate.scopeRef,
+            scopeKind: candidate.scopeKind,
+            applicability: candidate.applicability ?? { kind: "legacy" },
+            category: candidate.category,
+            title: candidate.title,
+            content: candidate.content,
+            explicitIds: candidate.supersedes,
+        }, MAX_RELATED_MEMORY_CANDIDATES);
+        // An explicit replacement must survive the complete write path. Never silently
+        // drop a missing/historical target and save an unrelated active record instead.
+        const unavailable = candidate.supersedes.filter(id => !selected.some(item => item.id === id));
+        if (unavailable.length)
+            throw new Error(`Explicit supersedes target is missing or no longer current: ${unavailable.join(", ")}. Session remains pending; use current memory IDs.`);
+        result.relatedCandidatesSelectedMax = Math.max(result.relatedCandidatesSelectedMax, selected.length);
+        const active = await Promise.all(selected.map((entry) => catalog.load(entry.id)));
+        for (const signal of candidateReviewSignals(candidate, active)) {
+            const target = selected.find(item => item.id === signal.memory?.id);
+            if (target)
+                pressure.observe(candidate.derivedFrom, pressureRecord(target), signal.kind);
+        }
+        const localIds = new Set(selected.filter((entry) => entry.layer === "local").map((entry) => entry.id));
+        const exact = active.find((item) => item.scope === candidate.scope && item.scopeRef === candidate.scopeRef && item.scopeKind === candidate.scopeKind && item.category === candidate.category && sameApplicability(item, candidate) && exactKnowledge(item.content) === exactKnowledge(candidate.content));
+        if (exact && !candidate.supersedes.length) {
+            if (localIds.has(exact.id)) {
+                exact.sourceSessions = unique([...exact.sourceSessions, candidate.sessionId]);
+                exact.derivedFrom = unique([...exact.derivedFrom, candidate.derivedFrom]);
+                exact.relations = uniqueRelations([...(exact.relations ?? []), ...candidate.relations]);
+                exact.updatedAt = now.toISOString();
+                await stageKnowledge(exact);
+                result.updated.push(exact.id);
+            }
+            result.duplicates.push(exact.id);
+            continue;
+        }
+        const supersededTargets = explicitSuperseded(candidate, active);
+        const replaced = supersededTargets.filter((item) => localIds.has(item.id));
+        const id = sortableId("mem", now);
+        const item = {
+            id,
+            scope: candidate.scope,
+            scopeRef: candidate.scopeRef,
+            scopeKind: candidate.scopeKind,
+            applicability: candidate.applicability ?? { kind: "legacy" },
+            version: 3,
+            category: candidate.category,
+            priority: candidate.priority,
+            status: "active",
+            title: candidate.title,
+            content: candidate.content,
+            createdAt: now.toISOString(),
+            updatedAt: now.toISOString(),
+            sourceSessions: [candidate.sessionId],
+            derivedFrom: [candidate.derivedFrom],
+            supersedes: unique(supersededTargets.flatMap((old) => [old.id, ...old.supersedes])),
+            related: [],
+            relations: uniqueRelations(candidate.relations),
+        };
+        Object.assign(item, withApplicability(inferredKnowledge(item), item.applicability));
+        inheritKnowledgePolicy(item, supersededTargets);
+        if (supersededTargets.length) {
+            for (const old of replaced) {
+                old.status = "superseded";
+                old.updatedAt = now.toISOString();
+                old.related = unique([...old.related, id]);
+                await stageKnowledge(old);
+                result.superseded.push(old.id);
+                result.updated.push(old.id);
+            }
+        }
+        else {
+            const sameBoundary = active.filter((old) => localIds.has(old.id) && old.scope === candidate.scope && old.scopeRef === candidate.scopeRef && old.scopeKind === candidate.scopeKind && old.category === candidate.category && sameApplicability(old, candidate));
+            const overlapping = sameBoundary.find((old) => mergeEligible(candidate, old));
+            if (overlapping) {
+                const compacted = compactContribution(overlapping.content, candidate.content);
+                overlapping.content = compacted.content;
+                result.compactedSegments += compacted.removed;
+                result.canonicalTokensAdded += estimateTokens(compacted.added);
+                overlapping.sourceSessions = unique([...overlapping.sourceSessions, candidate.sessionId]);
+                overlapping.derivedFrom = unique([...overlapping.derivedFrom, candidate.derivedFrom]);
+                overlapping.relations = uniqueRelations([...(overlapping.relations ?? []), ...candidate.relations]);
+                overlapping.updatedAt = now.toISOString();
+                await stageKnowledge(overlapping);
+                result.updated.push(overlapping.id);
+                if (compacted.added)
+                    result.consolidated.push(overlapping.id);
+                else
+                    result.duplicates.push(overlapping.id);
+                continue;
+            }
+            if (sameBoundary.some((old) => {
+                const score = similarity(`${candidate.title} ${candidate.content}`, `${old.title} ${old.content}`);
+                return score.jaccard >= 0.35 || score.coverage >= 0.5;
+            }))
+                result.conservativeMergesSkipped += 1;
+            // Similarity and protected-token changes only limit safe merging. Without
+            // semantic evidence, keep both records active for later reconciliation.
+        }
+        await stageKnowledge(item);
+        result.created.push(id);
+        result.canonicalTokensAdded += estimateTokens(knowledgeText(item));
+    }
+    result.created = unique(result.created);
+    result.updated = unique(result.updated);
+    result.consolidated = unique(result.consolidated);
+    result.superseded = unique(result.superseded);
+    result.duplicates = unique(result.duplicates);
+    result.conflicts = unique(result.conflicts);
+    result.memoryBodiesLoaded = catalog.bodiesLoaded;
+    result.writes = [...writes.values()];
+    result.memoryPressure = pressure.finish([...changedRecords.values()], result);
+    return result;
+}
+export const deterministicConsolidator = {
+    name: "deterministic-v1",
+    consolidate: deterministicConsolidate,
+};
+export async function consolidateSessions(root, sessions, now = new Date(), consolidator = deterministicConsolidator, processedSources = new Set()) {
+    await assertCompatibleProject(root);
+    return consolidator.consolidate(root, sessions, now, processedSources);
+}
+//# sourceMappingURL=consolidation.js.map

package/dist/core/constants.js

@@ -0,0 +1,21 @@
+import { readFileSync } from "node:fs";
+const packageIdentity = JSON.parse(readFileSync(new URL("../../package.json", import.meta.url), "utf8"));
+const releaseIdentity = JSON.parse(readFileSync(new URL("../../prodero.release.json", import.meta.url), "utf8"));
+if (releaseIdentity.formatVersion !== 1 || !Number.isInteger(releaseIdentity.schemaVersion) || releaseIdentity.schemaVersion < 1 || !Number.isInteger(releaseIdentity.knowledgeSchemaVersion) || releaseIdentity.knowledgeSchemaVersion < 1)
+    throw new Error("Invalid prodero.release.json");
+export const PRODERO_VERSION = packageIdentity.version;
+export const PRODERO_SCHEMA_VERSION = releaseIdentity.schemaVersion;
+export const PRODERO_KNOWLEDGE_SCHEMA_VERSION = releaseIdentity.knowledgeSchemaVersion;
+export const PRODERO_PACKAGE_NAME = packageIdentity.name;
+export const PRODERO_DIRECTORY = ".prodero";
+export const MANAGED_START = "<!-- prodero:start -->";
+export const MANAGED_END = "<!-- prodero:end -->";
+export const PROJECT_DIRECTORIES = [
+    "memory",
+    "sessions",
+    "skills",
+    "checkpoints",
+    "local/memory",
+    ".cache",
+];
+//# sourceMappingURL=constants.js.map

package/dist/core/context-allocation.js

@@ -0,0 +1,54 @@
+/** Resource allocation only: applicability, authority and ranking remain in Core retrieval. */
+export const REQUIRED_HARD_TOKENS = 16_000;
+export const REQUIRED_HARD_ITEMS = 128;
+export const RELEVANT_HARD_ITEMS = 128;
+export const SKILL_HARD_TOKENS = 1_500;
+export class ContextAllocation {
+    total;
+    reserve;
+    used = 0;
+    requiredTokens = 0;
+    requiredItems = 0;
+    skillTokens = 0;
+    skillItems = 0;
+    relevantItems = 0;
+    constructor(total, reserve) {
+        this.total = total;
+        this.reserve = reserve;
+    }
+    get remaining() { return Math.max(0, this.total - this.used); }
+    take(bucket, cost) {
+        if (bucket === 'required' && this.requiredTokens + cost > REQUIRED_HARD_TOKENS)
+            return 'required-token-ceiling';
+        if (bucket === 'required' && this.requiredItems >= REQUIRED_HARD_ITEMS)
+            return 'required-item-ceiling';
+        if (bucket === 'relevant' && this.relevantItems >= RELEVANT_HARD_ITEMS)
+            return 'relevant-item-ceiling';
+        if (bucket === 'skill' && this.skillItems >= 4)
+            return 'skill-item-ceiling';
+        if (bucket === 'skill' && this.skillTokens + cost > SKILL_HARD_TOKENS)
+            return 'skill-token-ceiling';
+        if (cost > this.remaining)
+            return 'total-capacity';
+        this.used += cost;
+        if (bucket === 'required') {
+            this.requiredTokens += cost;
+            this.requiredItems++;
+        }
+        if (bucket === 'relevant')
+            this.relevantItems++;
+        if (bucket === 'skill') {
+            this.skillTokens += cost;
+            this.skillItems++;
+        }
+        return null;
+    }
+}
+/** Approximate information novelty, not semantic equivalence. Never applied to Required. */
+export function marginalValue(words, query, supplied, rankScore, peakScore) {
+    const content = [...new Set(words.filter(t => !query.has(t) && !/^\d+$/.test(t)))];
+    const novel = content.filter(t => !supplied.has(t));
+    const fraction = content.length ? novel.length / content.length : 0;
+    return { useful: rankScore >= Math.max(12, peakScore * 0.3) && novel.length >= 3 && fraction >= 0.2, novelTerms: novel.length, novelFraction: fraction };
+}
+//# sourceMappingURL=context-allocation.js.map

package/dist/core/context-snapshot.js

@@ -0,0 +1,12 @@
+/** Associate the exact resolver result with its prepared session without copying canonical bodies. */
+export function associatePreparedContext(context, sessionId) {
+    context.snapshot = { version: 1, sessionId, task: context.query };
+    return context;
+}
+/** UI-safe, I/O-free lookup. Missing old/disposable state never triggers context resolution. */
+export function preparedContextSnapshot(context) {
+    if (!context || context.snapshot?.version !== 1 || !context.snapshot.sessionId)
+        return null;
+    return context;
+}
+//# sourceMappingURL=context-snapshot.js.map