@kontourai/lookout 0.1.0

package/dist/src/cli.js

@@ -0,0 +1,173 @@
+import { open } from "node:fs/promises";
+import { createCheckRunner } from "./check-runner.js";
+import { loadRegistry } from "./registry.js";
+import { createLookoutSnapshotStore } from "./snapshot-store.js";
+import { createObservationStore } from "./observation-store.js";
+import { createSurveyEmitter } from "./survey-emission.js";
+export async function runCli(options = {}) {
+    const argv = options.argv ?? process.argv.slice(2);
+    const stdout = options.stdout ?? process.stdout;
+    const stderr = options.stderr ?? process.stderr;
+    const parsed = parseArgs(argv);
+    if (typeof parsed === "string") {
+        stderr.write(`${parsed}\n`);
+        return 2;
+    }
+    let registry;
+    try {
+        registry = await (options.loadRegistry ?? loadRegistry)(parsed.registryPath);
+    }
+    catch (error) {
+        stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+        return 1;
+    }
+    if (parsed.command === "emit-survey") {
+        const source = registry.get(parsed.id);
+        if (!source) {
+            stderr.write(`Unknown source id: ${parsed.id}\n`);
+            return 1;
+        }
+        let value;
+        try {
+            value = await (options.readObservation ?? readObservation)(parsed.observationPath);
+        }
+        catch (error) {
+            stderr.write(`Could not read observation: ${error instanceof Error ? error.message : String(error)}\n`);
+            return 1;
+        }
+        const result = await (options.emitSurvey ?? emitSurvey)(parsed.id, value, registry, parsed.observationRoot);
+        if (!result.ok) {
+            stderr.write(`${result.error.kind}: ${result.error.message}\n`);
+            return 1;
+        }
+        stdout.write(`${JSON.stringify(result.value)}\n`);
+        return 0;
+    }
+    const runner = options.runner ?? createCheckRunner({
+        store: createLookoutSnapshotStore(parsed.snapshotRoot),
+    });
+    if (parsed.all) {
+        const results = await runner.checkAll(registry.list());
+        for (const result of results)
+            stdout.write(`${JSON.stringify(result)}\n`);
+        return 0;
+    }
+    const source = registry.get(parsed.id);
+    if (!source) {
+        stderr.write(`Unknown source id: ${parsed.id}\n`);
+        return 1;
+    }
+    stdout.write(`${JSON.stringify(await runner.check(source))}\n`);
+    return 0;
+}
+function parseArgs(argv) {
+    if (argv[0] === "emit-survey")
+        return parseEmitArgs(argv);
+    if (argv[0] !== "check")
+        return "Usage: lookout check <id>|--all [--registry path] [--snapshot-root path]\n       lookout emit-survey <id> --observation <path|-> [--registry path] [--observation-root path]";
+    const parsed = { command: "check", all: false };
+    for (let index = 1; index < argv.length; index += 1) {
+        const arg = argv[index];
+        if (arg === "--all") {
+            if (parsed.all)
+                return "--all may be specified only once";
+            parsed.all = true;
+        }
+        else if (arg === "--registry" || arg === "--snapshot-root") {
+            const value = argv[index + 1];
+            if (!value || value.startsWith("--"))
+                return `${arg} requires a path`;
+            if (arg === "--registry")
+                parsed.registryPath = value;
+            else
+                parsed.snapshotRoot = value;
+            index += 1;
+        }
+        else if (arg.startsWith("--")) {
+            return `Unknown option: ${arg}`;
+        }
+        else if (parsed.id) {
+            return "Only one source id may be checked at a time";
+        }
+        else {
+            parsed.id = arg;
+        }
+    }
+    if (parsed.all && parsed.id)
+        return "A source id and --all are mutually exclusive";
+    if (!parsed.all && !parsed.id)
+        return "check requires a source id or --all";
+    return parsed;
+}
+function parseEmitArgs(argv) {
+    let id;
+    let observationPath;
+    let registryPath;
+    let observationRoot;
+    for (let index = 1; index < argv.length; index += 1) {
+        const arg = argv[index];
+        if (arg === "--registry" || arg === "--observation" || arg === "--observation-root") {
+            const value = argv[++index];
+            if (!value || (value.startsWith("--") && value !== "-"))
+                return `${arg} requires a path`;
+            if (arg === "--registry")
+                registryPath = value;
+            else if (arg === "--observation")
+                observationPath = value;
+            else
+                observationRoot = value;
+        }
+        else if (arg.startsWith("--"))
+            return `Unknown option: ${arg}`;
+        else if (id)
+            return "Only one source id may be emitted at a time";
+        else
+            id = arg;
+    }
+    if (!id)
+        return "emit-survey requires a source id";
+    if (!observationPath)
+        return "emit-survey requires --observation <path|->";
+    return { command: "emit-survey", id, observationPath, registryPath, observationRoot };
+}
+async function readObservation(file) {
+    const maxBytes = 1024 * 1024;
+    const text = file === "-" ? await new Promise((resolve, reject) => { let body = ""; let bytes = 0; let rejected = false; process.stdin.setEncoding("utf8"); process.stdin.on("data", (chunk) => { if (rejected)
+        return; bytes += Buffer.byteLength(chunk); if (bytes > maxBytes) {
+        rejected = true;
+        reject(new Error(`observation exceeds ${maxBytes} bytes`));
+        return;
+    } body += chunk; }); process.stdin.on("end", () => { if (!rejected)
+        resolve(body); }); process.stdin.on("error", reject); }) : await readBoundedFile(file, maxBytes);
+    if (Buffer.byteLength(text) > maxBytes)
+        throw new Error(`observation exceeds ${maxBytes} bytes`);
+    return JSON.parse(text);
+}
+async function readBoundedFile(file, maxBytes) {
+    const handle = await open(file, "r");
+    try {
+        const buffer = Buffer.allocUnsafe(maxBytes + 1);
+        const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
+        if (bytesRead > maxBytes)
+            throw new Error(`observation exceeds ${maxBytes} bytes`);
+        return buffer.subarray(0, bytesRead).toString("utf8");
+    }
+    finally {
+        await handle.close();
+    }
+}
+function cliEntities(observation) {
+    const grouped = new Map();
+    for (const proposal of observation.proposals) {
+        const key = proposal.pathIndices?.length ? proposal.pathIndices.join(".") : "root";
+        grouped.set(key, [...(grouped.get(key) ?? []), proposal]);
+    }
+    return [...grouped].map(([key, proposals]) => ({ key, proposals }));
+}
+async function emitSurvey(sourceId, value, registry, observationRoot) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        return { ok: false, error: { kind: "invalid-input", message: "Observation document must be an object" } };
+    const document = value;
+    const source = registry.get(sourceId);
+    return createSurveyEmitter({ store: createObservationStore({ root: observationRoot }) }).emit({ source, current: document.observation, check: document.check, callbacks: { selectEntities: cliEntities, entityIdentity: (entity) => entity.key, proposalsFor: (entity) => entity.proposals, fieldIdentity: (_entity, proposal) => proposal.fieldPath } });
+}

package/dist/src/index.d.ts

@@ -0,0 +1,20 @@
+export { runCli } from "./cli.js";
+export type { RunCliOptions } from "./cli.js";
+export { createCheckRunner } from "./check-runner.js";
+export type { CheckRunner, CreateCheckRunnerOptions, FetchSource } from "./check-runner.js";
+export type { ChangedResult, CheckResult, CheckResultCommon, ErrorResult, LookoutErrorKind, Unchanged304Result, UnchangedHashResult, } from "./check-result.js";
+export { defaultProviderResolver } from "./provider-resolution.js";
+export type { ProviderResolver } from "./provider-resolution.js";
+export { loadRegistry, LookoutRegistry, parseRegistry, RegistryValidationError, } from "./registry.js";
+export type { LookoutRegistryDocument, LookoutSource, LookoutSourceKind, RenderPolicy, } from "./registry.js";
+export { createLookoutSnapshotStore } from "./snapshot-store.js";
+export { canonicalValueKey } from "./canonical-value.js";
+export type { CanonicalValueKey, CanonicalValueFormatVersion, CanonicalValueResult, DiffKernelError, DiffKernelErrorKind, DiffResult, IdentityResult, } from "./canonical-value.js";
+export { compareStructural, diffKeyedMultiset } from "./structural-diff.js";
+export type { KeyedMultisetFacts, KeyedMultisetOptions, StructuralComparison, StructuralComparisonOptions, StructuralFacts, } from "./structural-diff.js";
+export { diffProposalSets, extractionProposalIdentity } from "./proposal-diff.js";
+export type { FieldChangedEvent, FieldChangeKind, NewEntityAppearedEvent, ProposalDiffEvent, ProposalEvidence, ProposalIdentity, ProposalOccurrencePair, ProposalSetDiff, ProposalSetDiffInput, ProposalSetFacts, ProposalSetObservation, ProvenanceChangeFact, } from "./proposal-diff.js";
+export { createObservationStore } from "./observation-store.js";
+export type { CreateObservationStoreOptions, ObservationCheckAnchor, ObservationStore, ObservationStoreError, ObservationStoreErrorKind, ObservationStoreResult, ProposalObservationRecordInput, StoredProposalObservationV1 } from "./observation-store.js";
+export { createSurveyEmitter } from "./survey-emission.js";
+export type { BaselineEstablishedFact, CreateSurveyEmitterOptions, EmissionError, EmissionErrorKind, EmissionFact, EmissionResult, EmissionSuccess, EmitSurveyInput, SurveyEmitter } from "./survey-emission.js";

package/dist/src/index.js

@@ -0,0 +1,10 @@
+export { runCli } from "./cli.js";
+export { createCheckRunner } from "./check-runner.js";
+export { defaultProviderResolver } from "./provider-resolution.js";
+export { loadRegistry, LookoutRegistry, parseRegistry, RegistryValidationError, } from "./registry.js";
+export { createLookoutSnapshotStore } from "./snapshot-store.js";
+export { canonicalValueKey } from "./canonical-value.js";
+export { compareStructural, diffKeyedMultiset } from "./structural-diff.js";
+export { diffProposalSets, extractionProposalIdentity } from "./proposal-diff.js";
+export { createObservationStore } from "./observation-store.js";
+export { createSurveyEmitter } from "./survey-emission.js";

package/dist/src/observation-store.d.ts

@@ -0,0 +1,55 @@
+import type { ExtractionProposal } from "@kontourai/traverse";
+import type { ProposalSetObservation } from "./proposal-diff.js";
+export interface ObservationCheckAnchor {
+    readonly checkedAt: string;
+    readonly resultKind: "changed" | "unchanged-hash";
+    readonly currentSnapshotRef: string;
+}
+export interface ProposalObservationRecordInput {
+    readonly observation: ProposalSetObservation;
+    readonly recordedAt: string;
+    readonly check: ObservationCheckAnchor;
+}
+export interface StoredProposalObservationV1 {
+    readonly version: 1;
+    readonly observationId: string;
+    readonly sourceKey: string;
+    readonly sourceId: string;
+    readonly snapshotRef: string;
+    readonly observedAt: string;
+    readonly recordedAt: string;
+    readonly check: ObservationCheckAnchor;
+    readonly proposals: readonly ExtractionProposal[];
+}
+export type ObservationStoreErrorKind = "invalid-input" | "corrupt-state" | "continuity-conflict" | "io-error";
+export interface ObservationStoreError {
+    readonly kind: ObservationStoreErrorKind;
+    readonly message: string;
+    readonly cause?: unknown;
+}
+export type ObservationStoreResult<T> = {
+    readonly ok: true;
+    readonly value: T;
+    readonly warnings?: readonly string[];
+} | {
+    readonly ok: false;
+    readonly error: ObservationStoreError;
+};
+export interface ObservationStore {
+    loadLatest(sourceId: string): Promise<ObservationStoreResult<StoredProposalObservationV1 | null>>;
+    commit(input: ProposalObservationRecordInput, expectedPriorId: string | null): Promise<ObservationStoreResult<StoredProposalObservationV1>>;
+}
+export interface ObservationStoreFaults {
+    readonly beforeSerialize?: () => void;
+    readonly beforeTempWrite?: (kind: "record" | "pointer") => void;
+    readonly beforeFileSync?: (kind: "record" | "pointer") => void;
+    readonly beforeRecordRename?: () => void;
+    readonly beforePointerRename?: () => void;
+    readonly beforeDirectorySync?: (kind: "record" | "pointer") => void;
+    readonly beforePrune?: () => void;
+}
+export interface CreateObservationStoreOptions {
+    readonly root?: string;
+    readonly faults?: ObservationStoreFaults;
+}
+export declare function createObservationStore(options?: CreateObservationStoreOptions): ObservationStore;

package/dist/src/observation-store.js

@@ -0,0 +1,241 @@
+import { createHash } from "node:crypto";
+import { lstat, mkdir, open, readFile, readdir, rename, rm, unlink } from "node:fs/promises";
+import path from "node:path";
+function sourceKey(sourceId) {
+    return `${encodeURIComponent(sourceId).replaceAll("%", "_").slice(0, 48)}-${createHash("sha256").update(sourceId).digest("hex").slice(0, 16)}`;
+}
+function stable(value) {
+    if (Array.isArray(value))
+        return value.map(stable);
+    if (value && typeof value === "object")
+        return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => [key, stable(item)]));
+    return value;
+}
+function canonical(value) { return `${JSON.stringify(stable(value))}\n`; }
+function digest(body) { return createHash("sha256").update(canonical(body)).digest("hex"); }
+function buildRecord(input) {
+    try {
+        const { observation, check } = input;
+        if (!observation || typeof observation.sourceId !== "string" || observation.sourceId === "" || typeof observation.snapshotRef !== "string" || observation.snapshotRef === "" || typeof observation.observedAt !== "string" || observation.observedAt === "" || !Array.isArray(observation.proposals) || observation.proposals.some((proposal) => !validProposal(proposal))) {
+            return { ok: false, error: { kind: "invalid-input", message: "Current proposal observation is malformed" } };
+        }
+        if (!check || check.currentSnapshotRef !== observation.snapshotRef || typeof check.checkedAt !== "string" || (check.resultKind !== "changed" && check.resultKind !== "unchanged-hash") || typeof input.recordedAt !== "string") {
+            return { ok: false, error: { kind: "invalid-input", message: "Check anchor must match the current observation snapshot" } };
+        }
+        const proposals = [...observation.proposals].sort((a, b) => canonical(a).localeCompare(canonical(b)));
+        const body = { version: 1, sourceKey: sourceKey(observation.sourceId), sourceId: observation.sourceId, snapshotRef: observation.snapshotRef, observedAt: observation.observedAt, recordedAt: input.recordedAt, check, proposals };
+        try {
+            return { ok: true, value: { ...body, observationId: digest(body) } };
+        }
+        catch (cause) {
+            return { ok: false, error: { kind: "invalid-input", message: "Observation could not be serialized", cause } };
+        }
+    }
+    catch (cause) {
+        return { ok: false, error: { kind: "invalid-input", message: "Current proposal observation could not be inspected", cause } };
+    }
+}
+function validProposal(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        return false;
+    const item = value;
+    if (typeof item.fieldPath !== "string" || item.fieldPath === "" || typeof item.confidence !== "number" || !Number.isFinite(item.confidence) || typeof item.extractor !== "string" || item.extractor === "")
+        return false;
+    if (item.pathIndices !== undefined && (!Array.isArray(item.pathIndices) || item.pathIndices.some((part) => !Number.isSafeInteger(part) || part < 0)))
+        return false;
+    if (!item.provenance || typeof item.provenance !== "object" || Array.isArray(item.provenance))
+        return false;
+    const provenance = item.provenance;
+    return typeof provenance.locator === "string" && provenance.locator !== "" && typeof provenance.excerpt === "string";
+}
+function validate(value, expectedSourceId) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        return { ok: false, error: { kind: "corrupt-state", message: "Stored observation is not an object" } };
+    const item = value;
+    if (item.version !== 1 || item.sourceId !== expectedSourceId || item.sourceKey !== sourceKey(expectedSourceId) || typeof item.observationId !== "string" || !/^[a-f0-9]{64}$/.test(item.observationId) || typeof item.snapshotRef !== "string" || item.snapshotRef === "" || typeof item.observedAt !== "string" || item.observedAt === "" || typeof item.recordedAt !== "string" || item.recordedAt === "" || !Array.isArray(item.proposals) || item.proposals.some((proposal) => !validProposal(proposal)) || !item.check || typeof item.check !== "object" || typeof item.check.checkedAt !== "string" || item.check.checkedAt === "" || (item.check.resultKind !== "changed" && item.check.resultKind !== "unchanged-hash") || item.check.currentSnapshotRef !== item.snapshotRef) {
+        return { ok: false, error: { kind: "corrupt-state", message: "Stored observation schema or continuity is invalid" } };
+    }
+    const { observationId, ...body } = item;
+    try {
+        if (digest(body) !== observationId)
+            return { ok: false, error: { kind: "corrupt-state", message: "Stored observation digest does not match its body" } };
+    }
+    catch (cause) {
+        return { ok: false, error: { kind: "corrupt-state", message: "Stored observation is not serializable", cause } };
+    }
+    return { ok: true, value: item };
+}
+async function rejectSymlink(file, allowMissing = true) {
+    try {
+        if ((await lstat(file)).isSymbolicLink())
+            throw new Error(`Refusing symbolic link: ${file}`);
+    }
+    catch (cause) {
+        if (allowMissing && cause.code === "ENOENT")
+            return;
+        throw cause;
+    }
+}
+async function syncDirectory(dir, fault) { fault?.(); const handle = await open(dir, "r"); try {
+    await handle.sync();
+}
+finally {
+    await handle.close();
+} }
+async function atomicWrite(file, bytes, kind, faults) {
+    const temp = `${file}.tmp-${process.pid}-${Math.random().toString(16).slice(2)}`;
+    await rejectSymlink(file);
+    faults?.beforeTempWrite?.(kind);
+    const handle = await open(temp, "wx", 0o600);
+    try {
+        await handle.writeFile(bytes, "utf8");
+        faults?.beforeFileSync?.(kind);
+        await handle.sync();
+        await handle.close();
+        kind === "record" ? faults?.beforeRecordRename?.() : faults?.beforePointerRename?.();
+        await rename(temp, file);
+        try {
+            await syncDirectory(path.dirname(file), () => faults?.beforeDirectorySync?.(kind));
+            return { renamed: true };
+        }
+        catch (cause) {
+            return { renamed: true, durabilityWarning: `${kind} rename completed but directory fsync failed: ${cause instanceof Error ? cause.message : String(cause)}` };
+        }
+    }
+    catch (error) {
+        await handle.close().catch(() => undefined);
+        await unlink(temp).catch(() => undefined);
+        throw error;
+    }
+}
+export function createObservationStore(options = {}) {
+    const root = options.root ?? path.join(process.cwd(), ".kontourai", "lookout", "observations");
+    async function loadLatest(sourceId) {
+        try {
+            const dir = path.join(root, sourceKey(sourceId));
+            await rejectSymlink(root);
+            await rejectSymlink(dir);
+            let pointerText;
+            const pointerPath = path.join(dir, "latest.json");
+            try {
+                await rejectSymlink(pointerPath);
+                pointerText = await readFile(pointerPath, "utf8");
+            }
+            catch (cause) {
+                if (cause.code === "ENOENT")
+                    return { ok: true, value: null };
+                throw cause;
+            }
+            const pointer = JSON.parse(pointerText);
+            if (pointer.version !== 1 || pointer.sourceId !== sourceId || typeof pointer.observationId !== "string" || !/^[a-f0-9]{64}$/.test(pointer.observationId))
+                return { ok: false, error: { kind: "corrupt-state", message: "Latest pointer is invalid" } };
+            const recordPath = path.join(dir, `${pointer.observationId}.json`);
+            await rejectSymlink(recordPath);
+            const parsed = JSON.parse(await readFile(recordPath, "utf8"));
+            const checked = validate(parsed, sourceId);
+            if (!checked.ok)
+                return checked;
+            if (checked.value.observationId !== pointer.observationId)
+                return { ok: false, error: { kind: "corrupt-state", message: "Latest pointer does not match the record" } };
+            return checked;
+        }
+        catch (cause) {
+            return { ok: false, error: { kind: "io-error", message: "Could not load latest observation", cause } };
+        }
+    }
+    async function commit(input, expectedPriorId) {
+        let made;
+        try {
+            options.faults?.beforeSerialize?.();
+            made = buildRecord(input);
+        }
+        catch (cause) {
+            return { ok: false, error: { kind: "invalid-input", message: "Observation serialization failed", cause } };
+        }
+        if (!made.ok)
+            return made;
+        const record = made.value;
+        const dir = path.join(root, record.sourceKey);
+        const lockPath = path.join(dir, ".lock");
+        try {
+            await rejectSymlink(root);
+            await mkdir(root, { recursive: true });
+            await rejectSymlink(root, false);
+            await rejectSymlink(dir);
+            await mkdir(dir, { recursive: true });
+            await rejectSymlink(dir, false);
+            await rejectSymlink(lockPath);
+        }
+        catch (cause) {
+            return { ok: false, error: { kind: "io-error", message: "Could not prepare observation directory", cause } };
+        }
+        let lock;
+        try {
+            lock = await open(lockPath, "wx", 0o600);
+        }
+        catch (cause) {
+            return cause.code === "EEXIST" ? { ok: false, error: { kind: "continuity-conflict", message: "Another writer holds the source lock", cause } } : { ok: false, error: { kind: "io-error", message: "Could not acquire source lock", cause } };
+        }
+        const recordPath = path.join(dir, `${record.observationId}.json`);
+        let createdRecord = false;
+        let pointerCommitted = false;
+        try {
+            const current = await loadLatest(record.sourceId);
+            if (!current.ok)
+                return current;
+            if ((current.value?.observationId ?? null) !== expectedPriorId)
+                return { ok: false, error: { kind: "continuity-conflict", message: "Latest observation changed before commit" } };
+            const recordBytes = canonical(record);
+            try {
+                await rejectSymlink(recordPath);
+                const existing = await readFile(recordPath, "utf8");
+                if (existing !== recordBytes)
+                    return { ok: false, error: { kind: "corrupt-state", message: "Digest-addressed record already exists with different bytes" } };
+            }
+            catch (cause) {
+                if (cause.code !== "ENOENT")
+                    throw cause;
+                const installed = await atomicWrite(recordPath, recordBytes, "record", options.faults);
+                createdRecord = installed.renamed;
+                if (installed.durabilityWarning) {
+                    await rm(recordPath, { force: true });
+                    await syncDirectory(dir);
+                    createdRecord = false;
+                    return { ok: false, error: { kind: "io-error", message: installed.durabilityWarning } };
+                }
+            }
+            const pointer = await atomicWrite(path.join(dir, "latest.json"), canonical({ version: 1, sourceId: record.sourceId, observationId: record.observationId }), "pointer", options.faults);
+            pointerCommitted = pointer.renamed;
+            const warnings = pointer.durabilityWarning ? [pointer.durabilityWarning] : [];
+            try {
+                options.faults?.beforePrune?.();
+                const records = (await readdir(dir)).filter((name) => /^[a-f0-9]{64}\.json$/.test(name));
+                const inspected = await Promise.all(records.map(async (name) => ({ name, text: await readFile(path.join(dir, name), "utf8") })));
+                const valid = inspected.map(({ name, text }) => { try {
+                    const checked = validate(JSON.parse(text), record.sourceId);
+                    return checked.ok ? { name, recordedAt: checked.value.recordedAt, id: checked.value.observationId } : null;
+                }
+                catch {
+                    return null;
+                } }).filter((item) => item !== null);
+                const preserve = new Set([record.observationId, expectedPriorId].filter((item) => item !== null));
+                const extras = valid.filter((item) => !preserve.has(item.id)).sort((a, b) => b.recordedAt.localeCompare(a.recordedAt) || b.id.localeCompare(a.id));
+                await Promise.all(extras.map((item) => rm(path.join(dir, item.name), { force: true })));
+            }
+            catch (cause) {
+                warnings.push(`Observation committed but retention cleanup failed: ${cause instanceof Error ? cause.message : String(cause)}`);
+            }
+            return { ok: true, value: record, ...(warnings.length ? { warnings } : {}) };
+        }
+        catch (cause) {
+            if (createdRecord && !pointerCommitted)
+                await rm(recordPath, { force: true }).catch(() => undefined);
+            return { ok: false, error: { kind: "io-error", message: "Could not commit observation", cause } };
+        }
+        finally {
+            await lock.close().catch(() => undefined);
+            await rm(lockPath, { force: true }).catch(() => undefined);
+        }
+    }
+    return { loadLatest, commit };
+}

package/dist/src/proposal-diff.d.ts

@@ -0,0 +1,80 @@
+import type { ExtractionProposal } from "@kontourai/traverse";
+import { type DiffResult, type IdentityResult } from "./canonical-value.js";
+declare const proposalIdentityBrand: unique symbol;
+export type ProposalIdentity = string & {
+    readonly [proposalIdentityBrand]: "ProposalIdentity";
+};
+export interface ProposalSetObservation {
+    readonly sourceId: string;
+    readonly snapshotRef: string;
+    readonly observedAt: string;
+    readonly proposals: readonly ExtractionProposal[];
+}
+export interface ProposalEvidence {
+    readonly sourceId: string;
+    readonly snapshotRef: string;
+    readonly observedAt: string;
+    readonly entityKey: string;
+    readonly fieldKey: string;
+    readonly value: unknown;
+    readonly confidence: number;
+    readonly provenance: ExtractionProposal["provenance"];
+    readonly extractor: string;
+    readonly fieldPath: string;
+    readonly pathIndices?: readonly number[];
+}
+export type FieldChangeKind = "value-populated" | "value-updated" | "items-added" | "items-removed" | "value-replaced";
+export interface NewEntityAppearedEvent {
+    readonly kind: "new-entity-appeared";
+    readonly entityKey: string;
+    readonly current: readonly ProposalEvidence[];
+}
+interface FieldChangedEventCommon {
+    readonly kind: "field-changed";
+    readonly entityKey: string;
+    readonly fieldKey: string;
+    readonly changeKind: FieldChangeKind;
+}
+export type FieldChangedEvent = FieldChangedEventCommon & ({
+    readonly prior: ProposalEvidence;
+    readonly current: ProposalEvidence;
+} | {
+    readonly prior: ProposalEvidence;
+    readonly current?: never;
+} | {
+    readonly prior?: never;
+    readonly current: ProposalEvidence;
+});
+export type ProposalDiffEvent = NewEntityAppearedEvent | FieldChangedEvent;
+export interface ProposalOccurrencePair {
+    readonly prior: ExtractionProposal;
+    readonly current: ExtractionProposal;
+}
+export interface ProvenanceChangeFact {
+    readonly entityKey: string;
+    readonly fieldKey: string;
+    readonly prior: ProposalEvidence;
+    readonly current: ProposalEvidence;
+}
+export interface ProposalSetFacts {
+    readonly retainedProposalOccurrences: readonly ProposalOccurrencePair[];
+    readonly addedProposalOccurrences: readonly ExtractionProposal[];
+    readonly removedProposalOccurrences: readonly ExtractionProposal[];
+    readonly provenanceChanges: readonly ProvenanceChangeFact[];
+    readonly removedEntities: readonly string[];
+}
+export interface ProposalSetDiff {
+    readonly events: readonly ProposalDiffEvent[];
+    readonly facts: ProposalSetFacts;
+}
+export interface ProposalSetDiffInput<E> {
+    readonly prior: ProposalSetObservation;
+    readonly current: ProposalSetObservation;
+    readonly selectEntities: (observation: ProposalSetObservation) => readonly E[];
+    readonly entityIdentity: (entity: E) => string | IdentityResult;
+    readonly proposalsFor: (entity: E) => readonly ExtractionProposal[];
+    readonly fieldIdentity: (entity: E, proposal: ExtractionProposal) => string | IdentityResult;
+}
+export declare function extractionProposalIdentity(proposal: ExtractionProposal): IdentityResult<ProposalIdentity>;
+export declare function diffProposalSets<E>(input: ProposalSetDiffInput<E>): DiffResult<ProposalSetDiff>;
+export {};