cloving 1.1.0 → 1.2.0

package/dist/src/daemon/chats-batch-binding.js

@@ -0,0 +1,99 @@
+import { join } from "node:path";
+import { record } from "../protocol.js";
+import { initialAnalysisCheckpoint, batchAnalysisCheckpoint, } from "../chats/analysis-checkpoint.js";
+import { analysisEvidence, } from "../chats/analysis-input.js";
+import { mergeAnalysisBatch } from "../chats/analysis-merge.js";
+import { prepareAnalysisBatch } from "../chats/analysis-batches.js";
+import { checkpointProposal } from "../chats/analysis-proposal.js";
+import { findAnalysisCapture, readCapturedAnalysisSnapshot, } from "./chats-capture.js";
+import { boundedAnalysisFile, inspectAnalysisFiles } from "./chats-files.js";
+export async function boundAnalysisBatch(directory, input) {
+    const capture = await findAnalysisCapture(directory, {
+        runId: input.runId,
+        importId: input.importId,
+    });
+    if (capture === undefined)
+        throw new Error("analysis-output-failed");
+    const snapshot = await readCapturedAnalysisSnapshot(directory, capture);
+    const binding = record(input.batch);
+    const batchIndex = binding?.["batchIndex"] ?? 0;
+    if (typeof batchIndex !== "number")
+        throw new Error("analysis-output-failed");
+    const previous = await previousCheckpoint(join(directory, `capture-${capture.key}`), {
+        capture,
+        batchIndex,
+        snapshot,
+    });
+    const prepared = prepareAnalysisBatch(snapshot, {
+        nextRecordIndex: previous.nextRecordIndex,
+        completedBatchCount: previous.completedBatchCount,
+        previousResult: previous.result,
+    });
+    if (prepared.kind === "refused")
+        throw new Error("analysis-output-failed");
+    if (input.snapshot !== undefined &&
+        JSON.stringify(input.batch === undefined ? snapshot : prepared.snapshot) !==
+            JSON.stringify(input.snapshot))
+        throw new Error("analysis-output-failed");
+    if (!matchingBatchBinding(input.batch, {
+        capturedDigest: capture.digest,
+        batchIndex,
+        nextRecordIndex: prepared.nextRecordIndex,
+        totalRecords: capture.totalRecords,
+    }))
+        throw new Error("analysis-output-failed");
+    return {
+        capture,
+        snapshot,
+        batch: { batchIndex, nextRecordIndex: prepared.nextRecordIndex },
+        previous,
+    };
+}
+function matchingBatchBinding(binding, expected) {
+    if (binding === undefined)
+        return expected.nextRecordIndex === expected.totalRecords;
+    const fields = record(binding);
+    return ["capturedDigest", "batchIndex", "nextRecordIndex"].every((name) => fields?.[name] === Reflect.get(expected, name));
+}
+export async function previousCheckpoint(folder, input) {
+    const { capture, batchIndex } = input;
+    if (!Number.isSafeInteger(batchIndex) || batchIndex < 0 || batchIndex > 3)
+        throw new Error("analysis-input-refused");
+    const initial = initialAnalysisCheckpoint(capture);
+    if (batchIndex === 0)
+        return initial;
+    const name = `analysis-checkpoint-${String(batchIndex)}.json`;
+    const text = await boundedAnalysisFile(join(folder, name), {
+        permissions: 0o600,
+    });
+    const checkpoint = restoreCheckpoint(text, input);
+    await inspectAnalysisFiles(folder, {
+        cycleId: `checkpoint-${String(batchIndex)}`,
+        analysis: JSON.stringify(checkpoint) + "\n",
+        proposal: checkpointProposal(checkpoint),
+    });
+    return checkpoint;
+}
+function restoreCheckpoint(text, input) {
+    const value = record(JSON.parse(text));
+    const result = record(value?.["result"]);
+    const nextRecordIndex = value?.["nextRecordIndex"];
+    if (result === undefined || typeof nextRecordIndex !== "number")
+        throw new Error("analysis-output-failed");
+    const merged = mergeAnalysisBatch({
+        previousResult: null,
+        candidate: Object.fromEntries(Object.entries(result).filter(([key]) => key !== "limitations")),
+        evidence: analysisEvidence(input.snapshot),
+        nextRecordIndex,
+    });
+    if (!merged.accepted)
+        throw new Error("analysis-output-failed");
+    const checkpoint = batchAnalysisCheckpoint(input.capture, {
+        batchIndex: input.batchIndex - 1,
+        nextRecordIndex,
+        result: merged.result,
+    });
+    if (text !== JSON.stringify(checkpoint) + "\n")
+        throw new Error("analysis-output-failed");
+    return checkpoint;
+}

package/dist/src/daemon/chats-capture.d.ts

@@ -0,0 +1,18 @@
+import { type AnalysisSnapshot } from "../chats/analysis-input.ts";
+export interface AnalysisCapture {
+    readonly schemaVersion: 1;
+    readonly key: string;
+    readonly runId: string;
+    readonly importId: string;
+    readonly digest: string;
+    readonly totalRecords: number;
+}
+export declare function captureAnalysisSnapshot(directory: string, binding: {
+    readonly runId: string;
+    readonly importId: string;
+}): Promise<AnalysisCapture>;
+export declare function findAnalysisCapture(directory: string, binding: {
+    readonly runId: string;
+    readonly importId: string;
+}): Promise<AnalysisCapture | undefined>;
+export declare function readCapturedAnalysisSnapshot(directory: string, capture: AnalysisCapture): Promise<AnalysisSnapshot>;

package/dist/src/daemon/chats-capture.js

@@ -0,0 +1,194 @@
+import { createHash } from "node:crypto";
+import { lstat, mkdtemp, readdir, rename, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { record } from "../protocol.js";
+import { analysisEvidence, validAnalysisImportId, } from "../chats/analysis-input.js";
+import { validateDestination } from "../chats/destination.js";
+import { initialAnalysisCheckpoint } from "../chats/analysis-checkpoint.js";
+import { checkpointProposal } from "../chats/analysis-proposal.js";
+import { boundedAnalysisFile, matchingPrivateFile, readAnalysisCorpus, readAnalysisSnapshot, } from "./chats-files.js";
+export async function captureAnalysisSnapshot(directory, binding) {
+    const key = captureKey(binding);
+    const folder = join(directory, `capture-${key}`);
+    await validateDestination(directory);
+    await reclaimAbandonedCaptures(directory);
+    const existing = await findAnalysisCapture(directory, binding);
+    if (existing !== undefined)
+        return existing;
+    const temporary = await mkdtemp(join(directory, `.capture-${key}-${String(process.pid)}-`));
+    try {
+        const capture = await writeCapture(directory, {
+            ...binding,
+            folder: temporary,
+        });
+        try {
+            await rename(temporary, folder);
+        }
+        catch (error) {
+            if (!(error instanceof Error &&
+                "code" in error &&
+                (error.code === "EEXIST" || error.code === "ENOTEMPTY")))
+                throw error;
+            const published = await findAnalysisCapture(directory, binding);
+            if (published === undefined)
+                throw error;
+            return published;
+        }
+        return capture;
+    }
+    catch (error) {
+        if (missingRedaction(error))
+            throw new Error("redaction-required", { cause: error });
+        throw error;
+    }
+    finally {
+        await rm(temporary, { recursive: true, force: true });
+    }
+}
+function missingRedaction(error) {
+    if (!(error instanceof Error) ||
+        !("code" in error) ||
+        error.code !== "ENOENT" ||
+        !("path" in error) ||
+        typeof error.path !== "string")
+        return false;
+    return (error.path.endsWith("/redaction.json") ||
+        error.path.endsWith("/redacted.jsonl"));
+}
+async function reclaimAbandonedCaptures(directory) {
+    for (const entry of await readdir(directory, { withFileTypes: true })) {
+        if (entry.isDirectory() && absentCaptureOwner(entry.name))
+            await rm(join(directory, entry.name), { recursive: true, force: true });
+    }
+}
+function absentCaptureOwner(name) {
+    const owner = /^\.capture-[a-f0-9]{64}-([1-9][0-9]*)-[A-Za-z0-9]{6}$/.exec(name)?.[1];
+    if (owner === undefined)
+        return false;
+    const processId = Number(owner);
+    if (!Number.isSafeInteger(processId))
+        return false;
+    try {
+        process.kill(processId, 0);
+        return false;
+    }
+    catch (error) {
+        return error instanceof Error && "code" in error && error.code === "ESRCH";
+    }
+}
+export async function findAnalysisCapture(directory, binding) {
+    const folder = join(directory, `capture-${captureKey(binding)}`);
+    const stats = await lstat(folder).catch((error) => {
+        if (!(error instanceof Error && "code" in error && error.code === "ENOENT"))
+            throw error;
+        return undefined;
+    });
+    if (stats === undefined)
+        return undefined;
+    const capture = await capturedReference(folder, binding);
+    await readCapturedAnalysisSnapshot(directory, capture);
+    return capture;
+}
+async function writeCapture(directory, input) {
+    const snapshot = await copiedSnapshot(directory, {
+        folder: input.folder,
+        importId: input.importId,
+    });
+    const evidence = analysisEvidence(snapshot);
+    const capture = {
+        schemaVersion: 1,
+        key: captureKey(input),
+        runId: input.runId,
+        importId: input.importId,
+        digest: snapshotDigest(snapshot),
+        totalRecords: evidence.records.length,
+    };
+    await matchingPrivateFile(join(input.folder, "capture.json"), JSON.stringify(capture) + "\n");
+    await ensureInitialCheckpoint(input.folder, capture);
+    return capture;
+}
+export async function readCapturedAnalysisSnapshot(directory, capture) {
+    const key = captureKey(capture);
+    if (capture.key !== key)
+        throw new Error("analysis-input-refused");
+    const folder = join(directory, `capture-${key}`);
+    const stored = await capturedReference(folder, capture);
+    if (stored.digest !== capture.digest ||
+        stored.totalRecords !== capture.totalRecords ||
+        record(capture)?.["schemaVersion"] !== 1)
+        throw new Error("analysis-input-refused");
+    const snapshot = await readAnalysisSnapshot(folder, {
+        importId: capture.importId,
+        permissions: 0o600,
+    });
+    const evidence = analysisEvidence(snapshot);
+    if (snapshotDigest(snapshot) !== capture.digest ||
+        evidence.records.length !== capture.totalRecords)
+        throw new Error("analysis-input-refused");
+    await ensureInitialCheckpoint(folder, capture);
+    return snapshot;
+}
+async function ensureInitialCheckpoint(folder, capture) {
+    const checkpoint = initialAnalysisCheckpoint(capture);
+    for (const [name, expected] of [
+        ["checkpoint-0.json", JSON.stringify(checkpoint) + "\n"],
+        ["proposal-0.txt", checkpointProposal(checkpoint)],
+    ]) {
+        await matchingPrivateFile(join(folder, name), expected);
+    }
+}
+function captureKey(binding) {
+    if (!validAnalysisImportId(binding.importId) ||
+        !/^[A-Za-z0-9_-]{1,128}$/.test(binding.runId))
+        throw new Error("analysis-input-refused");
+    return createHash("sha256")
+        .update(JSON.stringify([binding.runId, binding.importId]))
+        .digest("hex");
+}
+function snapshotDigest(snapshot) {
+    return createHash("sha256").update(JSON.stringify(snapshot)).digest("hex");
+}
+async function copiedSnapshot(directory, capture) {
+    const redaction = await boundedAnalysisFile(join(directory, "redaction.json"));
+    await matchingPrivateFile(join(capture.folder, "redaction.json"), redaction);
+    const corpus = await readAnalysisCorpus(join(directory, "redacted.jsonl"), {
+        output: join(capture.folder, "redacted.jsonl"),
+    });
+    const structure = await boundedAnalysisFile(join(directory, "structure.json"));
+    await matchingPrivateFile(join(capture.folder, "structure.json"), structure);
+    return { importId: capture.importId, redaction, corpus, structure };
+}
+async function capturedReference(folder, binding) {
+    await validateDestination(folder);
+    if (((await lstat(folder)).mode & 0o777) !== 0o700)
+        throw new Error("analysis-output-failed");
+    const value = record(JSON.parse(await boundedAnalysisFile(join(folder, "capture.json"), {
+        permissions: 0o600,
+    })));
+    const identity = {
+        schemaVersion: 1,
+        key: captureKey(binding),
+        runId: binding.runId,
+        importId: binding.importId,
+    };
+    if (value === undefined ||
+        Object.keys(value).length !== 6 ||
+        !Object.entries(identity).every(([key, expected]) => value[key] === expected))
+        throw new Error("analysis-input-refused");
+    const digest = value["digest"];
+    const totalRecords = value["totalRecords"];
+    if (typeof digest !== "string" ||
+        !/^[a-f0-9]{64}$/.test(digest) ||
+        typeof totalRecords !== "number" ||
+        !Number.isSafeInteger(totalRecords) ||
+        totalRecords < 0)
+        throw new Error("analysis-input-refused");
+    return {
+        schemaVersion: 1,
+        key: identity.key,
+        runId: binding.runId,
+        importId: binding.importId,
+        digest,
+        totalRecords,
+    };
+}

package/dist/src/daemon/chats-checkpoints.d.ts

@@ -0,0 +1,41 @@
+import { type AnalysisCheckpoint } from "../chats/analysis-checkpoint.ts";
+import { type AnalysisSnapshot } from "../chats/analysis-input.ts";
+import { type AnalysisBatchPreparation } from "../chats/analysis-batches.ts";
+import { type AnalysisCapture } from "./chats-capture.ts";
+import { type AnalysisDraftInput } from "./chats-files.ts";
+interface CheckpointInput {
+    readonly capture: AnalysisCapture;
+    readonly candidate: unknown;
+    readonly batchIndex: number;
+    readonly nextRecordIndex: number;
+    readonly mode: "deliver" | "reconcile";
+}
+interface FinalizeInput {
+    readonly runId: string;
+    readonly importId: string;
+    readonly cycleId: string;
+    readonly snapshot: AnalysisSnapshot | undefined;
+    readonly candidate: unknown;
+    readonly mode: "deliver" | "reconcile";
+    readonly batch?: unknown;
+    readonly draft?: AnalysisDraftInput;
+}
+export declare function finalizeAnalysis(directory: string, input: FinalizeInput): Promise<(Awaited<ReturnType<typeof saveAnalysisCheckpoint>> & {
+    readonly next?: AnalysisBatchPreparation | undefined;
+}) | {
+    readonly kind: "absent";
+}>;
+export declare function saveAnalysisCheckpoint(directory: string, input: CheckpointInput): Promise<{
+    kind: "committed" | "absent";
+    checkpoint: AnalysisCheckpoint & {
+        readonly result: import("../chats/analysis-contract.ts").AnalysisResult;
+    };
+    receipt: string;
+} | {
+    kind: "refused";
+    codes: readonly import("../chats/analysis-merge.ts").AnalysisMergeCode[];
+} | {
+    kind: "absent";
+    codes?: never;
+}>;
+export {};

package/dist/src/daemon/chats-checkpoints.js

@@ -0,0 +1,186 @@
+import { boundAnalysisBatch, previousCheckpoint, } from "./chats-batch-binding.js";
+import { createHash } from "node:crypto";
+import { join } from "node:path";
+import { batchAnalysisCheckpoint, } from "../chats/analysis-checkpoint.js";
+import { analysisEvidence, analysisDocumentBytes, } from "../chats/analysis-input.js";
+import { mergeAnalysisBatch } from "../chats/analysis-merge.js";
+import { prepareAnalysisBatch, } from "../chats/analysis-batches.js";
+import { analysisProposal, checkpointProposal, } from "../chats/analysis-proposal.js";
+import { readCapturedAnalysisSnapshot, } from "./chats-capture.js";
+import { boundedAnalysisFile, matchingPrivateFile, inspectAnalysisFiles, saveAnalysisFiles, } from "./chats-files.js";
+export async function finalizeAnalysis(directory, input) {
+    const { capture, batch, snapshot } = await boundAnalysisBatch(directory, input);
+    const inspected = await saveAnalysisCheckpoint(directory, {
+        ...batch,
+        capture,
+        candidate: input.candidate,
+        mode: "reconcile",
+    });
+    if (inspected.kind === "refused")
+        return inspected;
+    if (!("checkpoint" in inspected)) {
+        if (input.mode === "reconcile")
+            return { kind: "absent" };
+        return saveAnalysisCheckpoint(directory, {
+            ...batch,
+            capture,
+            candidate: input.candidate,
+            mode: "deliver",
+        });
+    }
+    return commitAnalysisBatch(directory, {
+        input,
+        batch: { ...batch, capture, candidate: input.candidate, mode: input.mode },
+        snapshot,
+        inspected,
+    });
+}
+async function commitAnalysisBatch(directory, prepared) {
+    const { input, batch, snapshot, inspected } = prepared;
+    const output = {
+        cycleId: input.cycleId,
+        analysis: JSON.stringify(inspected.checkpoint.result) + "\n",
+        proposal: analysisProposal(inspected.checkpoint.result),
+        ...(input.draft === undefined ? {} : { draft: input.draft }),
+    };
+    const complete = inspected.checkpoint.status === "complete";
+    const legacy = complete
+        ? await inspectAnalysisFiles(directory, output)
+        : "present";
+    if (input.mode === "reconcile" &&
+        (inspected.kind === "absent" || legacy === "absent"))
+        return { kind: "absent" };
+    const saved = await saveAnalysisCheckpoint(directory, {
+        ...batch,
+        mode: input.mode,
+    });
+    if (saved.kind !== "committed")
+        return saved;
+    if (complete && input.mode === "deliver")
+        await saveAnalysisFiles(directory, output);
+    const next = complete
+        ? undefined
+        : prepareAnalysisBatch(snapshot, {
+            nextRecordIndex: saved.checkpoint.nextRecordIndex,
+            completedBatchCount: saved.checkpoint.completedBatchCount,
+            previousResult: saved.checkpoint.result,
+        });
+    return { ...saved, next };
+}
+export async function saveAnalysisCheckpoint(directory, input) {
+    const { capture, candidate } = input;
+    const snapshot = await readCapturedAnalysisSnapshot(directory, capture);
+    const folder = join(directory, `capture-${capture.key}`);
+    const previous = await previousCheckpoint(folder, { ...input, snapshot });
+    if (!Number.isSafeInteger(input.nextRecordIndex) ||
+        input.nextRecordIndex <= previous.nextRecordIndex ||
+        input.nextRecordIndex > capture.totalRecords ||
+        previous.status === "complete")
+        throw new Error("analysis-input-refused");
+    const merged = mergeAnalysisBatch({
+        previousResult: previous.result,
+        candidate,
+        evidence: analysisEvidence(snapshot),
+        nextRecordIndex: input.nextRecordIndex,
+    });
+    if (!merged.accepted && !merged.codes.includes("analysis-merge-conflict"))
+        return { kind: "refused", codes: merged.codes };
+    if (!merged.accepted &&
+        (await retainConflict(folder, { ...input, previous })) === "absent")
+        return { kind: "absent" };
+    if (!merged.accepted)
+        return { kind: "refused", codes: merged.codes };
+    const checkpoint = batchAnalysisCheckpoint(capture, {
+        ...input,
+        result: merged.result,
+    });
+    return persistCheckpoint(folder, { ...input, checkpoint });
+}
+async function bindBatch(folder, input) {
+    const { capture, batchIndex, nextRecordIndex, candidate, mode } = input;
+    const text = JSON.stringify({
+        schemaVersion: 1,
+        runId: capture.runId,
+        importId: capture.importId,
+        capturedDigest: capture.digest,
+        batchIndex,
+        nextRecordIndex,
+        candidateDigest: createHash("sha256")
+            .update(JSON.stringify(candidate))
+            .digest("hex"),
+    }) + "\n";
+    const path = join(folder, `batch-claim-${String(batchIndex)}.json`);
+    if (mode === "deliver")
+        await matchingPrivateFile(path, text);
+    else
+        await inspectMatchingPrivateFile(path, text);
+}
+async function persistCheckpoint(folder, input) {
+    const { checkpoint, mode } = input;
+    const output = {
+        cycleId: `checkpoint-${String(checkpoint.completedBatchCount)}`,
+        analysis: JSON.stringify(checkpoint) + "\n",
+        proposal: checkpointProposal(checkpoint),
+    };
+    const state = await inspectAnalysisFiles(folder, output);
+    await bindBatch(folder, input);
+    if (mode === "deliver")
+        await saveAnalysisFiles(folder, output);
+    return {
+        kind: mode === "reconcile" && state === "absent"
+            ? "absent"
+            : "committed",
+        checkpoint,
+        receipt: createHash("sha256").update(output.analysis).digest("hex"),
+    };
+}
+async function retainConflict(folder, input) {
+    const { capture, candidate, mode, batchIndex, nextRecordIndex, previous } = input;
+    const incomingText = JSON.stringify(candidate) + "\n";
+    if (Buffer.byteLength(incomingText) > analysisDocumentBytes)
+        throw new Error("analysis-output-failed");
+    const incomingFile = `conflict-${String(batchIndex)}-incoming.json`;
+    const text = JSON.stringify({
+        schemaVersion: 1,
+        runId: capture.runId,
+        importId: capture.importId,
+        capturedDigest: capture.digest,
+        batchIndex,
+        nextRecordIndex,
+        accepted: {
+            file: `analysis-checkpoint-${String(previous.completedBatchCount)}.json`,
+            digest: createHash("sha256")
+                .update(JSON.stringify(previous) + "\n")
+                .digest("hex"),
+        },
+        incoming: {
+            file: incomingFile,
+            digest: createHash("sha256").update(incomingText).digest("hex"),
+        },
+    }) + "\n";
+    const path = join(folder, `conflict-${String(batchIndex)}.json`);
+    const incomingPath = join(folder, incomingFile);
+    await bindBatch(folder, input);
+    if (mode === "reconcile") {
+        const variant = await inspectMatchingPrivateFile(incomingPath, incomingText);
+        const envelope = await inspectMatchingPrivateFile(path, text);
+        return variant === "absent" || envelope === "absent"
+            ? "absent"
+            : "retained";
+    }
+    await matchingPrivateFile(incomingPath, incomingText);
+    await matchingPrivateFile(path, text);
+    return "retained";
+}
+async function inspectMatchingPrivateFile(path, expected) {
+    const text = await boundedAnalysisFile(path, { permissions: 0o600 }).catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return undefined;
+        throw error;
+    });
+    if (text === undefined)
+        return "absent";
+    if (text !== expected)
+        throw new Error("analysis-output-failed");
+    return "retained";
+}

package/dist/src/daemon/chats-files.d.ts

@@ -1,15 +1,42 @@
 import { type AnalysisSnapshot } from "../chats/analysis-input.ts";
+export declare function readAnalysisCorpus(path: string, options?: {
+    readonly output?: string;
+    readonly permissions?: number;
+}): Promise<string>;
 export declare function analysisDirectory(runtime: string, importId: string): Promise<string>;
 export declare function boundedAnalysisFile(path: string, options?: {
     readonly permissions?: number;
 }): Promise<string>;
-export declare function readAnalysisSnapshot(directory: string, importId: string): Promise<AnalysisSnapshot>;
+export declare function readAnalysisSnapshot(directory: string, input: {
+    readonly importId: string;
+    readonly permissions?: number;
+}): Promise<AnalysisSnapshot>;
 export declare function matchingPrivateFile(path: string, text: string): Promise<void>;
-interface AnalysisOutput {
+export interface AnalysisOutput {
     readonly cycleId: string;
     readonly analysis: string;
     readonly proposal: string;
+    readonly draft?: AnalysisDraftInput;
 }
 export declare function inspectAnalysisFiles(directory: string, output: AnalysisOutput): Promise<"complete" | "absent">;
 export declare function saveAnalysisFiles(directory: string, output: AnalysisOutput): Promise<void>;
+export interface AnalysisDraftInput {
+    readonly runId: string;
+    readonly cycleId: string;
+    readonly operationId: string;
+    readonly inputDigest: string;
+    readonly snapshotDigest: string;
+    readonly reply: string;
+    readonly batch?: unknown;
+}
+interface AnalysisDraftMetadata {
+    readonly candidateDigest: string;
+    readonly expiresAt: string;
+}
+export declare function inspectAnalysisDraft(directory: string, input: AnalysisDraftInput): Promise<AnalysisDraftMetadata | undefined>;
+export declare function inspectAnalysisDraftRecord(directory: string, input: AnalysisDraftInput): Promise<AnalysisDraftMetadata | undefined>;
+export declare function saveAnalysisDraft(directory: string, saving: {
+    readonly input: AnalysisDraftInput;
+    readonly now: () => string;
+}): Promise<AnalysisDraftMetadata>;
 export {};