cloving 1.0.1 → 1.2.0

package/dist/src/chats/import.js

@@ -1,23 +1,97 @@
+import { readPreviousImport } from "./previous-import.js";
+import { pairEvidence, compareEvidence, } from "./revision.js";
+import { structureImport } from "./structure.js";
 import { randomUUID } from "node:crypto";
 import { rm } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { createPrivateFile, importDirectory, validateDestination, } from "./destination.js";
 import { readTranscript } from "./reader.js";
-async function writeImport(directory, request) {
+import { selectChatSources, } from "./selection.js";
+const corpusLimit = 52_428_800;
+function boundedCorpusLine(record, previousBytes) {
+    const line = `${JSON.stringify(record)}\n`;
+    const bytes = previousBytes + Buffer.byteLength(line);
+    if (bytes > corpusLimit)
+        throw new Error("corpus-too-large");
+    return { line, bytes };
+}
+async function writeImport(directory, data) {
+    const { request, previous } = data;
+    const selection = await selectChatSources(request);
+    const keys = selection.entries.map((selected) => selectionKey(selected, selection.roots));
+    const pairs = pairEvidence({ previous, current: keys });
+    const evidence = [];
     const files = [];
+    const records = [];
+    let corpusBytes = 0;
     const corpus = await createPrivateFile(join(directory, "corpus.jsonl"));
     try {
-        for (const selected of request.paths) {
-            const result = await readTranscript(resolve(request.workingDirectory, selected));
-            files.push(result.coverage);
-            for (const record of result.records)
-                await corpus.writeFile(`${JSON.stringify(record)}\n`);
+        for (const [index, selected] of selection.entries.entries()) {
+            const matched = pairs[index];
+            const before = matched === null || matched === undefined
+                ? undefined
+                : previous[matched];
+            const result = await readSelected(selected, before?.byteSize);
+            evidence.push({
+                ...selectionKey(selected, selection.roots),
+                ...result.coverage,
+                ...(result.prefixDigest === undefined
+                    ? {}
+                    : { prefixDigest: result.prefixDigest }),
+            });
+            files.push({
+                ...result.coverage,
+                kind: selected.kind,
+                selection: selected.selection,
+            });
+            for (const record of result.records) {
+                const next = boundedCorpusLine(record, corpusBytes);
+                await corpus.writeFile(next.line);
+                corpusBytes = next.bytes;
+                records.push(record);
+            }
         }
     }
     finally {
         await corpus.close();
     }
-    return files;
+    return { roots: selection.roots, files, records, evidence };
+}
+function selectionKey(selected, roots) {
+    return {
+        path: selected.path,
+        root: selected.selection.kind === "root"
+            ? (roots[selected.selection.index] ?? null)
+            : null,
+    };
+}
+async function readSelected(selected, prefixBytes) {
+    if (selected.outcome !== undefined)
+        return {
+            coverage: {
+                path: selected.path,
+                ...selected.outcome,
+                digest: null,
+                recordCount: 0,
+                omissions: { metadataRecords: 0, thinkingBlocks: 0 },
+            },
+            records: [],
+        };
+    return readTranscript(selected.path, {
+        ...(selected.inspection === undefined
+            ? {}
+            : { inspection: selected.inspection }),
+        ...(prefixBytes === undefined ? {} : { prefixBytes }),
+    });
+}
+async function writeMetadata(path, data) {
+    const handle = await createPrivateFile(path);
+    try {
+        await handle.writeFile(`${JSON.stringify({ schemaVersion: 1, ...data }, null, 2)}\n`);
+    }
+    finally {
+        await handle.close();
+    }
 }
 async function writeCoverage(directory, coverage) {
     const manifest = join(directory, "coverage.json");
@@ -41,31 +115,129 @@ async function writeCoverage(directory, coverage) {
     }
 }
 export async function importChats(request) {
-    const empty = { importId: null, included: 0, refused: 0, records: 0 };
+    const empty = emptyImport(request);
+    if (process.platform !== "darwin")
+        return { ...empty, code: "platform-unsupported" };
     if (request.runtimeDirectory === undefined)
         return { ...empty, code: "runtime-directory-unknown" };
     try {
+        const runtime = resolve(request.workingDirectory, request.runtimeDirectory);
+        const previous = request.previousImportId === undefined
+            ? undefined
+            : await readPreviousImport(runtime, request.previousImportId);
         const importId = randomUUID();
-        const directory = await importDirectory(resolve(request.workingDirectory, request.runtimeDirectory), importId);
-        const files = await writeImport(directory, request);
-        const included = files.filter((file) => file.status === "included").length;
-        const records = files.reduce((count, file) => count + file.recordCount, 0);
-        const refused = files.length - included;
-        await writeCoverage(directory, { schemaVersion: 1, importId, files });
-        return {
-            code: refused === 0 ? "import-complete" : "import-partial",
-            importId,
-            included,
-            refused,
-            records,
-        };
+        const directory = await importDirectory(runtime, importId);
+        return await completeImport(directory, { request, previous, importId });
     }
     catch (error) {
         return {
             ...empty,
-            code: error instanceof Error && error.message === "destination-refused"
-                ? "destination-refused"
+            code: error instanceof Error &&
+                (error.message === "destination-refused" ||
+                    error.message === "previous-import-refused")
+                ? error.message
                 : "output-failed",
         };
     }
 }
+function importTotals(files) {
+    return {
+        included: files.filter((file) => file.status === "included").length,
+        refused: files.filter((file) => ["unreadable", "unsupported", "too-large"].includes(file.status)).length,
+        excluded: files.filter((file) => file.status === "excluded").length,
+        pending: files.filter((file) => file.status === "pending").length,
+    };
+}
+function coverageManifest(data) {
+    const { importId, roots, files, exclusions } = data;
+    if (roots.length > 0)
+        return {
+            schemaVersion: 2,
+            importId,
+            roots,
+            exclusions,
+            limits: {
+                depth: 4,
+                files: 100,
+                sourceBytes: 10_485_760,
+                corpusBytes: corpusLimit,
+            },
+            entries: files,
+        };
+    return {
+        schemaVersion: 1,
+        importId,
+        files: files.map((entry) => ({
+            path: entry.path,
+            status: entry.status,
+            reason: entry.reason,
+            digest: entry.digest,
+            ...(entry.byteSize === undefined ? {} : { byteSize: entry.byteSize }),
+            recordCount: entry.recordCount,
+            omissions: entry.omissions,
+        })),
+    };
+}
+function emptyImport(request) {
+    return {
+        importId: null,
+        included: 0,
+        refused: 0,
+        ...((request.roots?.length ?? 0) === 0 ? {} : { excluded: 0, pending: 0 }),
+        ...structureImport([]).counts,
+    };
+}
+async function completeImport(directory, data) {
+    const { request, previous, importId } = data;
+    const { roots, files, records, evidence } = await writeImport(directory, {
+        request,
+        previous: previous?.entries ?? [],
+    });
+    const structure = structureImport(records);
+    await writeMetadata(join(directory, "structure.json"), {
+        importId,
+        ...structure,
+    });
+    const partial = await writeRevision(directory, {
+        previous,
+        evidence,
+        importId,
+    });
+    const totals = importTotals(files);
+    const { refused, pending } = totals;
+    await writeCoverage(directory, coverageManifest({
+        importId,
+        roots,
+        files,
+        exclusions: request.excludeProjects ?? [],
+    }));
+    return {
+        code: refused + pending + partial === 0 ? "import-complete" : "import-partial",
+        importId,
+        included: totals.included,
+        refused,
+        ...(roots.length === 0 ? {} : { excluded: totals.excluded, pending }),
+        ...structure.counts,
+    };
+}
+async function writeRevision(directory, data) {
+    const { previous, evidence, importId } = data;
+    if (previous === undefined)
+        return 0;
+    const comparison = compareEvidence({
+        previous: previous.entries,
+        current: evidence,
+    });
+    try {
+        await writeMetadata(join(directory, "revision.json"), {
+            importId,
+            previousImportId: previous.importId,
+            evidenceRevision: previous.evidenceRevision,
+            ...comparison,
+        });
+    }
+    catch {
+        throw new Error("output-failed");
+    }
+    return comparison.counts.missing + comparison.counts.truncated;
+}

package/dist/src/chats/input-file.d.ts

@@ -0,0 +1,3 @@
+import { type FileHandle } from "node:fs/promises";
+export declare function openInput(path: string): Promise<FileHandle>;
+export declare function readInput(path: string): Promise<string>;

package/dist/src/chats/input-file.js

@@ -0,0 +1,32 @@
+import { constants } from "node:fs";
+import { lstat, open } from "node:fs/promises";
+export async function openInput(path) {
+    if (process.platform !== "darwin")
+        throw new Error("platform-unsupported");
+    const before = await lstat(path);
+    if (!before.isFile() || before.nlink !== 1)
+        throw new Error("invalid-import");
+    const handle = await open(path, constants.O_RDONLY | constants.O_NONBLOCK | 0x20000000);
+    try {
+        const opened = await handle.stat();
+        if (!opened.isFile() ||
+            opened.nlink !== 1 ||
+            opened.dev !== before.dev ||
+            opened.ino !== before.ino)
+            throw new Error("invalid-import");
+        return handle;
+    }
+    catch (error) {
+        await handle.close();
+        throw error;
+    }
+}
+export async function readInput(path) {
+    const handle = await openInput(path);
+    try {
+        return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(await handle.readFile());
+    }
+    finally {
+        await handle.close();
+    }
+}

package/dist/src/chats/normalize.d.ts

@@ -2,8 +2,7 @@ export interface Omissions {
     metadataRecords: number;
     thinkingBlocks: number;
 }
-export interface MessageRecord {
-    readonly schemaVersion: 1;
+interface MessageFields {
     readonly sourceId: string;
     readonly role: "user" | "assistant";
     readonly timestamp: string | null;
@@ -11,10 +10,20 @@ export interface MessageRecord {
     readonly text: string;
     readonly blocks: readonly Record<string, unknown>[];
 }
+export type MessageRecord = MessageFields & ({
+    readonly schemaVersion: 1;
+} | {
+    readonly schemaVersion: 2;
+    readonly uuid: string | null;
+    readonly isSidechain: boolean | null;
+    readonly parentUuid?: string | null;
+});
 export interface NormalizedTranscript {
     readonly status: "included" | "unsupported";
     readonly reason: string;
     readonly records: readonly MessageRecord[];
     readonly omissions: Omissions;
 }
+export declare function parsePreservedJson(text: string): unknown;
 export declare function normalizeTranscript(text: string, digest: string): NormalizedTranscript;
+export {};

package/dist/src/chats/normalize.js

@@ -58,6 +58,9 @@ function preserveJson(...argumentsList) {
         throw new Error("unsupported-record");
     return JSON.rawJSON(context["source"]);
 }
+export function parsePreservedJson(text) {
+    return JSON.parse(text, preserveJson);
+}
 function blocks(content, omissions) {
     if (typeof content === "string")
         return [{ position: 0, type: "text", text: content }];
@@ -73,6 +76,18 @@ function blocks(content, omissions) {
         return [{ ...supportedBlock(block), position }];
     });
 }
+function identity(value) {
+    const uuid = value["uuid"];
+    const parent = value["parentUuid"];
+    const sidechain = value["isSidechain"];
+    return {
+        uuid: typeof uuid === "string" && uuid !== "" ? uuid : null,
+        isSidechain: typeof sidechain === "boolean" ? sidechain : null,
+        ...(parent === null || (typeof parent === "string" && parent !== "")
+            ? { parentUuid: parent }
+            : {}),
+    };
+}
 function message(value, context) {
     if (!object(value) || typeof value["type"] !== "string")
         throw new Error("unsupported-record");
@@ -89,7 +104,8 @@ function message(value, context) {
         throw new Error("unsupported-message");
     const normalized = blocks(envelope["content"], context.omissions);
     return {
-        schemaVersion: 1,
+        schemaVersion: 2,
+        ...identity(value),
         sourceId: context.sourceId,
         role: value["type"],
         timestamp: optionalText(value["timestamp"]),
@@ -112,7 +128,7 @@ export function normalizeTranscript(text, digest) {
         lines.pop();
     try {
         for (const line of lines) {
-            const parsed = JSON.parse(line, preserveJson);
+            const parsed = parsePreservedJson(line);
             const record = message(parsed, {
                 sourceId: `${digest}:${String(offset)}`,
                 omissions,

package/dist/src/chats/previous-import.d.ts

@@ -0,0 +1,5 @@
+export declare function readPreviousImport(runtime: string, importId: string): Promise<{
+    importId: string;
+    entries: readonly import("./revision.ts").EvidenceSource[];
+    evidenceRevision: number;
+}>;

package/dist/src/chats/previous-import.js

@@ -0,0 +1,34 @@
+import { lstat } from "node:fs/promises";
+import { join } from "node:path";
+import { validateDestination } from "./destination.js";
+import { readInput } from "./input-file.js";
+import { parseEvidenceCoverage, nextEvidenceRevision, } from "./revision-input.js";
+async function optionalRevision(path) {
+    try {
+        await lstat(path);
+    }
+    catch (error) {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return;
+        throw error;
+    }
+    return readInput(path);
+}
+export async function readPreviousImport(runtime, importId) {
+    if (process.platform !== "darwin")
+        throw new Error("platform-unsupported");
+    try {
+        if (!/^[a-f\d]{8}-[a-f\d]{4}-4[a-f\d]{3}-[89ab][a-f\d]{3}-[a-f\d]{12}$/.test(importId))
+            throw new Error("invalid-import");
+        const directory = join(runtime, "chats", importId);
+        await validateDestination(directory);
+        if (!(await lstat(directory)).isDirectory())
+            throw new Error("invalid-import");
+        const entries = parseEvidenceCoverage(await readInput(join(directory, "coverage.json")), importId);
+        const evidenceRevision = nextEvidenceRevision(await optionalRevision(join(directory, "revision.json")), importId);
+        return { importId, entries, evidenceRevision };
+    }
+    catch {
+        throw new Error("previous-import-refused");
+    }
+}

package/dist/src/chats/reader.d.ts

@@ -1,3 +1,4 @@
+import { type ScanSource } from "./source.ts";
 import { type MessageRecord, type Omissions } from "./normalize.ts";
 export declare const transcriptLimit = 10485760;
 export interface SelectedTranscript {
@@ -5,11 +6,16 @@ export interface SelectedTranscript {
     readonly status: "included" | "unreadable" | "unsupported" | "too-large";
     readonly reason: string;
     readonly digest: string | null;
+    readonly byteSize?: number;
     readonly recordCount: number;
     readonly omissions: Omissions;
 }
 export interface ReadTranscript {
     readonly coverage: SelectedTranscript;
     readonly records: readonly MessageRecord[];
+    readonly prefixDigest?: string;
 }
-export declare function readTranscript(path: string): Promise<ReadTranscript>;
+export declare function readTranscript(path: string, options?: {
+    readonly inspection?: ScanSource;
+    readonly prefixBytes?: number;
+}): Promise<ReadTranscript>;

package/dist/src/chats/reader.js

@@ -1,6 +1,7 @@
 import { createHash } from "node:crypto";
 import { constants } from "node:fs";
 import { lstat, open } from "node:fs/promises";
+import { sourceFailure, verifyScanSource, verifyOpenSource, } from "./source.js";
 import { normalizeTranscript, } from "./normalize.js";
 export const transcriptLimit = 10_485_760;
 function refused(path, reason) {
@@ -37,7 +38,8 @@ async function boundedBytes(handle) {
         return "file-too-large";
     return Buffer.concat(chunks, total);
 }
-function normalizeBytes(path, bytes) {
+function normalizeBytes(path, options) {
+    const { bytes } = options;
     const digest = `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
     let text;
     try {
@@ -54,34 +56,63 @@ function normalizeBytes(path, bytes) {
             status: result.status,
             reason: result.reason,
             digest,
+            ...(result.status === "included" ? { byteSize: bytes.length } : {}),
             recordCount: result.records.length,
             omissions: result.omissions,
         },
         records: result.records,
+        ...(options.prefixBytes !== undefined &&
+            options.prefixBytes >= 0 &&
+            options.prefixBytes < bytes.length
+            ? {
+                prefixDigest: `sha256:${createHash("sha256").update(bytes.subarray(0, options.prefixBytes)).digest("hex")}`,
+            }
+            : {}),
     };
 }
-export async function readTranscript(path) {
+async function readSourceBytes(path, inspection) {
+    const handle = await open(path, constants.O_RDONLY |
+        constants.O_NONBLOCK |
+        (inspection === undefined ? constants.O_NOFOLLOW : 0x20000000));
     try {
+        if (inspection !== undefined) {
+            verifyOpenSource(inspection, await handle.stat());
+            await verifyScanSource(inspection);
+        }
+        return await boundedBytes(handle);
+    }
+    finally {
+        await handle.close();
+    }
+}
+export async function readTranscript(path, options = {}) {
+    const { inspection } = options;
+    try {
+        if (inspection !== undefined)
+            await verifyScanSource(inspection);
         const stats = await lstat(path);
         if (stats.isSymbolicLink())
             return refused(path, "symbolic-link");
         if (!stats.isFile())
             return refused(path, "nonregular-file");
-        const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
-        let bytes;
-        try {
-            bytes = await boundedBytes(handle);
-        }
-        finally {
-            await handle.close();
-        }
-        return typeof bytes === "string"
+        const bytes = await readSourceBytes(path, inspection);
+        const result = typeof bytes === "string"
             ? refused(path, bytes)
-            : normalizeBytes(path, bytes);
+            : normalizeBytes(path, { bytes, ...options });
+        return scannedResult(result, inspection);
     }
     catch (error) {
-        if (error instanceof Error && "code" in error && error.code === "ELOOP")
-            return refused(path, "symbolic-link");
-        return refused(path, "source-unreadable");
+        return readFailure(path, { error, inspection });
     }
 }
+function scannedResult(result, inspection) {
+    if (inspection === undefined || result.coverage.status !== "too-large")
+        return result;
+    return { ...result, coverage: { ...result.coverage, status: "unsupported" } };
+}
+function readFailure(path, data) {
+    const { error, inspection } = data;
+    if (error instanceof Error && "code" in error && error.code === "ELOOP")
+        return refused(path, "symbolic-link");
+    return refused(path, inspection === undefined ? "source-unreadable" : sourceFailure(error));
+}

package/dist/src/chats/redact-import.d.ts

@@ -0,0 +1,14 @@
+import { type RedactionSettings } from "./redaction.ts";
+export interface RedactImportRequest {
+    readonly importId: string;
+    readonly workingDirectory: string;
+    readonly runtimeDirectory: string | undefined;
+    readonly settings: RedactionSettings;
+}
+export declare function redactImport(request: RedactImportRequest): Promise<{
+    code: string;
+    importId: string;
+    recordCount: number;
+    counts: import("./redaction.ts").RedactionCounts;
+    totalCount: number;
+}>;

package/dist/src/chats/redact-import.js

@@ -0,0 +1,141 @@
+import { openInput, readInput } from "./input-file.js";
+import { lstat, rm } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { createPrivateFile, validateDestination } from "./destination.js";
+import { parseCorpusRecord, validateCoverage } from "./corpus-input.js";
+import { emptyCounts, redactRecord, categories, } from "./redaction.js";
+async function absent(path) {
+    try {
+        await lstat(path);
+    }
+    catch (error) {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return;
+        throw error;
+    }
+    throw new Error("outputs-exist");
+}
+async function* corpusLines(handle) {
+    const buffer = Buffer.alloc(65_536);
+    const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+    let pending = "";
+    let bytesRead = -1;
+    while (bytesRead !== 0) {
+        ({ bytesRead } = await handle.read({ buffer, position: null }));
+        pending += decoder.decode(buffer.subarray(0, bytesRead), {
+            stream: bytesRead !== 0,
+        });
+        let start = 0;
+        let end = pending.indexOf("\n");
+        while (end !== -1) {
+            yield pending.slice(start, end);
+            start = end + 1;
+            end = pending.indexOf("\n", start);
+        }
+        pending = pending.slice(start);
+    }
+    if (pending !== "")
+        yield pending;
+}
+async function redactCorpus(handles, settings) {
+    const counts = emptyCounts();
+    let recordCount = 0;
+    for await (const line of corpusLines(handles.input)) {
+        const result = redactRecord(parseCorpusRecord(line), settings);
+        for (const category of categories)
+            counts[category] += result.counts[category];
+        await handles.output.writeFile(JSON.stringify(result.record) + "\n");
+        recordCount++;
+    }
+    return {
+        counts,
+        recordCount,
+        totalCount: Object.values(counts).reduce((sum, count) => sum + count, 0),
+    };
+}
+async function writeRedactedCorpus(directory, settings) {
+    const path = join(directory, "redacted.jsonl");
+    let created = false;
+    try {
+        const input = await openInput(join(directory, "corpus.jsonl"));
+        try {
+            const output = await createPrivateFile(path);
+            created = true;
+            try {
+                return await redactCorpus({ input, output }, settings);
+            }
+            finally {
+                await output.close();
+            }
+        }
+        finally {
+            await input.close();
+        }
+    }
+    catch (error) {
+        if (created) {
+            await validateDestination(directory);
+            await rm(path, { force: true });
+        }
+        throw error;
+    }
+}
+async function writeManifest(directory, manifest) {
+    const path = join(directory, "redaction.json");
+    let created = false;
+    try {
+        const handle = await createPrivateFile(path);
+        created = true;
+        try {
+            await handle.writeFile(JSON.stringify(manifest) + "\n");
+        }
+        finally {
+            await handle.close();
+        }
+    }
+    catch (error) {
+        if (created) {
+            await validateDestination(directory);
+            await rm(path, { force: true });
+        }
+        throw error;
+    }
+}
+export async function redactImport(request) {
+    const empty = {
+        importId: request.importId,
+        recordCount: 0,
+        counts: emptyCounts(),
+        totalCount: 0,
+    };
+    if (process.platform !== "darwin")
+        return { ...empty, code: "platform-unsupported" };
+    if (request.runtimeDirectory === undefined)
+        return { ...empty, code: "runtime-directory-unknown" };
+    try {
+        if (!/^[a-f\d]{8}-[a-f\d]{4}-4[a-f\d]{3}-[89ab][a-f\d]{3}-[a-f\d]{12}$/.test(request.importId))
+            throw new Error("invalid-import");
+        const directory = resolve(request.workingDirectory, request.runtimeDirectory, "chats", request.importId);
+        await validateDestination(directory);
+        if (!(await lstat(directory)).isDirectory())
+            throw new Error("invalid-import");
+        await absent(join(directory, "redacted.jsonl"));
+        await absent(join(directory, "redaction.json"));
+        validateCoverage(await readInput(join(directory, "coverage.json")), request.importId);
+        const summary = await writeRedactedCorpus(directory, request.settings);
+        await writeManifest(directory, {
+            schemaVersion: 1,
+            importId: request.importId,
+            ruleVersion: 1,
+            ...summary,
+        });
+        return {
+            code: "redaction-complete",
+            importId: request.importId,
+            ...summary,
+        };
+    }
+    catch {
+        return { ...empty, code: "redaction-failed" };
+    }
+}