cloving 0.3.20 → 1.1.0

package/dist/src/chats/corpus-input.js

@@ -0,0 +1,288 @@
+import { parsePreservedJson } from "./normalize.js";
+function object(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function keys(value, fields) {
+    const { required, optional = [] } = fields;
+    return (required.every((key) => Object.hasOwn(value, key)) &&
+        Object.keys(value).every((key) => required.includes(key) || optional.includes(key)));
+}
+function natural(value) {
+    return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+}
+function optionalText(value) {
+    return value === null || typeof value === "string";
+}
+function block(value) {
+    if (!object(value) || !natural(value["position"]))
+        return false;
+    if (value["type"] === "text")
+        return (keys(value, { required: ["type", "position", "text"] }) &&
+            typeof value["text"] === "string");
+    if (value["type"] === "tool_use")
+        return (keys(value, { required: ["type", "position", "id", "name", "input"] }) &&
+            typeof value["id"] === "string" &&
+            typeof value["name"] === "string");
+    if (value["type"] !== "tool_result")
+        return false;
+    return toolResult(value);
+}
+function toolResult(value) {
+    return (keys(value, {
+        required: ["type", "position", "tool_use_id", "content"],
+        optional: ["is_error"],
+    }) &&
+        typeof value["tool_use_id"] === "string" &&
+        (!Object.hasOwn(value, "is_error") ||
+            typeof value["is_error"] === "boolean"));
+}
+function identityFields(value) {
+    return ((value["uuid"] === null ||
+        (typeof value["uuid"] === "string" && value["uuid"] !== "")) &&
+        (value["isSidechain"] === null ||
+            typeof value["isSidechain"] === "boolean") &&
+        (!Object.hasOwn(value, "parentUuid") ||
+            value["parentUuid"] === null ||
+            (typeof value["parentUuid"] === "string" && value["parentUuid"] !== "")));
+}
+function versionFields(value) {
+    const required = [
+        "schemaVersion",
+        "sourceId",
+        "role",
+        "timestamp",
+        "sessionId",
+        "text",
+        "blocks",
+    ];
+    const version = value["schemaVersion"];
+    if (version !== 1 && version !== 2)
+        return false;
+    const fields = version === 1
+        ? { required }
+        : {
+            required: [...required, "uuid", "isSidechain"],
+            optional: ["parentUuid"],
+        };
+    return keys(value, fields) && (version === 1 || identityFields(value));
+}
+function messageFields(value) {
+    return (versionFields(value) &&
+        typeof value["sourceId"] === "string" &&
+        value["sourceId"] !== "" &&
+        ["user", "assistant"].includes(String(value["role"])) &&
+        optionalText(value["timestamp"]) &&
+        optionalText(value["sessionId"]) &&
+        typeof value["text"] === "string");
+}
+function message(value) {
+    if (!object(value) || !messageFields(value))
+        return false;
+    if (!Array.isArray(value["blocks"]) || !value["blocks"].every(block))
+        return false;
+    let position = -1;
+    for (const item of value["blocks"]) {
+        const next = item["position"];
+        if (!natural(next) || next <= position)
+            return false;
+        position = next;
+    }
+    return true;
+}
+export function parseCorpusRecord(line) {
+    const value = parsePreservedJson(line);
+    if (!message(value))
+        throw new Error("invalid-import");
+    return value;
+}
+export function parseCorpus(text) {
+    const lines = text.split("\n");
+    if (lines.at(-1) === "")
+        lines.pop();
+    return lines.map(parseCorpusRecord);
+}
+function coverageFile(value) {
+    if (!object(value) ||
+        typeof value["path"] !== "string" ||
+        typeof value["reason"] !== "string")
+        return false;
+    if (!["included", "unreadable", "unsupported", "too-large"].includes(String(value["status"])))
+        return false;
+    if (!optionalText(value["digest"]) || !natural(value["recordCount"]))
+        return false;
+    if (!coverageSize(value))
+        return false;
+    const omissions = value["omissions"];
+    return (object(omissions) &&
+        natural(omissions["metadataRecords"]) &&
+        natural(omissions["thinkingBlocks"]));
+}
+export function validateCoverage(text, importId) {
+    const value = JSON.parse(text);
+    if (!object(value) || value["importId"] !== importId)
+        throw new Error("invalid-import");
+    if (value["schemaVersion"] === 2 && scanCoverage(value))
+        return;
+    if (value["schemaVersion"] === 1 &&
+        Array.isArray(value["files"]) &&
+        value["files"].every(coverageFile))
+        return;
+    throw new Error("invalid-import");
+}
+function scanCoverage(value) {
+    const roots = value["roots"];
+    const exclusions = value["exclusions"];
+    const limits = value["limits"];
+    const entries = value["entries"];
+    return (keys(value, {
+        required: [
+            "schemaVersion",
+            "importId",
+            "roots",
+            "exclusions",
+            "limits",
+            "entries",
+        ],
+    }) &&
+        Array.isArray(roots) &&
+        roots.length > 0 &&
+        roots.every(absolutePath) &&
+        Array.isArray(exclusions) &&
+        exclusions.every((name) => typeof name === "string" &&
+            name !== "" &&
+            name !== "." &&
+            name !== ".." &&
+            !["/", "\\", "\0", "*", "?", "[", "]"].some((character) => name.includes(character))) &&
+        coverageLimits(limits) &&
+        Array.isArray(entries) &&
+        entries.every((entry) => scanCoverageEntry(entry, roots)));
+}
+function coverageLimits(value) {
+    return (object(value) &&
+        keys(value, {
+            required: ["depth", "files", "sourceBytes", "corpusBytes"],
+        }) &&
+        value["depth"] === 4 &&
+        value["files"] === 100 &&
+        value["sourceBytes"] === 10_485_760 &&
+        value["corpusBytes"] === 52_428_800);
+}
+function scanCoverageEntry(value, roots) {
+    if (!object(value) || !coverageFile({ ...value, status: "included" }))
+        return false;
+    if (!keys(value, {
+        required: [
+            "kind",
+            "path",
+            "selection",
+            "status",
+            "reason",
+            "digest",
+            "recordCount",
+            "omissions",
+        ],
+        optional: ["byteSize"],
+    }))
+        return false;
+    const selection = value["selection"];
+    if (!coverageSelection(selection, roots.length))
+        return false;
+    if (!["file", "subtree"].includes(String(value["kind"])))
+        return false;
+    if (selection["kind"] === "file" && value["kind"] !== "file")
+        return false;
+    if (selection.kind === "root" &&
+        !withinRoot(roots[selection.index], value["path"]))
+        return false;
+    return scanStatus(value, selection["kind"]);
+}
+function coverageSelection(value, rootCount) {
+    if (!object(value) ||
+        !keys(value, { required: ["kind", "index"] }) ||
+        !natural(value["index"]))
+        return false;
+    return (value["kind"] === "file" ||
+        (value["kind"] === "root" && value["index"] < rootCount));
+}
+function scanStatus(value, selection) {
+    const status = value["status"];
+    const reason = value["reason"];
+    if (status === "included")
+        return (value["kind"] === "file" &&
+            reason === "supported" &&
+            typeof value["digest"] === "string");
+    if (status === "unreadable")
+        return reason === "source-unreadable" && emptyCoverageEntry(value);
+    if (["too-large", "excluded", "pending"].includes(String(status)))
+        return scanOmission(value, selection);
+    return scanRefusal(value);
+}
+function scanRefusal(value) {
+    if (value["status"] !== "unsupported" || value["recordCount"] !== 0)
+        return false;
+    const contentReasons = [
+        "invalid-utf8",
+        "empty-file",
+        "invalid-json",
+        "unsupported-record",
+        "unsupported-message",
+        "unsupported-block",
+    ];
+    if (contentReasons.includes(String(value["reason"])))
+        return value["kind"] === "file" && typeof value["digest"] === "string";
+    const reasons = value["kind"] === "subtree"
+        ? [
+            "symbolic-link",
+            "nonregular-file",
+            "outside-root",
+            "repository-root",
+            "source-changed",
+            "non-directory-root",
+        ]
+        : [
+            "symbolic-link",
+            "nonregular-file",
+            "outside-root",
+            "source-changed",
+            "unsupported-extension",
+            "file-too-large",
+        ];
+    return reasons.includes(String(value["reason"])) && emptyCoverageEntry(value);
+}
+function absolutePath(value) {
+    return (typeof value === "string" &&
+        value.startsWith("/") &&
+        !value.includes("\0") &&
+        value.split("/").every((part) => part !== "." && part !== ".."));
+}
+export function withinRoot(root, path) {
+    if (root === undefined || !absolutePath(path))
+        return false;
+    const parent = root.split("/").filter(Boolean);
+    const child = path.split("/").filter(Boolean);
+    return parent.every((part, index) => part === child[index]);
+}
+function scanOmission(value, selection) {
+    if (!emptyCoverageEntry(value))
+        return false;
+    if (value["status"] === "too-large")
+        return selection === "file" && value["reason"] === "file-too-large";
+    if (selection !== "root")
+        return false;
+    if (value["status"] === "excluded")
+        return (value["kind"] === "subtree" && value["reason"] === "project-excluded");
+    return (value["reason"] ===
+        (value["kind"] === "subtree" ? "depth-limit" : "file-limit"));
+}
+function emptyCoverageEntry(value) {
+    const omissions = value["omissions"];
+    return (value["digest"] === null &&
+        value["recordCount"] === 0 &&
+        object(omissions) &&
+        omissions["metadataRecords"] === 0 &&
+        omissions["thinkingBlocks"] === 0);
+}
+function coverageSize(value) {
+    return (!Object.hasOwn(value, "byteSize") ||
+        (natural(value["byteSize"]) && value["byteSize"] <= 10_485_760));
+}

package/dist/src/chats/destination.d.ts

@@ -0,0 +1,4 @@
+import { type FileHandle } from "node:fs/promises";
+export declare function importDirectory(runtime: string, importId: string): Promise<string>;
+export declare function validateDestination(directory: string): Promise<void>;
+export declare function createPrivateFile(path: string): Promise<FileHandle>;

package/dist/src/chats/destination.js

@@ -0,0 +1,96 @@
+import { constants } from "node:fs";
+import { lstat, mkdir, open, realpath, } from "node:fs/promises";
+import { basename, dirname, join, resolve } from "node:path";
+function missing(error) {
+    return error instanceof Error && "code" in error && error.code === "ENOENT";
+}
+async function canonical(path) {
+    try {
+        await lstat(path);
+    }
+    catch (error) {
+        if (!missing(error))
+            throw error;
+        return join(await canonical(dirname(path)), basename(path));
+    }
+    return realpath(path);
+}
+async function outsideRepository(path) {
+    let ancestor = path;
+    for (;;) {
+        try {
+            await lstat(join(ancestor, ".git"));
+            throw new Error("destination-refused");
+        }
+        catch (error) {
+            if (!missing(error))
+                throw error;
+        }
+        const parent = dirname(ancestor);
+        if (parent === ancestor)
+            return;
+        ancestor = parent;
+    }
+}
+async function privateDirectory(path) {
+    try {
+        const stats = await lstat(path);
+        if (!stats.isDirectory())
+            throw new Error("destination-refused");
+        return;
+    }
+    catch (error) {
+        if (!missing(error))
+            throw error;
+    }
+    await privateDirectory(dirname(path));
+    try {
+        await mkdir(path, { mode: 0o700 });
+    }
+    catch (error) {
+        if (!(error instanceof Error && "code" in error && error.code === "EEXIST"))
+            throw error;
+        if (!(await lstat(path)).isDirectory())
+            throw new Error("destination-refused", { cause: error });
+    }
+}
+export async function importDirectory(runtime, importId) {
+    if (process.platform !== "darwin")
+        throw new Error("platform-unsupported");
+    let chats;
+    try {
+        chats = await canonical(join(resolve(runtime), "chats"));
+        await outsideRepository(chats);
+    }
+    catch {
+        throw new Error("destination-refused");
+    }
+    await privateDirectory(chats);
+    await validateDestination(chats);
+    const directory = join(chats, importId);
+    await mkdir(directory, { mode: 0o700 });
+    await validateDestination(directory);
+    return directory;
+}
+export async function validateDestination(directory) {
+    if (process.platform !== "darwin")
+        throw new Error("platform-unsupported");
+    if ((await realpath(directory)) !== directory)
+        throw new Error("destination-refused");
+    await outsideRepository(directory);
+}
+export async function createPrivateFile(path) {
+    await validateDestination(dirname(path));
+    const darwinNoFollowAny = 0x20000000;
+    try {
+        return await open(path, constants.O_WRONLY |
+            constants.O_CREAT |
+            constants.O_EXCL |
+            darwinNoFollowAny, 0o600);
+    }
+    catch (error) {
+        if (error instanceof Error && "code" in error && error.code === "ELOOP")
+            throw new Error("destination-refused", { cause: error });
+        throw error;
+    }
+}

package/dist/src/chats/import.d.ts

@@ -0,0 +1,15 @@
+import { type StructureCounts } from "./structure.ts";
+import { type SelectionRequest } from "./selection.ts";
+export interface ImportRequest extends SelectionRequest {
+    readonly runtimeDirectory: string | undefined;
+    readonly previousImportId?: string;
+}
+export interface ImportSummary extends StructureCounts {
+    readonly code: "import-complete" | "import-partial" | "runtime-directory-unknown" | "platform-unsupported" | "destination-refused" | "output-failed" | "previous-import-refused";
+    readonly importId: string | null;
+    readonly included: number;
+    readonly refused: number;
+    readonly excluded?: number;
+    readonly pending?: number;
+}
+export declare function importChats(request: ImportRequest): Promise<ImportSummary>;

package/dist/src/chats/import.js

@@ -0,0 +1,243 @@
+import { readPreviousImport } from "./previous-import.js";
+import { pairEvidence, compareEvidence, } from "./revision.js";
+import { structureImport } from "./structure.js";
+import { randomUUID } from "node:crypto";
+import { rm } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { createPrivateFile, importDirectory, validateDestination, } from "./destination.js";
+import { readTranscript } from "./reader.js";
+import { selectChatSources, } from "./selection.js";
+const corpusLimit = 52_428_800;
+function boundedCorpusLine(record, previousBytes) {
+    const line = `${JSON.stringify(record)}\n`;
+    const bytes = previousBytes + Buffer.byteLength(line);
+    if (bytes > corpusLimit)
+        throw new Error("corpus-too-large");
+    return { line, bytes };
+}
+async function writeImport(directory, data) {
+    const { request, previous } = data;
+    const selection = await selectChatSources(request);
+    const keys = selection.entries.map((selected) => selectionKey(selected, selection.roots));
+    const pairs = pairEvidence({ previous, current: keys });
+    const evidence = [];
+    const files = [];
+    const records = [];
+    let corpusBytes = 0;
+    const corpus = await createPrivateFile(join(directory, "corpus.jsonl"));
+    try {
+        for (const [index, selected] of selection.entries.entries()) {
+            const matched = pairs[index];
+            const before = matched === null || matched === undefined
+                ? undefined
+                : previous[matched];
+            const result = await readSelected(selected, before?.byteSize);
+            evidence.push({
+                ...selectionKey(selected, selection.roots),
+                ...result.coverage,
+                ...(result.prefixDigest === undefined
+                    ? {}
+                    : { prefixDigest: result.prefixDigest }),
+            });
+            files.push({
+                ...result.coverage,
+                kind: selected.kind,
+                selection: selected.selection,
+            });
+            for (const record of result.records) {
+                const next = boundedCorpusLine(record, corpusBytes);
+                await corpus.writeFile(next.line);
+                corpusBytes = next.bytes;
+                records.push(record);
+            }
+        }
+    }
+    finally {
+        await corpus.close();
+    }
+    return { roots: selection.roots, files, records, evidence };
+}
+function selectionKey(selected, roots) {
+    return {
+        path: selected.path,
+        root: selected.selection.kind === "root"
+            ? (roots[selected.selection.index] ?? null)
+            : null,
+    };
+}
+async function readSelected(selected, prefixBytes) {
+    if (selected.outcome !== undefined)
+        return {
+            coverage: {
+                path: selected.path,
+                ...selected.outcome,
+                digest: null,
+                recordCount: 0,
+                omissions: { metadataRecords: 0, thinkingBlocks: 0 },
+            },
+            records: [],
+        };
+    return readTranscript(selected.path, {
+        ...(selected.inspection === undefined
+            ? {}
+            : { inspection: selected.inspection }),
+        ...(prefixBytes === undefined ? {} : { prefixBytes }),
+    });
+}
+async function writeMetadata(path, data) {
+    const handle = await createPrivateFile(path);
+    try {
+        await handle.writeFile(`${JSON.stringify({ schemaVersion: 1, ...data }, null, 2)}\n`);
+    }
+    finally {
+        await handle.close();
+    }
+}
+async function writeCoverage(directory, coverage) {
+    const manifest = join(directory, "coverage.json");
+    let created = false;
+    try {
+        const handle = await createPrivateFile(manifest);
+        created = true;
+        try {
+            await handle.writeFile(`${JSON.stringify(coverage, null, 2)}\n`);
+        }
+        finally {
+            await handle.close();
+        }
+    }
+    catch (error) {
+        if (created) {
+            await validateDestination(directory);
+            await rm(manifest, { force: true });
+        }
+        throw error;
+    }
+}
+export async function importChats(request) {
+    const empty = emptyImport(request);
+    if (process.platform !== "darwin")
+        return { ...empty, code: "platform-unsupported" };
+    if (request.runtimeDirectory === undefined)
+        return { ...empty, code: "runtime-directory-unknown" };
+    try {
+        const runtime = resolve(request.workingDirectory, request.runtimeDirectory);
+        const previous = request.previousImportId === undefined
+            ? undefined
+            : await readPreviousImport(runtime, request.previousImportId);
+        const importId = randomUUID();
+        const directory = await importDirectory(runtime, importId);
+        return await completeImport(directory, { request, previous, importId });
+    }
+    catch (error) {
+        return {
+            ...empty,
+            code: error instanceof Error &&
+                (error.message === "destination-refused" ||
+                    error.message === "previous-import-refused")
+                ? error.message
+                : "output-failed",
+        };
+    }
+}
+function importTotals(files) {
+    return {
+        included: files.filter((file) => file.status === "included").length,
+        refused: files.filter((file) => ["unreadable", "unsupported", "too-large"].includes(file.status)).length,
+        excluded: files.filter((file) => file.status === "excluded").length,
+        pending: files.filter((file) => file.status === "pending").length,
+    };
+}
+function coverageManifest(data) {
+    const { importId, roots, files, exclusions } = data;
+    if (roots.length > 0)
+        return {
+            schemaVersion: 2,
+            importId,
+            roots,
+            exclusions,
+            limits: {
+                depth: 4,
+                files: 100,
+                sourceBytes: 10_485_760,
+                corpusBytes: corpusLimit,
+            },
+            entries: files,
+        };
+    return {
+        schemaVersion: 1,
+        importId,
+        files: files.map((entry) => ({
+            path: entry.path,
+            status: entry.status,
+            reason: entry.reason,
+            digest: entry.digest,
+            ...(entry.byteSize === undefined ? {} : { byteSize: entry.byteSize }),
+            recordCount: entry.recordCount,
+            omissions: entry.omissions,
+        })),
+    };
+}
+function emptyImport(request) {
+    return {
+        importId: null,
+        included: 0,
+        refused: 0,
+        ...((request.roots?.length ?? 0) === 0 ? {} : { excluded: 0, pending: 0 }),
+        ...structureImport([]).counts,
+    };
+}
+async function completeImport(directory, data) {
+    const { request, previous, importId } = data;
+    const { roots, files, records, evidence } = await writeImport(directory, {
+        request,
+        previous: previous?.entries ?? [],
+    });
+    const structure = structureImport(records);
+    await writeMetadata(join(directory, "structure.json"), {
+        importId,
+        ...structure,
+    });
+    const partial = await writeRevision(directory, {
+        previous,
+        evidence,
+        importId,
+    });
+    const totals = importTotals(files);
+    const { refused, pending } = totals;
+    await writeCoverage(directory, coverageManifest({
+        importId,
+        roots,
+        files,
+        exclusions: request.excludeProjects ?? [],
+    }));
+    return {
+        code: refused + pending + partial === 0 ? "import-complete" : "import-partial",
+        importId,
+        included: totals.included,
+        refused,
+        ...(roots.length === 0 ? {} : { excluded: totals.excluded, pending }),
+        ...structure.counts,
+    };
+}
+async function writeRevision(directory, data) {
+    const { previous, evidence, importId } = data;
+    if (previous === undefined)
+        return 0;
+    const comparison = compareEvidence({
+        previous: previous.entries,
+        current: evidence,
+    });
+    try {
+        await writeMetadata(join(directory, "revision.json"), {
+            importId,
+            previousImportId: previous.importId,
+            evidenceRevision: previous.evidenceRevision,
+            ...comparison,
+        });
+    }
+    catch {
+        throw new Error("output-failed");
+    }
+    return comparison.counts.missing + comparison.counts.truncated;
+}

package/dist/src/chats/input-file.d.ts

@@ -0,0 +1,3 @@
+import { type FileHandle } from "node:fs/promises";
+export declare function openInput(path: string): Promise<FileHandle>;
+export declare function readInput(path: string): Promise<string>;

package/dist/src/chats/input-file.js

@@ -0,0 +1,32 @@
+import { constants } from "node:fs";
+import { lstat, open } from "node:fs/promises";
+export async function openInput(path) {
+    if (process.platform !== "darwin")
+        throw new Error("platform-unsupported");
+    const before = await lstat(path);
+    if (!before.isFile() || before.nlink !== 1)
+        throw new Error("invalid-import");
+    const handle = await open(path, constants.O_RDONLY | constants.O_NONBLOCK | 0x20000000);
+    try {
+        const opened = await handle.stat();
+        if (!opened.isFile() ||
+            opened.nlink !== 1 ||
+            opened.dev !== before.dev ||
+            opened.ino !== before.ino)
+            throw new Error("invalid-import");
+        return handle;
+    }
+    catch (error) {
+        await handle.close();
+        throw error;
+    }
+}
+export async function readInput(path) {
+    const handle = await openInput(path);
+    try {
+        return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(await handle.readFile());
+    }
+    finally {
+        await handle.close();
+    }
+}