cloving 1.1.0 → 1.3.0

package/dist/src/chats/redact-import.js

@@ -1,141 +0,0 @@
-import { openInput, readInput } from "./input-file.js";
-import { lstat, rm } from "node:fs/promises";
-import { join, resolve } from "node:path";
-import { createPrivateFile, validateDestination } from "./destination.js";
-import { parseCorpusRecord, validateCoverage } from "./corpus-input.js";
-import { emptyCounts, redactRecord, categories, } from "./redaction.js";
-async function absent(path) {
-    try {
-        await lstat(path);
-    }
-    catch (error) {
-        if (error instanceof Error && "code" in error && error.code === "ENOENT")
-            return;
-        throw error;
-    }
-    throw new Error("outputs-exist");
-}
-async function* corpusLines(handle) {
-    const buffer = Buffer.alloc(65_536);
-    const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
-    let pending = "";
-    let bytesRead = -1;
-    while (bytesRead !== 0) {
-        ({ bytesRead } = await handle.read({ buffer, position: null }));
-        pending += decoder.decode(buffer.subarray(0, bytesRead), {
-            stream: bytesRead !== 0,
-        });
-        let start = 0;
-        let end = pending.indexOf("\n");
-        while (end !== -1) {
-            yield pending.slice(start, end);
-            start = end + 1;
-            end = pending.indexOf("\n", start);
-        }
-        pending = pending.slice(start);
-    }
-    if (pending !== "")
-        yield pending;
-}
-async function redactCorpus(handles, settings) {
-    const counts = emptyCounts();
-    let recordCount = 0;
-    for await (const line of corpusLines(handles.input)) {
-        const result = redactRecord(parseCorpusRecord(line), settings);
-        for (const category of categories)
-            counts[category] += result.counts[category];
-        await handles.output.writeFile(JSON.stringify(result.record) + "\n");
-        recordCount++;
-    }
-    return {
-        counts,
-        recordCount,
-        totalCount: Object.values(counts).reduce((sum, count) => sum + count, 0),
-    };
-}
-async function writeRedactedCorpus(directory, settings) {
-    const path = join(directory, "redacted.jsonl");
-    let created = false;
-    try {
-        const input = await openInput(join(directory, "corpus.jsonl"));
-        try {
-            const output = await createPrivateFile(path);
-            created = true;
-            try {
-                return await redactCorpus({ input, output }, settings);
-            }
-            finally {
-                await output.close();
-            }
-        }
-        finally {
-            await input.close();
-        }
-    }
-    catch (error) {
-        if (created) {
-            await validateDestination(directory);
-            await rm(path, { force: true });
-        }
-        throw error;
-    }
-}
-async function writeManifest(directory, manifest) {
-    const path = join(directory, "redaction.json");
-    let created = false;
-    try {
-        const handle = await createPrivateFile(path);
-        created = true;
-        try {
-            await handle.writeFile(JSON.stringify(manifest) + "\n");
-        }
-        finally {
-            await handle.close();
-        }
-    }
-    catch (error) {
-        if (created) {
-            await validateDestination(directory);
-            await rm(path, { force: true });
-        }
-        throw error;
-    }
-}
-export async function redactImport(request) {
-    const empty = {
-        importId: request.importId,
-        recordCount: 0,
-        counts: emptyCounts(),
-        totalCount: 0,
-    };
-    if (process.platform !== "darwin")
-        return { ...empty, code: "platform-unsupported" };
-    if (request.runtimeDirectory === undefined)
-        return { ...empty, code: "runtime-directory-unknown" };
-    try {
-        if (!/^[a-f\d]{8}-[a-f\d]{4}-4[a-f\d]{3}-[89ab][a-f\d]{3}-[a-f\d]{12}$/.test(request.importId))
-            throw new Error("invalid-import");
-        const directory = resolve(request.workingDirectory, request.runtimeDirectory, "chats", request.importId);
-        await validateDestination(directory);
-        if (!(await lstat(directory)).isDirectory())
-            throw new Error("invalid-import");
-        await absent(join(directory, "redacted.jsonl"));
-        await absent(join(directory, "redaction.json"));
-        validateCoverage(await readInput(join(directory, "coverage.json")), request.importId);
-        const summary = await writeRedactedCorpus(directory, request.settings);
-        await writeManifest(directory, {
-            schemaVersion: 1,
-            importId: request.importId,
-            ruleVersion: 1,
-            ...summary,
-        });
-        return {
-            code: "redaction-complete",
-            importId: request.importId,
-            ...summary,
-        };
-    }
-    catch {
-        return { ...empty, code: "redaction-failed" };
-    }
-}

package/dist/src/chats/redaction.d.ts

@@ -1,41 +0,0 @@
-import type { MessageRecord } from "./normalize.ts";
-export interface RedactionSettings {
-    readonly home: string | null;
-    readonly privateTexts: readonly string[];
-}
-export declare const replacements: {
-    readonly privateKey: "[PRIVATE_KEY]";
-    readonly credentialUrl: "[CREDENTIAL_URL]";
-    readonly homePath: "[HOME_PATH]";
-    readonly token: "[TOKEN]";
-    readonly email: "[EMAIL]";
-    readonly privateText: "[PRIVATE_TEXT]";
-};
-export declare const categories: readonly ["privateKey", "credentialUrl", "homePath", "token", "email", "privateText"];
-export type Category = keyof typeof replacements;
-export type RedactionCounts = Record<Category, number>;
-export declare function emptyCounts(): RedactionCounts;
-export declare function redactRecord(record: MessageRecord, settings: RedactionSettings): {
-    record: {
-        text: string;
-        blocks: Record<string, unknown>[];
-        sourceId: string;
-        role: "user" | "assistant";
-        timestamp: string | null;
-        sessionId: string | null;
-        schemaVersion: 1;
-    } | {
-        text: string;
-        blocks: Record<string, unknown>[];
-        sourceId: string;
-        role: "user" | "assistant";
-        timestamp: string | null;
-        sessionId: string | null;
-        schemaVersion: 2;
-        uuid: string | null;
-        isSidechain: boolean | null;
-        parentUuid?: string | null;
-    };
-    counts: RedactionCounts;
-    totalCount: number;
-};

package/dist/src/chats/redaction.js

@@ -1,161 +0,0 @@
-export const replacements = {
-    privateKey: "[PRIVATE_KEY]",
-    credentialUrl: "[CREDENTIAL_URL]",
-    homePath: "[HOME_PATH]",
-    token: "[TOKEN]",
-    email: "[EMAIL]",
-    privateText: "[PRIVATE_TEXT]",
-};
-export const categories = [
-    "privateKey",
-    "credentialUrl",
-    "homePath",
-    "token",
-    "email",
-    "privateText",
-];
-export function emptyCounts() {
-    return {
-        privateKey: 0,
-        credentialUrl: 0,
-        homePath: 0,
-        token: 0,
-        email: 0,
-        privateText: 0,
-    };
-}
-function escaped(text) {
-    return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-function patternSpans(text, rule) {
-    return [...text.matchAll(rule.pattern)].map((match) => ({
-        start: match.index,
-        end: match.index + match[0].length,
-        category: rule.category,
-    }));
-}
-function homeRoot(home) {
-    if (home === null || home === "")
-        return "~";
-    if (home === "/")
-        return "(?:(?=/)|~)";
-    return `(?:${escaped(home.replace(/\/$/, ""))}|~)`;
-}
-function homeSpans(text, home) {
-    const root = homeRoot(home);
-    const pattern = new RegExp(String.raw `(?<![\p{L}\p{N}_./~\\-])${root}(?=$|[\s/\x22'\x60<>,;:!?()\[\]{}])(?:/[^\s\x22'\x60<>]*)?`, "gu");
-    return patternSpans(text, { pattern, category: "homePath" }).map((span) => {
-        const quote = text[span.start - 1];
-        if (quote !== '"' && quote !== "'" && quote !== "`")
-            return span;
-        const end = text.indexOf(quote, span.end);
-        return end === -1 ? span : { ...span, end };
-    });
-}
-function spans(text, settings) {
-    const rules = [
-        {
-            pattern: /-----BEGIN ((?:[A-Z0-9]+ )*PRIVATE KEY)-----[\s\S]*?-----END \1-----/g,
-            category: "privateKey",
-        },
-        {
-            pattern: /\b[a-z][a-z\d+.-]*:\/\/[^\s<>"'`/?#]*@[^\s<>"'`]+/gi,
-            category: "credentialUrl",
-        },
-        {
-            pattern: /(?<![\p{L}\p{N}_])(?:gh[pousr]_|github_pat_|sk-ant-|sk-(?:proj-|svcacct-)?)[a-z\d_-]+/giu,
-            category: "token",
-        },
-        {
-            pattern: /[\w.!#$%&'*+/=?^`{|}~-]+@[\w.-]+\.[a-z]{2,}/gi,
-            category: "email",
-        },
-    ];
-    const matches = rules.flatMap((rule) => patternSpans(text, rule));
-    for (const match of homeSpans(text, settings.home))
-        matches.push(match);
-    for (const privateText of settings.privateTexts) {
-        if (privateText.trim() === "")
-            throw new Error("invalid-settings");
-        const pattern = new RegExp(`(?=(${escaped(privateText)}))`, "gi");
-        for (const match of text.matchAll(pattern))
-            matches.push({
-                start: match.index,
-                end: match.index + String(match[1]).length,
-                category: "privateText",
-            });
-    }
-    return mergeSpans(matches);
-}
-function mergeSpans(matches) {
-    const priority = categories;
-    const merged = [];
-    for (const match of matches.sort((left, right) => left.start - right.start)) {
-        const previous = merged.at(-1);
-        if (previous === undefined || previous.end <= match.start) {
-            merged.push({ ...match });
-            continue;
-        }
-        previous.end = Math.max(previous.end, match.end);
-        if (priority.indexOf(match.category) < priority.indexOf(previous.category))
-            previous.category = match.category;
-    }
-    return merged;
-}
-function redactText(text, context) {
-    const { settings, counts } = context;
-    let result = "";
-    let cursor = 0;
-    for (const span of spans(text, settings)) {
-        result += text.slice(cursor, span.start) + replacements[span.category];
-        cursor = span.end;
-        counts[span.category]++;
-    }
-    return result + text.slice(cursor);
-}
-function rawJsonApi(value) {
-    return "isRawJSON" in value && typeof value.isRawJSON === "function";
-}
-function rawJson(value) {
-    return rawJsonApi(JSON) && JSON.isRawJSON(value);
-}
-function nested(value, context) {
-    if (typeof value === "string")
-        return redactText(value, context);
-    if (rawJson(value))
-        return value;
-    if (Array.isArray(value))
-        return value.map((item) => nested(item, context));
-    if (value === null || typeof value !== "object")
-        return value;
-    const result = {};
-    for (const [key, item] of Object.entries(value)) {
-        const redactedKey = redactText(key, context);
-        if (Object.hasOwn(result, redactedKey))
-            throw new Error("member-name-collision");
-        Object.defineProperty(result, redactedKey, {
-            value: nested(item, context),
-            enumerable: true,
-        });
-    }
-    return result;
-}
-export function redactRecord(record, settings) {
-    const counts = emptyCounts();
-    const context = { settings, counts };
-    const blocks = record.blocks.map((block) => {
-        if (block["type"] === "tool_use")
-            return { ...block, input: nested(block["input"], context) };
-        if (block["type"] === "tool_result")
-            return { ...block, content: nested(block["content"], context) };
-        if (block["type"] === "text" && typeof block["text"] === "string")
-            return { ...block, text: redactText(block["text"], context) };
-        return block;
-    });
-    const text = redactText(record.text, context);
-    return {
-        record: { ...record, text, blocks },
-        counts,
-        totalCount: Object.values(counts).reduce((sum, count) => sum + count, 0),
-    };
-}

package/dist/src/chats/revision-input.d.ts

@@ -1,3 +0,0 @@
-import { type EvidenceSource } from "./revision.ts";
-export declare function parseEvidenceCoverage(text: string, importId: string): readonly EvidenceSource[];
-export declare function nextEvidenceRevision(text: string | undefined, importId: string): number;

package/dist/src/chats/revision-input.js

@@ -1,180 +0,0 @@
-import { validateCoverage, withinRoot } from "./corpus-input.js";
-import { evidenceReason, revisionReasons, } from "./revision.js";
-function object(value) {
-    return value !== null && typeof value === "object" && !Array.isArray(value);
-}
-function exact(value, fields) {
-    const { required, optional = [] } = fields;
-    return (required.every((key) => Object.hasOwn(value, key)) &&
-        Object.keys(value).every((key) => required.includes(key) || optional.includes(key)));
-}
-function natural(value) {
-    return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
-}
-function absolute(value) {
-    return (typeof value === "string" &&
-        value.startsWith("/") &&
-        !value.includes("\0") &&
-        value.split("/").every((part) => part !== "." && part !== ".."));
-}
-function digest(value) {
-    return (value === null ||
-        (typeof value === "string" && /^sha256:[a-f\d]{64}$/.test(value)));
-}
-function evidenceValue(value) {
-    return (object(value) &&
-        digest(value["digest"]) &&
-        (!Object.hasOwn(value, "byteSize") ||
-            (natural(value["byteSize"]) && value["byteSize"] <= 10_485_760)));
-}
-function source(value, roots) {
-    if (!evidenceValue(value) ||
-        !absolute(value["path"]) ||
-        typeof value["status"] !== "string")
-        throw new Error("invalid-import");
-    return {
-        path: value["path"],
-        root: selectedRoot(value["selection"], roots),
-        status: value["status"],
-        digest: value.digest,
-        ...(value.byteSize === undefined ? {} : { byteSize: value.byteSize }),
-    };
-}
-export function parseEvidenceCoverage(text, importId) {
-    validateCoverage(text, importId);
-    const value = JSON.parse(text);
-    if (!object(value))
-        throw new Error("invalid-import");
-    const entries = value["schemaVersion"] === 1 ? value["files"] : value["entries"];
-    const roots = value["schemaVersion"] === 1 ? [] : value["roots"];
-    if (!Array.isArray(entries) ||
-        !Array.isArray(roots) ||
-        !roots.every((root) => typeof root === "string"))
-        throw new Error("invalid-import");
-    return entries.map((entry) => source(entry, roots));
-}
-function revisionValue(value) {
-    return (value === null ||
-        (evidenceValue(value) &&
-            exact(value, { required: ["digest"], optional: ["byteSize"] })));
-}
-function revisionEntry(value) {
-    if (!object(value) ||
-        !exact(value, {
-            required: ["path", "root", "before", "after", "status", "reason"],
-        }))
-        return false;
-    if (!selectionKey(value))
-        return false;
-    const reason = value["reason"];
-    if (typeof reason !== "string" || !Object.hasOwn(revisionReasons, reason))
-        return false;
-    const expected = Object.entries(revisionReasons).find(([key]) => key === reason)?.[1];
-    return (value["status"] === expected &&
-        revisionPresence(value) &&
-        consistentReason(value));
-}
-function revisionPresence(value) {
-    const reason = value["reason"];
-    if (reason === "no-previous-entry")
-        return value["before"] === null && value["after"] !== null;
-    if (reason === "not-selected")
-        return value["before"] !== null && value["after"] === null;
-    return value["before"] !== null && value["after"] !== null;
-}
-function validCounts(value, entries) {
-    const statuses = [
-        "new",
-        "missing",
-        "unchanged",
-        "changed",
-        "truncated",
-        "extended",
-    ];
-    return (object(value) &&
-        exact(value, { required: statuses }) &&
-        statuses.every((status) => natural(value[status]) &&
-            value[status] ===
-                entries.filter((entry) => entry["status"] === status).length));
-}
-export function nextEvidenceRevision(text, importId) {
-    if (text === undefined)
-        return 1;
-    const value = JSON.parse(text);
-    if (!object(value) ||
-        !exact(value, {
-            required: [
-                "schemaVersion",
-                "importId",
-                "previousImportId",
-                "evidenceRevision",
-                "counts",
-                "entries",
-            ],
-        }))
-        throw new Error("invalid-import");
-    if (!revisionHeader(value, importId))
-        throw new Error("invalid-import");
-    const entries = value["entries"];
-    if (!Array.isArray(entries) ||
-        !entries.every(revisionEntry) ||
-        !validCounts(value["counts"], entries))
-        throw new Error("invalid-import");
-    const revision = value["evidenceRevision"];
-    if (!natural(revision))
-        throw new Error("invalid-import");
-    return revision + 1;
-}
-function selectedRoot(selection, roots) {
-    if (roots.length === 0 || !object(selection) || selection["kind"] !== "root")
-        return null;
-    const index = selection["index"];
-    if (!natural(index) || roots[index] === undefined)
-        throw new Error("invalid-import");
-    return roots[index];
-}
-function selectionKey(value) {
-    return (absolute(value["path"]) &&
-        (value["root"] === null ||
-            (absolute(value["root"]) && withinRoot(value["root"], value["path"]))));
-}
-function revisionHeader(value, importId) {
-    const revision = value["evidenceRevision"];
-    if (value["schemaVersion"] !== 1 ||
-        value["importId"] !== importId ||
-        !natural(revision) ||
-        revision === 0 ||
-        revision === Number.MAX_SAFE_INTEGER)
-        return false;
-    const previous = value["previousImportId"];
-    if (typeof previous !== "string" ||
-        previous === importId ||
-        !/^[a-f\d]{8}-[a-f\d]{4}-4[a-f\d]{3}-[89ab][a-f\d]{3}-[a-f\d]{12}$/.test(previous))
-        return false;
-    return true;
-}
-function consistentReason(value) {
-    const before = value["before"];
-    const after = value["after"];
-    if (!revisionValue(before) || !revisionValue(after))
-        return false;
-    const reason = value["reason"];
-    const unverified = [
-        "no-previous-entry",
-        "not-selected",
-        "not-included",
-    ].includes(String(reason));
-    if (!unverified && after?.digest === null)
-        return false;
-    const prefix = reason === "verified-prefix" ? before?.digest : undefined;
-    return (evidenceReason({
-        before: comparable(before, { status: "included" }),
-        after: comparable(after, {
-            status: reason === "not-included" ? "unsupported" : "included",
-            prefixDigest: prefix ?? undefined,
-        }),
-    }) === reason);
-}
-function comparable(value, state) {
-    return value === null ? null : { ...value, ...state };
-}

package/dist/src/chats/revision.d.ts

@@ -1,48 +0,0 @@
-export interface SelectionKey {
-    readonly path: string;
-    readonly root: string | null;
-}
-export interface EvidenceValue {
-    readonly digest: string | null;
-    readonly byteSize?: number | undefined;
-}
-export interface ComparableEvidence extends EvidenceValue {
-    readonly status: string;
-    readonly prefixDigest?: string | undefined;
-}
-export interface EvidenceSource extends SelectionKey, ComparableEvidence {
-}
-export declare const revisionReasons: {
-    readonly "no-previous-entry": "new";
-    readonly "not-selected": "missing";
-    readonly "not-included": "missing";
-    readonly "same-digest": "unchanged";
-    readonly "previous-digest-unknown": "changed";
-    readonly "previous-size-unknown": "changed";
-    readonly "shorter-source": "truncated";
-    readonly "verified-prefix": "extended";
-    readonly "digest-changed": "changed";
-};
-export type RevisionReason = keyof typeof revisionReasons;
-export type RevisionStatus = (typeof revisionReasons)[RevisionReason];
-export interface RevisionEntry extends SelectionKey {
-    readonly before: EvidenceValue | null;
-    readonly after: EvidenceValue | null;
-    readonly status: RevisionStatus;
-    readonly reason: RevisionReason;
-}
-export declare function pairEvidence(data: {
-    readonly previous: readonly SelectionKey[];
-    readonly current: readonly SelectionKey[];
-}): readonly (number | null)[];
-export declare function evidenceReason(data: {
-    readonly before: ComparableEvidence | null;
-    readonly after: ComparableEvidence | null;
-}): RevisionReason;
-export declare function compareEvidence(data: {
-    readonly previous: readonly EvidenceSource[];
-    readonly current: readonly EvidenceSource[];
-}): {
-    counts: Record<RevisionStatus, number>;
-    entries: RevisionEntry[];
-};

package/dist/src/chats/revision.js

@@ -1,94 +0,0 @@
-export const revisionReasons = {
-    "no-previous-entry": "new",
-    "not-selected": "missing",
-    "not-included": "missing",
-    "same-digest": "unchanged",
-    "previous-digest-unknown": "changed",
-    "previous-size-unknown": "changed",
-    "shorter-source": "truncated",
-    "verified-prefix": "extended",
-    "digest-changed": "changed",
-};
-export function pairEvidence(data) {
-    const queues = new Map();
-    const key = (entry) => JSON.stringify([entry.path, entry.root]);
-    data.previous.forEach((entry, index) => {
-        const encoded = key(entry);
-        const queue = queues.get(encoded) ?? [];
-        queue.push(index);
-        queues.set(encoded, queue);
-    });
-    return data.current.map((entry) => queues.get(key(entry))?.shift() ?? null);
-}
-export function evidenceReason(data) {
-    const { before, after } = data;
-    if (before === null)
-        return "no-previous-entry";
-    if (after === null)
-        return "not-selected";
-    if (after.status !== "included")
-        return "not-included";
-    if (before.digest !== null && before.digest === after.digest)
-        return "same-digest";
-    if (before.digest === null)
-        return "previous-digest-unknown";
-    if (before.byteSize === undefined)
-        return "previous-size-unknown";
-    return sizeReason({ ...before, byteSize: before.byteSize }, after);
-}
-function sizeReason(before, after) {
-    if (after.byteSize !== undefined && after.byteSize < before.byteSize)
-        return "shorter-source";
-    if (after.byteSize !== undefined &&
-        after.byteSize > before.byteSize &&
-        after.prefixDigest === before.digest)
-        return "verified-prefix";
-    return "digest-changed";
-}
-function value(source) {
-    if (source === undefined)
-        return null;
-    return {
-        digest: source.digest,
-        ...(source.byteSize === undefined ? {} : { byteSize: source.byteSize }),
-    };
-}
-function entry(before, after) {
-    const key = after ?? before;
-    if (key === undefined)
-        throw new Error("invalid-evidence");
-    const why = evidenceReason({ before: before ?? null, after: after ?? null });
-    return {
-        path: key.path,
-        root: key.root,
-        before: value(before),
-        after: value(after),
-        status: revisionReasons[why],
-        reason: why,
-    };
-}
-export function compareEvidence(data) {
-    const pairs = pairEvidence(data);
-    const used = new Set(pairs);
-    const entries = data.current.map((after, index) => {
-        const matched = pairs[index];
-        return entry(matched === null || matched === undefined
-            ? undefined
-            : data.previous[matched], after);
-    });
-    data.previous.forEach((before, index) => {
-        if (!used.has(index))
-            entries.push(entry(before, undefined));
-    });
-    const counts = {
-        new: 0,
-        missing: 0,
-        unchanged: 0,
-        changed: 0,
-        truncated: 0,
-        extended: 0,
-    };
-    for (const item of entries)
-        counts[item.status]++;
-    return { counts, entries };
-}