cloving 1.0.1 → 1.1.0

package/dist/src/chats/redaction.d.ts

@@ -0,0 +1,41 @@
+import type { MessageRecord } from "./normalize.ts";
+export interface RedactionSettings {
+    readonly home: string | null;
+    readonly privateTexts: readonly string[];
+}
+export declare const replacements: {
+    readonly privateKey: "[PRIVATE_KEY]";
+    readonly credentialUrl: "[CREDENTIAL_URL]";
+    readonly homePath: "[HOME_PATH]";
+    readonly token: "[TOKEN]";
+    readonly email: "[EMAIL]";
+    readonly privateText: "[PRIVATE_TEXT]";
+};
+export declare const categories: readonly ["privateKey", "credentialUrl", "homePath", "token", "email", "privateText"];
+export type Category = keyof typeof replacements;
+export type RedactionCounts = Record<Category, number>;
+export declare function emptyCounts(): RedactionCounts;
+export declare function redactRecord(record: MessageRecord, settings: RedactionSettings): {
+    record: {
+        text: string;
+        blocks: Record<string, unknown>[];
+        sourceId: string;
+        role: "user" | "assistant";
+        timestamp: string | null;
+        sessionId: string | null;
+        schemaVersion: 1;
+    } | {
+        text: string;
+        blocks: Record<string, unknown>[];
+        sourceId: string;
+        role: "user" | "assistant";
+        timestamp: string | null;
+        sessionId: string | null;
+        schemaVersion: 2;
+        uuid: string | null;
+        isSidechain: boolean | null;
+        parentUuid?: string | null;
+    };
+    counts: RedactionCounts;
+    totalCount: number;
+};

package/dist/src/chats/redaction.js

@@ -0,0 +1,161 @@
+export const replacements = {
+    privateKey: "[PRIVATE_KEY]",
+    credentialUrl: "[CREDENTIAL_URL]",
+    homePath: "[HOME_PATH]",
+    token: "[TOKEN]",
+    email: "[EMAIL]",
+    privateText: "[PRIVATE_TEXT]",
+};
+export const categories = [
+    "privateKey",
+    "credentialUrl",
+    "homePath",
+    "token",
+    "email",
+    "privateText",
+];
+export function emptyCounts() {
+    return {
+        privateKey: 0,
+        credentialUrl: 0,
+        homePath: 0,
+        token: 0,
+        email: 0,
+        privateText: 0,
+    };
+}
+function escaped(text) {
+    return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function patternSpans(text, rule) {
+    return [...text.matchAll(rule.pattern)].map((match) => ({
+        start: match.index,
+        end: match.index + match[0].length,
+        category: rule.category,
+    }));
+}
+function homeRoot(home) {
+    if (home === null || home === "")
+        return "~";
+    if (home === "/")
+        return "(?:(?=/)|~)";
+    return `(?:${escaped(home.replace(/\/$/, ""))}|~)`;
+}
+function homeSpans(text, home) {
+    const root = homeRoot(home);
+    const pattern = new RegExp(String.raw `(?<![\p{L}\p{N}_./~\\-])${root}(?=$|[\s/\x22'\x60<>,;:!?()\[\]{}])(?:/[^\s\x22'\x60<>]*)?`, "gu");
+    return patternSpans(text, { pattern, category: "homePath" }).map((span) => {
+        const quote = text[span.start - 1];
+        if (quote !== '"' && quote !== "'" && quote !== "`")
+            return span;
+        const end = text.indexOf(quote, span.end);
+        return end === -1 ? span : { ...span, end };
+    });
+}
+function spans(text, settings) {
+    const rules = [
+        {
+            pattern: /-----BEGIN ((?:[A-Z0-9]+ )*PRIVATE KEY)-----[\s\S]*?-----END \1-----/g,
+            category: "privateKey",
+        },
+        {
+            pattern: /\b[a-z][a-z\d+.-]*:\/\/[^\s<>"'`/?#]*@[^\s<>"'`]+/gi,
+            category: "credentialUrl",
+        },
+        {
+            pattern: /(?<![\p{L}\p{N}_])(?:gh[pousr]_|github_pat_|sk-ant-|sk-(?:proj-|svcacct-)?)[a-z\d_-]+/giu,
+            category: "token",
+        },
+        {
+            pattern: /[\w.!#$%&'*+/=?^`{|}~-]+@[\w.-]+\.[a-z]{2,}/gi,
+            category: "email",
+        },
+    ];
+    const matches = rules.flatMap((rule) => patternSpans(text, rule));
+    for (const match of homeSpans(text, settings.home))
+        matches.push(match);
+    for (const privateText of settings.privateTexts) {
+        if (privateText.trim() === "")
+            throw new Error("invalid-settings");
+        const pattern = new RegExp(`(?=(${escaped(privateText)}))`, "gi");
+        for (const match of text.matchAll(pattern))
+            matches.push({
+                start: match.index,
+                end: match.index + String(match[1]).length,
+                category: "privateText",
+            });
+    }
+    return mergeSpans(matches);
+}
+function mergeSpans(matches) {
+    const priority = categories;
+    const merged = [];
+    for (const match of matches.sort((left, right) => left.start - right.start)) {
+        const previous = merged.at(-1);
+        if (previous === undefined || previous.end <= match.start) {
+            merged.push({ ...match });
+            continue;
+        }
+        previous.end = Math.max(previous.end, match.end);
+        if (priority.indexOf(match.category) < priority.indexOf(previous.category))
+            previous.category = match.category;
+    }
+    return merged;
+}
+function redactText(text, context) {
+    const { settings, counts } = context;
+    let result = "";
+    let cursor = 0;
+    for (const span of spans(text, settings)) {
+        result += text.slice(cursor, span.start) + replacements[span.category];
+        cursor = span.end;
+        counts[span.category]++;
+    }
+    return result + text.slice(cursor);
+}
+function rawJsonApi(value) {
+    return "isRawJSON" in value && typeof value.isRawJSON === "function";
+}
+function rawJson(value) {
+    return rawJsonApi(JSON) && JSON.isRawJSON(value);
+}
+function nested(value, context) {
+    if (typeof value === "string")
+        return redactText(value, context);
+    if (rawJson(value))
+        return value;
+    if (Array.isArray(value))
+        return value.map((item) => nested(item, context));
+    if (value === null || typeof value !== "object")
+        return value;
+    const result = {};
+    for (const [key, item] of Object.entries(value)) {
+        const redactedKey = redactText(key, context);
+        if (Object.hasOwn(result, redactedKey))
+            throw new Error("member-name-collision");
+        Object.defineProperty(result, redactedKey, {
+            value: nested(item, context),
+            enumerable: true,
+        });
+    }
+    return result;
+}
+export function redactRecord(record, settings) {
+    const counts = emptyCounts();
+    const context = { settings, counts };
+    const blocks = record.blocks.map((block) => {
+        if (block["type"] === "tool_use")
+            return { ...block, input: nested(block["input"], context) };
+        if (block["type"] === "tool_result")
+            return { ...block, content: nested(block["content"], context) };
+        if (block["type"] === "text" && typeof block["text"] === "string")
+            return { ...block, text: redactText(block["text"], context) };
+        return block;
+    });
+    const text = redactText(record.text, context);
+    return {
+        record: { ...record, text, blocks },
+        counts,
+        totalCount: Object.values(counts).reduce((sum, count) => sum + count, 0),
+    };
+}

package/dist/src/chats/revision-input.d.ts

@@ -0,0 +1,3 @@
+import { type EvidenceSource } from "./revision.ts";
+export declare function parseEvidenceCoverage(text: string, importId: string): readonly EvidenceSource[];
+export declare function nextEvidenceRevision(text: string | undefined, importId: string): number;

package/dist/src/chats/revision-input.js

@@ -0,0 +1,180 @@
+import { validateCoverage, withinRoot } from "./corpus-input.js";
+import { evidenceReason, revisionReasons, } from "./revision.js";
+function object(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function exact(value, fields) {
+    const { required, optional = [] } = fields;
+    return (required.every((key) => Object.hasOwn(value, key)) &&
+        Object.keys(value).every((key) => required.includes(key) || optional.includes(key)));
+}
+function natural(value) {
+    return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+}
+function absolute(value) {
+    return (typeof value === "string" &&
+        value.startsWith("/") &&
+        !value.includes("\0") &&
+        value.split("/").every((part) => part !== "." && part !== ".."));
+}
+function digest(value) {
+    return (value === null ||
+        (typeof value === "string" && /^sha256:[a-f\d]{64}$/.test(value)));
+}
+function evidenceValue(value) {
+    return (object(value) &&
+        digest(value["digest"]) &&
+        (!Object.hasOwn(value, "byteSize") ||
+            (natural(value["byteSize"]) && value["byteSize"] <= 10_485_760)));
+}
+function source(value, roots) {
+    if (!evidenceValue(value) ||
+        !absolute(value["path"]) ||
+        typeof value["status"] !== "string")
+        throw new Error("invalid-import");
+    return {
+        path: value["path"],
+        root: selectedRoot(value["selection"], roots),
+        status: value["status"],
+        digest: value.digest,
+        ...(value.byteSize === undefined ? {} : { byteSize: value.byteSize }),
+    };
+}
+export function parseEvidenceCoverage(text, importId) {
+    validateCoverage(text, importId);
+    const value = JSON.parse(text);
+    if (!object(value))
+        throw new Error("invalid-import");
+    const entries = value["schemaVersion"] === 1 ? value["files"] : value["entries"];
+    const roots = value["schemaVersion"] === 1 ? [] : value["roots"];
+    if (!Array.isArray(entries) ||
+        !Array.isArray(roots) ||
+        !roots.every((root) => typeof root === "string"))
+        throw new Error("invalid-import");
+    return entries.map((entry) => source(entry, roots));
+}
+function revisionValue(value) {
+    return (value === null ||
+        (evidenceValue(value) &&
+            exact(value, { required: ["digest"], optional: ["byteSize"] })));
+}
+function revisionEntry(value) {
+    if (!object(value) ||
+        !exact(value, {
+            required: ["path", "root", "before", "after", "status", "reason"],
+        }))
+        return false;
+    if (!selectionKey(value))
+        return false;
+    const reason = value["reason"];
+    if (typeof reason !== "string" || !Object.hasOwn(revisionReasons, reason))
+        return false;
+    const expected = Object.entries(revisionReasons).find(([key]) => key === reason)?.[1];
+    return (value["status"] === expected &&
+        revisionPresence(value) &&
+        consistentReason(value));
+}
+function revisionPresence(value) {
+    const reason = value["reason"];
+    if (reason === "no-previous-entry")
+        return value["before"] === null && value["after"] !== null;
+    if (reason === "not-selected")
+        return value["before"] !== null && value["after"] === null;
+    return value["before"] !== null && value["after"] !== null;
+}
+function validCounts(value, entries) {
+    const statuses = [
+        "new",
+        "missing",
+        "unchanged",
+        "changed",
+        "truncated",
+        "extended",
+    ];
+    return (object(value) &&
+        exact(value, { required: statuses }) &&
+        statuses.every((status) => natural(value[status]) &&
+            value[status] ===
+                entries.filter((entry) => entry["status"] === status).length));
+}
+export function nextEvidenceRevision(text, importId) {
+    if (text === undefined)
+        return 1;
+    const value = JSON.parse(text);
+    if (!object(value) ||
+        !exact(value, {
+            required: [
+                "schemaVersion",
+                "importId",
+                "previousImportId",
+                "evidenceRevision",
+                "counts",
+                "entries",
+            ],
+        }))
+        throw new Error("invalid-import");
+    if (!revisionHeader(value, importId))
+        throw new Error("invalid-import");
+    const entries = value["entries"];
+    if (!Array.isArray(entries) ||
+        !entries.every(revisionEntry) ||
+        !validCounts(value["counts"], entries))
+        throw new Error("invalid-import");
+    const revision = value["evidenceRevision"];
+    if (!natural(revision))
+        throw new Error("invalid-import");
+    return revision + 1;
+}
+function selectedRoot(selection, roots) {
+    if (roots.length === 0 || !object(selection) || selection["kind"] !== "root")
+        return null;
+    const index = selection["index"];
+    if (!natural(index) || roots[index] === undefined)
+        throw new Error("invalid-import");
+    return roots[index];
+}
+function selectionKey(value) {
+    return (absolute(value["path"]) &&
+        (value["root"] === null ||
+            (absolute(value["root"]) && withinRoot(value["root"], value["path"]))));
+}
+function revisionHeader(value, importId) {
+    const revision = value["evidenceRevision"];
+    if (value["schemaVersion"] !== 1 ||
+        value["importId"] !== importId ||
+        !natural(revision) ||
+        revision === 0 ||
+        revision === Number.MAX_SAFE_INTEGER)
+        return false;
+    const previous = value["previousImportId"];
+    if (typeof previous !== "string" ||
+        previous === importId ||
+        !/^[a-f\d]{8}-[a-f\d]{4}-4[a-f\d]{3}-[89ab][a-f\d]{3}-[a-f\d]{12}$/.test(previous))
+        return false;
+    return true;
+}
+function consistentReason(value) {
+    const before = value["before"];
+    const after = value["after"];
+    if (!revisionValue(before) || !revisionValue(after))
+        return false;
+    const reason = value["reason"];
+    const unverified = [
+        "no-previous-entry",
+        "not-selected",
+        "not-included",
+    ].includes(String(reason));
+    if (!unverified && after?.digest === null)
+        return false;
+    const prefix = reason === "verified-prefix" ? before?.digest : undefined;
+    return (evidenceReason({
+        before: comparable(before, { status: "included" }),
+        after: comparable(after, {
+            status: reason === "not-included" ? "unsupported" : "included",
+            prefixDigest: prefix ?? undefined,
+        }),
+    }) === reason);
+}
+function comparable(value, state) {
+    return value === null ? null : { ...value, ...state };
+}

package/dist/src/chats/revision.d.ts

@@ -0,0 +1,48 @@
+export interface SelectionKey {
+    readonly path: string;
+    readonly root: string | null;
+}
+export interface EvidenceValue {
+    readonly digest: string | null;
+    readonly byteSize?: number | undefined;
+}
+export interface ComparableEvidence extends EvidenceValue {
+    readonly status: string;
+    readonly prefixDigest?: string | undefined;
+}
+export interface EvidenceSource extends SelectionKey, ComparableEvidence {
+}
+export declare const revisionReasons: {
+    readonly "no-previous-entry": "new";
+    readonly "not-selected": "missing";
+    readonly "not-included": "missing";
+    readonly "same-digest": "unchanged";
+    readonly "previous-digest-unknown": "changed";
+    readonly "previous-size-unknown": "changed";
+    readonly "shorter-source": "truncated";
+    readonly "verified-prefix": "extended";
+    readonly "digest-changed": "changed";
+};
+export type RevisionReason = keyof typeof revisionReasons;
+export type RevisionStatus = (typeof revisionReasons)[RevisionReason];
+export interface RevisionEntry extends SelectionKey {
+    readonly before: EvidenceValue | null;
+    readonly after: EvidenceValue | null;
+    readonly status: RevisionStatus;
+    readonly reason: RevisionReason;
+}
+export declare function pairEvidence(data: {
+    readonly previous: readonly SelectionKey[];
+    readonly current: readonly SelectionKey[];
+}): readonly (number | null)[];
+export declare function evidenceReason(data: {
+    readonly before: ComparableEvidence | null;
+    readonly after: ComparableEvidence | null;
+}): RevisionReason;
+export declare function compareEvidence(data: {
+    readonly previous: readonly EvidenceSource[];
+    readonly current: readonly EvidenceSource[];
+}): {
+    counts: Record<RevisionStatus, number>;
+    entries: RevisionEntry[];
+};

package/dist/src/chats/revision.js

@@ -0,0 +1,94 @@
+export const revisionReasons = {
+    "no-previous-entry": "new",
+    "not-selected": "missing",
+    "not-included": "missing",
+    "same-digest": "unchanged",
+    "previous-digest-unknown": "changed",
+    "previous-size-unknown": "changed",
+    "shorter-source": "truncated",
+    "verified-prefix": "extended",
+    "digest-changed": "changed",
+};
+export function pairEvidence(data) {
+    const queues = new Map();
+    const key = (entry) => JSON.stringify([entry.path, entry.root]);
+    data.previous.forEach((entry, index) => {
+        const encoded = key(entry);
+        const queue = queues.get(encoded) ?? [];
+        queue.push(index);
+        queues.set(encoded, queue);
+    });
+    return data.current.map((entry) => queues.get(key(entry))?.shift() ?? null);
+}
+export function evidenceReason(data) {
+    const { before, after } = data;
+    if (before === null)
+        return "no-previous-entry";
+    if (after === null)
+        return "not-selected";
+    if (after.status !== "included")
+        return "not-included";
+    if (before.digest !== null && before.digest === after.digest)
+        return "same-digest";
+    if (before.digest === null)
+        return "previous-digest-unknown";
+    if (before.byteSize === undefined)
+        return "previous-size-unknown";
+    return sizeReason({ ...before, byteSize: before.byteSize }, after);
+}
+function sizeReason(before, after) {
+    if (after.byteSize !== undefined && after.byteSize < before.byteSize)
+        return "shorter-source";
+    if (after.byteSize !== undefined &&
+        after.byteSize > before.byteSize &&
+        after.prefixDigest === before.digest)
+        return "verified-prefix";
+    return "digest-changed";
+}
+function value(source) {
+    if (source === undefined)
+        return null;
+    return {
+        digest: source.digest,
+        ...(source.byteSize === undefined ? {} : { byteSize: source.byteSize }),
+    };
+}
+function entry(before, after) {
+    const key = after ?? before;
+    if (key === undefined)
+        throw new Error("invalid-evidence");
+    const why = evidenceReason({ before: before ?? null, after: after ?? null });
+    return {
+        path: key.path,
+        root: key.root,
+        before: value(before),
+        after: value(after),
+        status: revisionReasons[why],
+        reason: why,
+    };
+}
+export function compareEvidence(data) {
+    const pairs = pairEvidence(data);
+    const used = new Set(pairs);
+    const entries = data.current.map((after, index) => {
+        const matched = pairs[index];
+        return entry(matched === null || matched === undefined
+            ? undefined
+            : data.previous[matched], after);
+    });
+    data.previous.forEach((before, index) => {
+        if (!used.has(index))
+            entries.push(entry(before, undefined));
+    });
+    const counts = {
+        new: 0,
+        missing: 0,
+        unchanged: 0,
+        changed: 0,
+        truncated: 0,
+        extended: 0,
+    };
+    for (const item of entries)
+        counts[item.status]++;
+    return { counts, entries };
+}

package/dist/src/chats/selection.d.ts

@@ -0,0 +1,25 @@
+import { type ScanSource } from "./source.ts";
+export interface SelectionRequest {
+    readonly paths: readonly string[];
+    readonly roots?: readonly string[];
+    readonly excludeProjects?: readonly string[];
+    readonly workingDirectory: string;
+}
+export interface SelectionOutcome {
+    readonly status: "unreadable" | "unsupported" | "excluded" | "pending";
+    readonly reason: string;
+}
+export interface SourceSelection {
+    readonly kind: "file" | "subtree";
+    readonly path: string;
+    readonly selection: {
+        readonly kind: "file" | "root";
+        readonly index: number;
+    };
+    readonly outcome?: SelectionOutcome;
+    readonly inspection?: ScanSource;
+}
+export declare function selectChatSources(request: SelectionRequest): Promise<{
+    roots: string[];
+    entries: SourceSelection[];
+}>;

package/dist/src/chats/selection.js

@@ -0,0 +1,109 @@
+import { readdir } from "node:fs/promises";
+import { basename, join, relative, resolve } from "node:path";
+import { inspectScanSource, noFollowPath, refuseRepositoryRoot, sourceFailure, verifyScanSource, verifyScanAncestor, } from "./source.js";
+function outcome(reason) {
+    return {
+        status: reason === "source-unreadable" ? "unreadable" : "unsupported",
+        reason,
+    };
+}
+async function scanEntry(entry, context) {
+    try {
+        const { exclusions, depth } = context;
+        const root = entry.inspection?.root ?? entry.path;
+        const inspection = await inspectScanSource(entry.path, root);
+        if (entry.inspection !== undefined)
+            verifyScanAncestor(entry.inspection, inspection);
+        const inspected = { ...entry, kind: inspection.kind, inspection };
+        const disposition = entryDisposition(inspected, context);
+        if (disposition !== undefined)
+            return [{ ...inspected, outcome: disposition }];
+        if (inspection.kind === "file")
+            return [inspected];
+        const entries = [];
+        const children = await readdir(noFollowPath(entry.path), {
+            withFileTypes: true,
+        });
+        await verifyScanSource(inspection);
+        for (const child of children) {
+            const scanned = await scanEntry({
+                ...inspected,
+                kind: child.isDirectory() ? "subtree" : "file",
+                path: join(entry.path, child.name),
+            }, { exclusions, depth: depth + 1 });
+            for (const selected of scanned)
+                entries.push(selected);
+        }
+        return entries;
+    }
+    catch (error) {
+        return [{ ...entry, outcome: outcome(sourceFailure(error)) }];
+    }
+}
+function entryDisposition(entry, context) {
+    if (entry.kind === "file")
+        return entry.path.endsWith(".jsonl")
+            ? undefined
+            : outcome("unsupported-extension");
+    if (context.depth === 1 && context.exclusions.includes(basename(entry.path)))
+        return { status: "excluded", reason: "project-excluded" };
+    if (context.depth === 4)
+        return { status: "pending", reason: "depth-limit" };
+    return;
+}
+async function prepareRoot(path) {
+    const entry = {
+        path,
+        kind: "subtree",
+        selection: { kind: "root", index: 0 },
+    };
+    try {
+        const inspection = await inspectScanSource(path, path);
+        if (inspection.kind !== "subtree")
+            throw new Error("non-directory-root");
+        await refuseRepositoryRoot(inspection.root);
+        return { ...entry, path: inspection.root, inspection };
+    }
+    catch (error) {
+        return { ...entry, outcome: outcome(sourceFailure(error)) };
+    }
+}
+export async function selectChatSources(request) {
+    const entries = request.paths.map((path, index) => ({
+        path: resolve(request.workingDirectory, path),
+        kind: "file",
+        selection: { kind: "file", index },
+    }));
+    const selectedRoots = [];
+    for (const value of request.roots ?? [])
+        selectedRoots.push(await prepareRoot(resolve(request.workingDirectory, value)));
+    selectedRoots.sort((left, right) => stringOrder(left.path, right.path));
+    const roots = selectedRoots.map((root) => root.path);
+    for (const [index, root] of selectedRoots.entries()) {
+        const selected = { ...root, selection: { kind: "root", index } };
+        const scanned = root.outcome === undefined
+            ? await scanEntry(selected, {
+                exclusions: request.excludeProjects ?? [],
+                depth: 0,
+            })
+            : [selected];
+        scanned.sort((left, right) => stringOrder(relative(root.path, left.path), relative(root.path, right.path)));
+        for (const entry of scanned)
+            entries.push(entry);
+    }
+    let attempts = 0;
+    return {
+        roots,
+        entries: entries.map((entry) => {
+            if (entry.selection.kind === "file" || entry.outcome !== undefined)
+                return entry;
+            attempts++;
+            return attempts <= 100
+                ? entry
+                : { ...entry, outcome: { status: "pending", reason: "file-limit" } };
+        }),
+    };
+}
+function stringOrder(left, right) {
+    return left < right ? -1 : Number(left > right);
+}

package/dist/src/chats/source.d.ts

@@ -0,0 +1,23 @@
+import type { Stats } from "node:fs";
+export interface SourceIdentity {
+    readonly path: string;
+    readonly device: number;
+    readonly inode: number;
+}
+export interface ScanSource {
+    readonly path: string;
+    readonly root: string;
+    readonly identities: readonly SourceIdentity[];
+    readonly size: number;
+    readonly modified: number;
+    readonly changed: number;
+    readonly kind: "file" | "subtree";
+}
+export declare function contained(root: string, path: string): boolean;
+export declare function inspectScanSource(path: string, root: string): Promise<ScanSource>;
+export declare function noFollowPath(path: string): string;
+export declare function refuseRepositoryRoot(path: string): Promise<void>;
+export declare function sourceFailure(error: unknown): string;
+export declare function verifyScanSource(expected: ScanSource): Promise<void>;
+export declare function verifyOpenSource(expected: ScanSource, stats: Stats): void;
+export declare function verifyScanAncestor(expected: ScanSource, current: ScanSource): void;