cloving 1.0.1 → 1.2.0

package/dist/src/chats/redaction.d.ts

@@ -0,0 +1,41 @@
+import type { MessageRecord } from "./normalize.ts";
+export interface RedactionSettings {
+    readonly home: string | null;
+    readonly privateTexts: readonly string[];
+}
+export declare const replacements: {
+    readonly privateKey: "[PRIVATE_KEY]";
+    readonly credentialUrl: "[CREDENTIAL_URL]";
+    readonly homePath: "[HOME_PATH]";
+    readonly token: "[TOKEN]";
+    readonly email: "[EMAIL]";
+    readonly privateText: "[PRIVATE_TEXT]";
+};
+export declare const categories: readonly ["privateKey", "credentialUrl", "homePath", "token", "email", "privateText"];
+export type Category = keyof typeof replacements;
+export type RedactionCounts = Record<Category, number>;
+export declare function emptyCounts(): RedactionCounts;
+export declare function redactRecord(record: MessageRecord, settings: RedactionSettings): {
+    record: {
+        text: string;
+        blocks: Record<string, unknown>[];
+        sourceId: string;
+        role: "user" | "assistant";
+        timestamp: string | null;
+        sessionId: string | null;
+        schemaVersion: 1;
+    } | {
+        text: string;
+        blocks: Record<string, unknown>[];
+        sourceId: string;
+        role: "user" | "assistant";
+        timestamp: string | null;
+        sessionId: string | null;
+        schemaVersion: 2;
+        uuid: string | null;
+        isSidechain: boolean | null;
+        parentUuid?: string | null;
+    };
+    counts: RedactionCounts;
+    totalCount: number;
+};

package/dist/src/chats/redaction.js

@@ -0,0 +1,161 @@
+export const replacements = {
+    privateKey: "[PRIVATE_KEY]",
+    credentialUrl: "[CREDENTIAL_URL]",
+    homePath: "[HOME_PATH]",
+    token: "[TOKEN]",
+    email: "[EMAIL]",
+    privateText: "[PRIVATE_TEXT]",
+};
+export const categories = [
+    "privateKey",
+    "credentialUrl",
+    "homePath",
+    "token",
+    "email",
+    "privateText",
+];
+export function emptyCounts() {
+    return {
+        privateKey: 0,
+        credentialUrl: 0,
+        homePath: 0,
+        token: 0,
+        email: 0,
+        privateText: 0,
+    };
+}
+function escaped(text) {
+    return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function patternSpans(text, rule) {
+    return [...text.matchAll(rule.pattern)].map((match) => ({
+        start: match.index,
+        end: match.index + match[0].length,
+        category: rule.category,
+    }));
+}
+function homeRoot(home) {
+    if (home === null || home === "")
+        return "~";
+    if (home === "/")
+        return "(?:(?=/)|~)";
+    return `(?:${escaped(home.replace(/\/$/, ""))}|~)`;
+}
+function homeSpans(text, home) {
+    const root = homeRoot(home);
+    const pattern = new RegExp(String.raw `(?<![\p{L}\p{N}_./~\\-])${root}(?=$|[\s/\x22'\x60<>,;:!?()\[\]{}])(?:/[^\s\x22'\x60<>]*)?`, "gu");
+    return patternSpans(text, { pattern, category: "homePath" }).map((span) => {
+        const quote = text[span.start - 1];
+        if (quote !== '"' && quote !== "'" && quote !== "`")
+            return span;
+        const end = text.indexOf(quote, span.end);
+        return end === -1 ? span : { ...span, end };
+    });
+}
+function spans(text, settings) {
+    const rules = [
+        {
+            pattern: /-----BEGIN ((?:[A-Z0-9]+ )*PRIVATE KEY)-----[\s\S]*?-----END \1-----/g,
+            category: "privateKey",
+        },
+        {
+            pattern: /\b[a-z][a-z\d+.-]*:\/\/[^\s<>"'`/?#]*@[^\s<>"'`]+/gi,
+            category: "credentialUrl",
+        },
+        {
+            pattern: /(?<![\p{L}\p{N}_])(?:gh[pousr]_|github_pat_|sk-ant-|sk-(?:proj-|svcacct-)?)[a-z\d_-]+/giu,
+            category: "token",
+        },
+        {
+            pattern: /[\w.!#$%&'*+/=?^`{|}~-]+@[\w.-]+\.[a-z]{2,}/gi,
+            category: "email",
+        },
+    ];
+    const matches = rules.flatMap((rule) => patternSpans(text, rule));
+    for (const match of homeSpans(text, settings.home))
+        matches.push(match);
+    for (const privateText of settings.privateTexts) {
+        if (privateText.trim() === "")
+            throw new Error("invalid-settings");
+        const pattern = new RegExp(`(?=(${escaped(privateText)}))`, "gi");
+        for (const match of text.matchAll(pattern))
+            matches.push({
+                start: match.index,
+                end: match.index + String(match[1]).length,
+                category: "privateText",
+            });
+    }
+    return mergeSpans(matches);
+}
+function mergeSpans(matches) {
+    const priority = categories;
+    const merged = [];
+    for (const match of matches.sort((left, right) => left.start - right.start)) {
+        const previous = merged.at(-1);
+        if (previous === undefined || previous.end <= match.start) {
+            merged.push({ ...match });
+            continue;
+        }
+        previous.end = Math.max(previous.end, match.end);
+        if (priority.indexOf(match.category) < priority.indexOf(previous.category))
+            previous.category = match.category;
+    }
+    return merged;
+}
+function redactText(text, context) {
+    const { settings, counts } = context;
+    let result = "";
+    let cursor = 0;
+    for (const span of spans(text, settings)) {
+        result += text.slice(cursor, span.start) + replacements[span.category];
+        cursor = span.end;
+        counts[span.category]++;
+    }
+    return result + text.slice(cursor);
+}
+function rawJsonApi(value) {
+    return "isRawJSON" in value && typeof value.isRawJSON === "function";
+}
+function rawJson(value) {
+    return rawJsonApi(JSON) && JSON.isRawJSON(value);
+}
+function nested(value, context) {
+    if (typeof value === "string")
+        return redactText(value, context);
+    if (rawJson(value))
+        return value;
+    if (Array.isArray(value))
+        return value.map((item) => nested(item, context));
+    if (value === null || typeof value !== "object")
+        return value;
+    const result = {};
+    for (const [key, item] of Object.entries(value)) {
+        const redactedKey = redactText(key, context);
+        if (Object.hasOwn(result, redactedKey))
+            throw new Error("member-name-collision");
+        Object.defineProperty(result, redactedKey, {
+            value: nested(item, context),
+            enumerable: true,
+        });
+    }
+    return result;
+}
+export function redactRecord(record, settings) {
+    const counts = emptyCounts();
+    const context = { settings, counts };
+    const blocks = record.blocks.map((block) => {
+        if (block["type"] === "tool_use")
+            return { ...block, input: nested(block["input"], context) };
+        if (block["type"] === "tool_result")
+            return { ...block, content: nested(block["content"], context) };
+        if (block["type"] === "text" && typeof block["text"] === "string")
+            return { ...block, text: redactText(block["text"], context) };
+        return block;
+    });
+    const text = redactText(record.text, context);
+    return {
+        record: { ...record, text, blocks },
+        counts,
+        totalCount: Object.values(counts).reduce((sum, count) => sum + count, 0),
+    };
+}

package/dist/src/chats/reference-mapping.d.ts

@@ -0,0 +1,16 @@
+import { type RevisionEntry } from "./revision.ts";
+import type { OccurrenceReference } from "./structure.ts";
+export interface EvidenceMappingSnapshot {
+    readonly importId: string;
+    readonly coverage: string;
+    readonly corpus: string;
+}
+export interface EvidenceReferenceMapping {
+    readonly previous: OccurrenceReference;
+    readonly current: OccurrenceReference;
+}
+export declare function mapEvidenceReferences(data: {
+    readonly previous: EvidenceMappingSnapshot;
+    readonly current: EvidenceMappingSnapshot;
+    readonly revision: readonly RevisionEntry[];
+}): readonly EvidenceReferenceMapping[];

package/dist/src/chats/reference-mapping.js

@@ -0,0 +1,153 @@
+import { parseCorpus } from "./corpus-input.js";
+import { parseEvidenceCoverage } from "./revision-input.js";
+import { compareEvidence, pairEvidence, } from "./revision.js";
+function coverageEntries(coverage) {
+    if ("files" in coverage)
+        return coverage.files;
+    if ("entries" in coverage)
+        return coverage.entries;
+    throw new Error("analysis-reference-map-refused");
+}
+function coverageCounts(text) {
+    const coverage = JSON.parse(text);
+    if (coverage === null || typeof coverage !== "object")
+        throw new Error("analysis-reference-map-refused");
+    const entries = coverageEntries(coverage);
+    if (!Array.isArray(entries))
+        throw new Error("analysis-reference-map-refused");
+    return entries.map((entry) => {
+        if (entry === null ||
+            typeof entry !== "object" ||
+            !("recordCount" in entry) ||
+            typeof entry.recordCount !== "number" ||
+            !Number.isSafeInteger(entry.recordCount) ||
+            entry.recordCount < 0)
+            throw new Error("analysis-reference-map-refused");
+        return entry.recordCount;
+    });
+}
+function occurrence(source, reference) {
+    const match = /^(sha256:[a-f\d]{64}):(0|[1-9]\d*)$/.exec(reference.sourceId);
+    if (match === null || match[1] !== source.digest)
+        throw new Error("analysis-reference-map-refused");
+    const offset = Number(match[2]);
+    if (!Number.isSafeInteger(offset) ||
+        (source.byteSize !== undefined && offset >= source.byteSize))
+        throw new Error("analysis-reference-map-refused");
+    return { reference, offset };
+}
+function readSelections(snapshot) {
+    if (!/^[a-f\d]{8}-[a-f\d]{4}-4[a-f\d]{3}-[89ab][a-f\d]{3}-[a-f\d]{12}$/.test(snapshot.importId))
+        throw new Error("analysis-reference-map-refused");
+    const sources = parseEvidenceCoverage(snapshot.coverage, snapshot.importId);
+    const records = parseCorpus(snapshot.corpus);
+    const counts = coverageCounts(snapshot.coverage);
+    if (counts.reduce((total, count) => total + count, 0) !== records.length ||
+        sources.some((source, index) => source.status !== "included" && counts[index] !== 0))
+        throw new Error("analysis-reference-map-refused");
+    let index = 0;
+    return sources.map((source, selectionIndex) => {
+        const count = counts[selectionIndex];
+        if (count === undefined)
+            throw new Error("analysis-reference-map-refused");
+        const occurrences = records
+            .slice(index, index + count)
+            .map((record, offset) => occurrence(source, {
+            recordIndex: index + offset,
+            sourceId: record.sourceId,
+        }));
+        if (occurrences.some((item, offset) => offset > 0 && item.offset <= (occurrences[offset - 1]?.offset ?? -1)))
+            throw new Error("analysis-reference-map-refused");
+        index += count;
+        return { ...source, occurrences };
+    });
+}
+function revisionIdentity(entry) {
+    return JSON.stringify([
+        entry.path,
+        entry.root,
+        entry.status,
+        entry.reason,
+        entry.before === null ? null : [entry.before.digest, entry.before.byteSize],
+        entry.after === null ? null : [entry.after.digest, entry.after.byteSize],
+    ]);
+}
+function verifyRevision(data) {
+    const expected = compareEvidence({
+        previous: data.previous,
+        current: data.current.map((source, index) => {
+            const revision = data.revision[index];
+            return {
+                ...source,
+                prefixDigest: revision?.status === "extended"
+                    ? (revision.before?.digest ?? undefined)
+                    : undefined,
+            };
+        }),
+    }).entries;
+    if (data.revision.length !== expected.length ||
+        expected.some((entry, index) => {
+            const actual = data.revision[index];
+            return (actual === undefined ||
+                revisionIdentity(entry) !== revisionIdentity(actual));
+        }))
+        throw new Error("analysis-reference-map-refused");
+}
+function verifyUnchangedSelection(before, after) {
+    if (before.byteSize !== undefined &&
+        after.byteSize !== undefined &&
+        before.byteSize !== after.byteSize)
+        throw new Error("analysis-reference-map-refused");
+    if (before.occurrences.length !== after.occurrences.length)
+        throw new Error("analysis-reference-map-refused");
+}
+function verifyExtendedSelection(before, after) {
+    const prefix = after.occurrences.filter((item) => item.offset < (before.byteSize ?? 0));
+    if (prefix.length !== before.occurrences.length ||
+        prefix.some((item, index) => item.offset !== before.occurrences[index]?.offset))
+        throw new Error("analysis-reference-map-refused");
+}
+function mapSelection(data) {
+    const { before, after, revision } = data;
+    if (revision?.status !== "unchanged" && revision?.status !== "extended")
+        return [];
+    if (revision.status === "unchanged")
+        verifyUnchangedSelection(before, after);
+    if (revision.status === "extended")
+        verifyExtendedSelection(before, after);
+    const indexes = new Map(after.occurrences.map((item, index) => [item.offset, index]));
+    return before.occurrences.flatMap((item) => {
+        const index = indexes.get(item.offset);
+        const matched = index === undefined ? undefined : after.occurrences[index];
+        if (matched === undefined || index === undefined)
+            throw new Error("analysis-reference-map-refused");
+        const next = after.occurrences[index + 1];
+        if (revision.status === "extended" &&
+            (before.byteSize === undefined ||
+                next === undefined ||
+                next.offset > before.byteSize))
+            return [];
+        return [{ previous: item.reference, current: matched.reference }];
+    });
+}
+export function mapEvidenceReferences(data) {
+    try {
+        const previous = readSelections(data.previous);
+        const current = readSelections(data.current);
+        verifyRevision({ previous, current, revision: data.revision });
+        const pairs = pairEvidence({ previous, current });
+        const mappings = current.flatMap((after, index) => {
+            const beforeIndex = pairs[index];
+            const before = beforeIndex === null || beforeIndex === undefined
+                ? undefined
+                : previous[beforeIndex];
+            return before === undefined
+                ? []
+                : mapSelection({ before, after, revision: data.revision[index] });
+        });
+        return mappings.sort((left, right) => left.previous.recordIndex - right.previous.recordIndex);
+    }
+    catch {
+        throw new Error("analysis-reference-map-refused");
+    }
+}

package/dist/src/chats/revision-input.d.ts

@@ -0,0 +1,3 @@
+import { type EvidenceSource } from "./revision.ts";
+export declare function parseEvidenceCoverage(text: string, importId: string): readonly EvidenceSource[];
+export declare function nextEvidenceRevision(text: string | undefined, importId: string): number;

package/dist/src/chats/revision-input.js

@@ -0,0 +1,180 @@
+import { validateCoverage, withinRoot } from "./corpus-input.js";
+import { evidenceReason, revisionReasons, } from "./revision.js";
+function object(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function exact(value, fields) {
+    const { required, optional = [] } = fields;
+    return (required.every((key) => Object.hasOwn(value, key)) &&
+        Object.keys(value).every((key) => required.includes(key) || optional.includes(key)));
+}
+function natural(value) {
+    return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+}
+function absolute(value) {
+    return (typeof value === "string" &&
+        value.startsWith("/") &&
+        !value.includes("\0") &&
+        value.split("/").every((part) => part !== "." && part !== ".."));
+}
+function digest(value) {
+    return (value === null ||
+        (typeof value === "string" && /^sha256:[a-f\d]{64}$/.test(value)));
+}
+function evidenceValue(value) {
+    return (object(value) &&
+        digest(value["digest"]) &&
+        (!Object.hasOwn(value, "byteSize") ||
+            (natural(value["byteSize"]) && value["byteSize"] <= 10_485_760)));
+}
+function source(value, roots) {
+    if (!evidenceValue(value) ||
+        !absolute(value["path"]) ||
+        typeof value["status"] !== "string")
+        throw new Error("invalid-import");
+    return {
+        path: value["path"],
+        root: selectedRoot(value["selection"], roots),
+        status: value["status"],
+        digest: value.digest,
+        ...(value.byteSize === undefined ? {} : { byteSize: value.byteSize }),
+    };
+}
+export function parseEvidenceCoverage(text, importId) {
+    validateCoverage(text, importId);
+    const value = JSON.parse(text);
+    if (!object(value))
+        throw new Error("invalid-import");
+    const entries = value["schemaVersion"] === 1 ? value["files"] : value["entries"];
+    const roots = value["schemaVersion"] === 1 ? [] : value["roots"];
+    if (!Array.isArray(entries) ||
+        !Array.isArray(roots) ||
+        !roots.every((root) => typeof root === "string"))
+        throw new Error("invalid-import");
+    return entries.map((entry) => source(entry, roots));
+}
+function revisionValue(value) {
+    return (value === null ||
+        (evidenceValue(value) &&
+            exact(value, { required: ["digest"], optional: ["byteSize"] })));
+}
+function revisionEntry(value) {
+    if (!object(value) ||
+        !exact(value, {
+            required: ["path", "root", "before", "after", "status", "reason"],
+        }))
+        return false;
+    if (!selectionKey(value))
+        return false;
+    const reason = value["reason"];
+    if (typeof reason !== "string" || !Object.hasOwn(revisionReasons, reason))
+        return false;
+    const expected = Object.entries(revisionReasons).find(([key]) => key === reason)?.[1];
+    return (value["status"] === expected &&
+        revisionPresence(value) &&
+        consistentReason(value));
+}
+function revisionPresence(value) {
+    const reason = value["reason"];
+    if (reason === "no-previous-entry")
+        return value["before"] === null && value["after"] !== null;
+    if (reason === "not-selected")
+        return value["before"] !== null && value["after"] === null;
+    return value["before"] !== null && value["after"] !== null;
+}
+function validCounts(value, entries) {
+    const statuses = [
+        "new",
+        "missing",
+        "unchanged",
+        "changed",
+        "truncated",
+        "extended",
+    ];
+    return (object(value) &&
+        exact(value, { required: statuses }) &&
+        statuses.every((status) => natural(value[status]) &&
+            value[status] ===
+                entries.filter((entry) => entry["status"] === status).length));
+}
+export function nextEvidenceRevision(text, importId) {
+    if (text === undefined)
+        return 1;
+    const value = JSON.parse(text);
+    if (!object(value) ||
+        !exact(value, {
+            required: [
+                "schemaVersion",
+                "importId",
+                "previousImportId",
+                "evidenceRevision",
+                "counts",
+                "entries",
+            ],
+        }))
+        throw new Error("invalid-import");
+    if (!revisionHeader(value, importId))
+        throw new Error("invalid-import");
+    const entries = value["entries"];
+    if (!Array.isArray(entries) ||
+        !entries.every(revisionEntry) ||
+        !validCounts(value["counts"], entries))
+        throw new Error("invalid-import");
+    const revision = value["evidenceRevision"];
+    if (!natural(revision))
+        throw new Error("invalid-import");
+    return revision + 1;
+}
+function selectedRoot(selection, roots) {
+    if (roots.length === 0 || !object(selection) || selection["kind"] !== "root")
+        return null;
+    const index = selection["index"];
+    if (!natural(index) || roots[index] === undefined)
+        throw new Error("invalid-import");
+    return roots[index];
+}
+function selectionKey(value) {
+    return (absolute(value["path"]) &&
+        (value["root"] === null ||
+            (absolute(value["root"]) && withinRoot(value["root"], value["path"]))));
+}
+function revisionHeader(value, importId) {
+    const revision = value["evidenceRevision"];
+    if (value["schemaVersion"] !== 1 ||
+        value["importId"] !== importId ||
+        !natural(revision) ||
+        revision === 0 ||
+        revision === Number.MAX_SAFE_INTEGER)
+        return false;
+    const previous = value["previousImportId"];
+    if (typeof previous !== "string" ||
+        previous === importId ||
+        !/^[a-f\d]{8}-[a-f\d]{4}-4[a-f\d]{3}-[89ab][a-f\d]{3}-[a-f\d]{12}$/.test(previous))
+        return false;
+    return true;
+}
+function consistentReason(value) {
+    const before = value["before"];
+    const after = value["after"];
+    if (!revisionValue(before) || !revisionValue(after))
+        return false;
+    const reason = value["reason"];
+    const unverified = [
+        "no-previous-entry",
+        "not-selected",
+        "not-included",
+    ].includes(String(reason));
+    if (!unverified && after?.digest === null)
+        return false;
+    const prefix = reason === "verified-prefix" ? before?.digest : undefined;
+    return (evidenceReason({
+        before: comparable(before, { status: "included" }),
+        after: comparable(after, {
+            status: reason === "not-included" ? "unsupported" : "included",
+            prefixDigest: prefix ?? undefined,
+        }),
+    }) === reason);
+}
+function comparable(value, state) {
+    return value === null ? null : { ...value, ...state };
+}

package/dist/src/chats/revision.d.ts

@@ -0,0 +1,48 @@
+export interface SelectionKey {
+    readonly path: string;
+    readonly root: string | null;
+}
+export interface EvidenceValue {
+    readonly digest: string | null;
+    readonly byteSize?: number | undefined;
+}
+export interface ComparableEvidence extends EvidenceValue {
+    readonly status: string;
+    readonly prefixDigest?: string | undefined;
+}
+export interface EvidenceSource extends SelectionKey, ComparableEvidence {
+}
+export declare const revisionReasons: {
+    readonly "no-previous-entry": "new";
+    readonly "not-selected": "missing";
+    readonly "not-included": "missing";
+    readonly "same-digest": "unchanged";
+    readonly "previous-digest-unknown": "changed";
+    readonly "previous-size-unknown": "changed";
+    readonly "shorter-source": "truncated";
+    readonly "verified-prefix": "extended";
+    readonly "digest-changed": "changed";
+};
+export type RevisionReason = keyof typeof revisionReasons;
+export type RevisionStatus = (typeof revisionReasons)[RevisionReason];
+export interface RevisionEntry extends SelectionKey {
+    readonly before: EvidenceValue | null;
+    readonly after: EvidenceValue | null;
+    readonly status: RevisionStatus;
+    readonly reason: RevisionReason;
+}
+export declare function pairEvidence(data: {
+    readonly previous: readonly SelectionKey[];
+    readonly current: readonly SelectionKey[];
+}): readonly (number | null)[];
+export declare function evidenceReason(data: {
+    readonly before: ComparableEvidence | null;
+    readonly after: ComparableEvidence | null;
+}): RevisionReason;
+export declare function compareEvidence(data: {
+    readonly previous: readonly EvidenceSource[];
+    readonly current: readonly EvidenceSource[];
+}): {
+    counts: Record<RevisionStatus, number>;
+    entries: RevisionEntry[];
+};