cloving 1.2.0 → 1.4.0

package/dist/src/chats/reader.js

@@ -1,118 +0,0 @@
-import { createHash } from "node:crypto";
-import { constants } from "node:fs";
-import { lstat, open } from "node:fs/promises";
-import { sourceFailure, verifyScanSource, verifyOpenSource, } from "./source.js";
-import { normalizeTranscript, } from "./normalize.js";
-export const transcriptLimit = 10_485_760;
-function refused(path, reason) {
-    const status = reason === "source-unreadable" ? "unreadable" : "unsupported";
-    return {
-        coverage: {
-            path,
-            status: reason === "file-too-large" ? "too-large" : status,
-            reason,
-            digest: null,
-            recordCount: 0,
-            omissions: { metadataRecords: 0, thinkingBlocks: 0 },
-        },
-        records: [],
-    };
-}
-async function boundedBytes(handle) {
-    const stats = await handle.stat();
-    if (!stats.isFile())
-        return "nonregular-file";
-    if (stats.size > transcriptLimit)
-        return "file-too-large";
-    const chunks = [];
-    let total = 0;
-    while (total <= transcriptLimit) {
-        const buffer = Buffer.alloc(Math.min(65_536, transcriptLimit + 1 - total));
-        const { bytesRead } = await handle.read(buffer, 0, buffer.length, total);
-        if (bytesRead === 0)
-            break;
-        total += bytesRead;
-        chunks.push(buffer.subarray(0, bytesRead));
-    }
-    if (total > transcriptLimit || (await handle.stat()).size > transcriptLimit)
-        return "file-too-large";
-    return Buffer.concat(chunks, total);
-}
-function normalizeBytes(path, options) {
-    const { bytes } = options;
-    const digest = `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
-    let text;
-    try {
-        text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
-    }
-    catch {
-        const result = refused(path, "invalid-utf8");
-        return { ...result, coverage: { ...result.coverage, digest } };
-    }
-    const result = normalizeTranscript(text, digest);
-    return {
-        coverage: {
-            path,
-            status: result.status,
-            reason: result.reason,
-            digest,
-            ...(result.status === "included" ? { byteSize: bytes.length } : {}),
-            recordCount: result.records.length,
-            omissions: result.omissions,
-        },
-        records: result.records,
-        ...(options.prefixBytes !== undefined &&
-            options.prefixBytes >= 0 &&
-            options.prefixBytes < bytes.length
-            ? {
-                prefixDigest: `sha256:${createHash("sha256").update(bytes.subarray(0, options.prefixBytes)).digest("hex")}`,
-            }
-            : {}),
-    };
-}
-async function readSourceBytes(path, inspection) {
-    const handle = await open(path, constants.O_RDONLY |
-        constants.O_NONBLOCK |
-        (inspection === undefined ? constants.O_NOFOLLOW : 0x20000000));
-    try {
-        if (inspection !== undefined) {
-            verifyOpenSource(inspection, await handle.stat());
-            await verifyScanSource(inspection);
-        }
-        return await boundedBytes(handle);
-    }
-    finally {
-        await handle.close();
-    }
-}
-export async function readTranscript(path, options = {}) {
-    const { inspection } = options;
-    try {
-        if (inspection !== undefined)
-            await verifyScanSource(inspection);
-        const stats = await lstat(path);
-        if (stats.isSymbolicLink())
-            return refused(path, "symbolic-link");
-        if (!stats.isFile())
-            return refused(path, "nonregular-file");
-        const bytes = await readSourceBytes(path, inspection);
-        const result = typeof bytes === "string"
-            ? refused(path, bytes)
-            : normalizeBytes(path, { bytes, ...options });
-        return scannedResult(result, inspection);
-    }
-    catch (error) {
-        return readFailure(path, { error, inspection });
-    }
-}
-function scannedResult(result, inspection) {
-    if (inspection === undefined || result.coverage.status !== "too-large")
-        return result;
-    return { ...result, coverage: { ...result.coverage, status: "unsupported" } };
-}
-function readFailure(path, data) {
-    const { error, inspection } = data;
-    if (error instanceof Error && "code" in error && error.code === "ELOOP")
-        return refused(path, "symbolic-link");
-    return refused(path, inspection === undefined ? "source-unreadable" : sourceFailure(error));
-}

package/dist/src/chats/redact-import.d.ts

@@ -1,14 +0,0 @@
-import { type RedactionSettings } from "./redaction.ts";
-export interface RedactImportRequest {
-    readonly importId: string;
-    readonly workingDirectory: string;
-    readonly runtimeDirectory: string | undefined;
-    readonly settings: RedactionSettings;
-}
-export declare function redactImport(request: RedactImportRequest): Promise<{
-    code: string;
-    importId: string;
-    recordCount: number;
-    counts: import("./redaction.ts").RedactionCounts;
-    totalCount: number;
-}>;

package/dist/src/chats/redact-import.js

@@ -1,141 +0,0 @@
-import { openInput, readInput } from "./input-file.js";
-import { lstat, rm } from "node:fs/promises";
-import { join, resolve } from "node:path";
-import { createPrivateFile, validateDestination } from "./destination.js";
-import { parseCorpusRecord, validateCoverage } from "./corpus-input.js";
-import { emptyCounts, redactRecord, categories, } from "./redaction.js";
-async function absent(path) {
-    try {
-        await lstat(path);
-    }
-    catch (error) {
-        if (error instanceof Error && "code" in error && error.code === "ENOENT")
-            return;
-        throw error;
-    }
-    throw new Error("outputs-exist");
-}
-async function* corpusLines(handle) {
-    const buffer = Buffer.alloc(65_536);
-    const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
-    let pending = "";
-    let bytesRead = -1;
-    while (bytesRead !== 0) {
-        ({ bytesRead } = await handle.read({ buffer, position: null }));
-        pending += decoder.decode(buffer.subarray(0, bytesRead), {
-            stream: bytesRead !== 0,
-        });
-        let start = 0;
-        let end = pending.indexOf("\n");
-        while (end !== -1) {
-            yield pending.slice(start, end);
-            start = end + 1;
-            end = pending.indexOf("\n", start);
-        }
-        pending = pending.slice(start);
-    }
-    if (pending !== "")
-        yield pending;
-}
-async function redactCorpus(handles, settings) {
-    const counts = emptyCounts();
-    let recordCount = 0;
-    for await (const line of corpusLines(handles.input)) {
-        const result = redactRecord(parseCorpusRecord(line), settings);
-        for (const category of categories)
-            counts[category] += result.counts[category];
-        await handles.output.writeFile(JSON.stringify(result.record) + "\n");
-        recordCount++;
-    }
-    return {
-        counts,
-        recordCount,
-        totalCount: Object.values(counts).reduce((sum, count) => sum + count, 0),
-    };
-}
-async function writeRedactedCorpus(directory, settings) {
-    const path = join(directory, "redacted.jsonl");
-    let created = false;
-    try {
-        const input = await openInput(join(directory, "corpus.jsonl"));
-        try {
-            const output = await createPrivateFile(path);
-            created = true;
-            try {
-                return await redactCorpus({ input, output }, settings);
-            }
-            finally {
-                await output.close();
-            }
-        }
-        finally {
-            await input.close();
-        }
-    }
-    catch (error) {
-        if (created) {
-            await validateDestination(directory);
-            await rm(path, { force: true });
-        }
-        throw error;
-    }
-}
-async function writeManifest(directory, manifest) {
-    const path = join(directory, "redaction.json");
-    let created = false;
-    try {
-        const handle = await createPrivateFile(path);
-        created = true;
-        try {
-            await handle.writeFile(JSON.stringify(manifest) + "\n");
-        }
-        finally {
-            await handle.close();
-        }
-    }
-    catch (error) {
-        if (created) {
-            await validateDestination(directory);
-            await rm(path, { force: true });
-        }
-        throw error;
-    }
-}
-export async function redactImport(request) {
-    const empty = {
-        importId: request.importId,
-        recordCount: 0,
-        counts: emptyCounts(),
-        totalCount: 0,
-    };
-    if (process.platform !== "darwin")
-        return { ...empty, code: "platform-unsupported" };
-    if (request.runtimeDirectory === undefined)
-        return { ...empty, code: "runtime-directory-unknown" };
-    try {
-        if (!/^[a-f\d]{8}-[a-f\d]{4}-4[a-f\d]{3}-[89ab][a-f\d]{3}-[a-f\d]{12}$/.test(request.importId))
-            throw new Error("invalid-import");
-        const directory = resolve(request.workingDirectory, request.runtimeDirectory, "chats", request.importId);
-        await validateDestination(directory);
-        if (!(await lstat(directory)).isDirectory())
-            throw new Error("invalid-import");
-        await absent(join(directory, "redacted.jsonl"));
-        await absent(join(directory, "redaction.json"));
-        validateCoverage(await readInput(join(directory, "coverage.json")), request.importId);
-        const summary = await writeRedactedCorpus(directory, request.settings);
-        await writeManifest(directory, {
-            schemaVersion: 1,
-            importId: request.importId,
-            ruleVersion: 1,
-            ...summary,
-        });
-        return {
-            code: "redaction-complete",
-            importId: request.importId,
-            ...summary,
-        };
-    }
-    catch {
-        return { ...empty, code: "redaction-failed" };
-    }
-}

package/dist/src/chats/redaction.d.ts

@@ -1,41 +0,0 @@
-import type { MessageRecord } from "./normalize.ts";
-export interface RedactionSettings {
-    readonly home: string | null;
-    readonly privateTexts: readonly string[];
-}
-export declare const replacements: {
-    readonly privateKey: "[PRIVATE_KEY]";
-    readonly credentialUrl: "[CREDENTIAL_URL]";
-    readonly homePath: "[HOME_PATH]";
-    readonly token: "[TOKEN]";
-    readonly email: "[EMAIL]";
-    readonly privateText: "[PRIVATE_TEXT]";
-};
-export declare const categories: readonly ["privateKey", "credentialUrl", "homePath", "token", "email", "privateText"];
-export type Category = keyof typeof replacements;
-export type RedactionCounts = Record<Category, number>;
-export declare function emptyCounts(): RedactionCounts;
-export declare function redactRecord(record: MessageRecord, settings: RedactionSettings): {
-    record: {
-        text: string;
-        blocks: Record<string, unknown>[];
-        sourceId: string;
-        role: "user" | "assistant";
-        timestamp: string | null;
-        sessionId: string | null;
-        schemaVersion: 1;
-    } | {
-        text: string;
-        blocks: Record<string, unknown>[];
-        sourceId: string;
-        role: "user" | "assistant";
-        timestamp: string | null;
-        sessionId: string | null;
-        schemaVersion: 2;
-        uuid: string | null;
-        isSidechain: boolean | null;
-        parentUuid?: string | null;
-    };
-    counts: RedactionCounts;
-    totalCount: number;
-};

package/dist/src/chats/redaction.js

@@ -1,161 +0,0 @@
-export const replacements = {
-    privateKey: "[PRIVATE_KEY]",
-    credentialUrl: "[CREDENTIAL_URL]",
-    homePath: "[HOME_PATH]",
-    token: "[TOKEN]",
-    email: "[EMAIL]",
-    privateText: "[PRIVATE_TEXT]",
-};
-export const categories = [
-    "privateKey",
-    "credentialUrl",
-    "homePath",
-    "token",
-    "email",
-    "privateText",
-];
-export function emptyCounts() {
-    return {
-        privateKey: 0,
-        credentialUrl: 0,
-        homePath: 0,
-        token: 0,
-        email: 0,
-        privateText: 0,
-    };
-}
-function escaped(text) {
-    return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-function patternSpans(text, rule) {
-    return [...text.matchAll(rule.pattern)].map((match) => ({
-        start: match.index,
-        end: match.index + match[0].length,
-        category: rule.category,
-    }));
-}
-function homeRoot(home) {
-    if (home === null || home === "")
-        return "~";
-    if (home === "/")
-        return "(?:(?=/)|~)";
-    return `(?:${escaped(home.replace(/\/$/, ""))}|~)`;
-}
-function homeSpans(text, home) {
-    const root = homeRoot(home);
-    const pattern = new RegExp(String.raw `(?<![\p{L}\p{N}_./~\\-])${root}(?=$|[\s/\x22'\x60<>,;:!?()\[\]{}])(?:/[^\s\x22'\x60<>]*)?`, "gu");
-    return patternSpans(text, { pattern, category: "homePath" }).map((span) => {
-        const quote = text[span.start - 1];
-        if (quote !== '"' && quote !== "'" && quote !== "`")
-            return span;
-        const end = text.indexOf(quote, span.end);
-        return end === -1 ? span : { ...span, end };
-    });
-}
-function spans(text, settings) {
-    const rules = [
-        {
-            pattern: /-----BEGIN ((?:[A-Z0-9]+ )*PRIVATE KEY)-----[\s\S]*?-----END \1-----/g,
-            category: "privateKey",
-        },
-        {
-            pattern: /\b[a-z][a-z\d+.-]*:\/\/[^\s<>"'`/?#]*@[^\s<>"'`]+/gi,
-            category: "credentialUrl",
-        },
-        {
-            pattern: /(?<![\p{L}\p{N}_])(?:gh[pousr]_|github_pat_|sk-ant-|sk-(?:proj-|svcacct-)?)[a-z\d_-]+/giu,
-            category: "token",
-        },
-        {
-            pattern: /[\w.!#$%&'*+/=?^`{|}~-]+@[\w.-]+\.[a-z]{2,}/gi,
-            category: "email",
-        },
-    ];
-    const matches = rules.flatMap((rule) => patternSpans(text, rule));
-    for (const match of homeSpans(text, settings.home))
-        matches.push(match);
-    for (const privateText of settings.privateTexts) {
-        if (privateText.trim() === "")
-            throw new Error("invalid-settings");
-        const pattern = new RegExp(`(?=(${escaped(privateText)}))`, "gi");
-        for (const match of text.matchAll(pattern))
-            matches.push({
-                start: match.index,
-                end: match.index + String(match[1]).length,
-                category: "privateText",
-            });
-    }
-    return mergeSpans(matches);
-}
-function mergeSpans(matches) {
-    const priority = categories;
-    const merged = [];
-    for (const match of matches.sort((left, right) => left.start - right.start)) {
-        const previous = merged.at(-1);
-        if (previous === undefined || previous.end <= match.start) {
-            merged.push({ ...match });
-            continue;
-        }
-        previous.end = Math.max(previous.end, match.end);
-        if (priority.indexOf(match.category) < priority.indexOf(previous.category))
-            previous.category = match.category;
-    }
-    return merged;
-}
-function redactText(text, context) {
-    const { settings, counts } = context;
-    let result = "";
-    let cursor = 0;
-    for (const span of spans(text, settings)) {
-        result += text.slice(cursor, span.start) + replacements[span.category];
-        cursor = span.end;
-        counts[span.category]++;
-    }
-    return result + text.slice(cursor);
-}
-function rawJsonApi(value) {
-    return "isRawJSON" in value && typeof value.isRawJSON === "function";
-}
-function rawJson(value) {
-    return rawJsonApi(JSON) && JSON.isRawJSON(value);
-}
-function nested(value, context) {
-    if (typeof value === "string")
-        return redactText(value, context);
-    if (rawJson(value))
-        return value;
-    if (Array.isArray(value))
-        return value.map((item) => nested(item, context));
-    if (value === null || typeof value !== "object")
-        return value;
-    const result = {};
-    for (const [key, item] of Object.entries(value)) {
-        const redactedKey = redactText(key, context);
-        if (Object.hasOwn(result, redactedKey))
-            throw new Error("member-name-collision");
-        Object.defineProperty(result, redactedKey, {
-            value: nested(item, context),
-            enumerable: true,
-        });
-    }
-    return result;
-}
-export function redactRecord(record, settings) {
-    const counts = emptyCounts();
-    const context = { settings, counts };
-    const blocks = record.blocks.map((block) => {
-        if (block["type"] === "tool_use")
-            return { ...block, input: nested(block["input"], context) };
-        if (block["type"] === "tool_result")
-            return { ...block, content: nested(block["content"], context) };
-        if (block["type"] === "text" && typeof block["text"] === "string")
-            return { ...block, text: redactText(block["text"], context) };
-        return block;
-    });
-    const text = redactText(record.text, context);
-    return {
-        record: { ...record, text, blocks },
-        counts,
-        totalCount: Object.values(counts).reduce((sum, count) => sum + count, 0),
-    };
-}

package/dist/src/chats/reference-mapping.d.ts

@@ -1,16 +0,0 @@
-import { type RevisionEntry } from "./revision.ts";
-import type { OccurrenceReference } from "./structure.ts";
-export interface EvidenceMappingSnapshot {
-    readonly importId: string;
-    readonly coverage: string;
-    readonly corpus: string;
-}
-export interface EvidenceReferenceMapping {
-    readonly previous: OccurrenceReference;
-    readonly current: OccurrenceReference;
-}
-export declare function mapEvidenceReferences(data: {
-    readonly previous: EvidenceMappingSnapshot;
-    readonly current: EvidenceMappingSnapshot;
-    readonly revision: readonly RevisionEntry[];
-}): readonly EvidenceReferenceMapping[];

package/dist/src/chats/reference-mapping.js

@@ -1,153 +0,0 @@
-import { parseCorpus } from "./corpus-input.js";
-import { parseEvidenceCoverage } from "./revision-input.js";
-import { compareEvidence, pairEvidence, } from "./revision.js";
-function coverageEntries(coverage) {
-    if ("files" in coverage)
-        return coverage.files;
-    if ("entries" in coverage)
-        return coverage.entries;
-    throw new Error("analysis-reference-map-refused");
-}
-function coverageCounts(text) {
-    const coverage = JSON.parse(text);
-    if (coverage === null || typeof coverage !== "object")
-        throw new Error("analysis-reference-map-refused");
-    const entries = coverageEntries(coverage);
-    if (!Array.isArray(entries))
-        throw new Error("analysis-reference-map-refused");
-    return entries.map((entry) => {
-        if (entry === null ||
-            typeof entry !== "object" ||
-            !("recordCount" in entry) ||
-            typeof entry.recordCount !== "number" ||
-            !Number.isSafeInteger(entry.recordCount) ||
-            entry.recordCount < 0)
-            throw new Error("analysis-reference-map-refused");
-        return entry.recordCount;
-    });
-}
-function occurrence(source, reference) {
-    const match = /^(sha256:[a-f\d]{64}):(0|[1-9]\d*)$/.exec(reference.sourceId);
-    if (match === null || match[1] !== source.digest)
-        throw new Error("analysis-reference-map-refused");
-    const offset = Number(match[2]);
-    if (!Number.isSafeInteger(offset) ||
-        (source.byteSize !== undefined && offset >= source.byteSize))
-        throw new Error("analysis-reference-map-refused");
-    return { reference, offset };
-}
-function readSelections(snapshot) {
-    if (!/^[a-f\d]{8}-[a-f\d]{4}-4[a-f\d]{3}-[89ab][a-f\d]{3}-[a-f\d]{12}$/.test(snapshot.importId))
-        throw new Error("analysis-reference-map-refused");
-    const sources = parseEvidenceCoverage(snapshot.coverage, snapshot.importId);
-    const records = parseCorpus(snapshot.corpus);
-    const counts = coverageCounts(snapshot.coverage);
-    if (counts.reduce((total, count) => total + count, 0) !== records.length ||
-        sources.some((source, index) => source.status !== "included" && counts[index] !== 0))
-        throw new Error("analysis-reference-map-refused");
-    let index = 0;
-    return sources.map((source, selectionIndex) => {
-        const count = counts[selectionIndex];
-        if (count === undefined)
-            throw new Error("analysis-reference-map-refused");
-        const occurrences = records
-            .slice(index, index + count)
-            .map((record, offset) => occurrence(source, {
-            recordIndex: index + offset,
-            sourceId: record.sourceId,
-        }));
-        if (occurrences.some((item, offset) => offset > 0 && item.offset <= (occurrences[offset - 1]?.offset ?? -1)))
-            throw new Error("analysis-reference-map-refused");
-        index += count;
-        return { ...source, occurrences };
-    });
-}
-function revisionIdentity(entry) {
-    return JSON.stringify([
-        entry.path,
-        entry.root,
-        entry.status,
-        entry.reason,
-        entry.before === null ? null : [entry.before.digest, entry.before.byteSize],
-        entry.after === null ? null : [entry.after.digest, entry.after.byteSize],
-    ]);
-}
-function verifyRevision(data) {
-    const expected = compareEvidence({
-        previous: data.previous,
-        current: data.current.map((source, index) => {
-            const revision = data.revision[index];
-            return {
-                ...source,
-                prefixDigest: revision?.status === "extended"
-                    ? (revision.before?.digest ?? undefined)
-                    : undefined,
-            };
-        }),
-    }).entries;
-    if (data.revision.length !== expected.length ||
-        expected.some((entry, index) => {
-            const actual = data.revision[index];
-            return (actual === undefined ||
-                revisionIdentity(entry) !== revisionIdentity(actual));
-        }))
-        throw new Error("analysis-reference-map-refused");
-}
-function verifyUnchangedSelection(before, after) {
-    if (before.byteSize !== undefined &&
-        after.byteSize !== undefined &&
-        before.byteSize !== after.byteSize)
-        throw new Error("analysis-reference-map-refused");
-    if (before.occurrences.length !== after.occurrences.length)
-        throw new Error("analysis-reference-map-refused");
-}
-function verifyExtendedSelection(before, after) {
-    const prefix = after.occurrences.filter((item) => item.offset < (before.byteSize ?? 0));
-    if (prefix.length !== before.occurrences.length ||
-        prefix.some((item, index) => item.offset !== before.occurrences[index]?.offset))
-        throw new Error("analysis-reference-map-refused");
-}
-function mapSelection(data) {
-    const { before, after, revision } = data;
-    if (revision?.status !== "unchanged" && revision?.status !== "extended")
-        return [];
-    if (revision.status === "unchanged")
-        verifyUnchangedSelection(before, after);
-    if (revision.status === "extended")
-        verifyExtendedSelection(before, after);
-    const indexes = new Map(after.occurrences.map((item, index) => [item.offset, index]));
-    return before.occurrences.flatMap((item) => {
-        const index = indexes.get(item.offset);
-        const matched = index === undefined ? undefined : after.occurrences[index];
-        if (matched === undefined || index === undefined)
-            throw new Error("analysis-reference-map-refused");
-        const next = after.occurrences[index + 1];
-        if (revision.status === "extended" &&
-            (before.byteSize === undefined ||
-                next === undefined ||
-                next.offset > before.byteSize))
-            return [];
-        return [{ previous: item.reference, current: matched.reference }];
-    });
-}
-export function mapEvidenceReferences(data) {
-    try {
-        const previous = readSelections(data.previous);
-        const current = readSelections(data.current);
-        verifyRevision({ previous, current, revision: data.revision });
-        const pairs = pairEvidence({ previous, current });
-        const mappings = current.flatMap((after, index) => {
-            const beforeIndex = pairs[index];
-            const before = beforeIndex === null || beforeIndex === undefined
-                ? undefined
-                : previous[beforeIndex];
-            return before === undefined
-                ? []
-                : mapSelection({ before, after, revision: data.revision[index] });
-        });
-        return mappings.sort((left, right) => left.previous.recordIndex - right.previous.recordIndex);
-    }
-    catch {
-        throw new Error("analysis-reference-map-refused");
-    }
-}

package/dist/src/chats/revision-input.d.ts

@@ -1,3 +0,0 @@
-import { type EvidenceSource } from "./revision.ts";
-export declare function parseEvidenceCoverage(text: string, importId: string): readonly EvidenceSource[];
-export declare function nextEvidenceRevision(text: string | undefined, importId: string): number;