cloving 1.2.0 → 1.3.0

package/dist/src/chats/normalize.d.ts

@@ -1,29 +0,0 @@
-export interface Omissions {
-    metadataRecords: number;
-    thinkingBlocks: number;
-}
-interface MessageFields {
-    readonly sourceId: string;
-    readonly role: "user" | "assistant";
-    readonly timestamp: string | null;
-    readonly sessionId: string | null;
-    readonly text: string;
-    readonly blocks: readonly Record<string, unknown>[];
-}
-export type MessageRecord = MessageFields & ({
-    readonly schemaVersion: 1;
-} | {
-    readonly schemaVersion: 2;
-    readonly uuid: string | null;
-    readonly isSidechain: boolean | null;
-    readonly parentUuid?: string | null;
-});
-export interface NormalizedTranscript {
-    readonly status: "included" | "unsupported";
-    readonly reason: string;
-    readonly records: readonly MessageRecord[];
-    readonly omissions: Omissions;
-}
-export declare function parsePreservedJson(text: string): unknown;
-export declare function normalizeTranscript(text: string, digest: string): NormalizedTranscript;
-export {};

package/dist/src/chats/normalize.js

@@ -1,155 +0,0 @@
-function object(value) {
-    return value !== null && typeof value === "object" && !Array.isArray(value);
-}
-function supportedBlock(block) {
-    const type = block["type"];
-    if (type === "text" && typeof block["text"] === "string")
-        return { type, text: block["text"] };
-    if (type === "tool_use" &&
-        typeof block["id"] === "string" &&
-        typeof block["name"] === "string" &&
-        Object.hasOwn(block, "input")) {
-        return {
-            type,
-            id: block["id"],
-            name: block["name"],
-            input: block["input"],
-        };
-    }
-    if (type === "tool_result")
-        return toolResult(block);
-    throw new Error("unsupported-block");
-}
-function toolResult(block) {
-    if (typeof block["tool_use_id"] !== "string" ||
-        !Object.hasOwn(block, "content"))
-        throw new Error("unsupported-block");
-    if (Object.hasOwn(block, "is_error") &&
-        typeof block["is_error"] !== "boolean")
-        throw new Error("unsupported-block");
-    return {
-        type: "tool_result",
-        tool_use_id: block["tool_use_id"],
-        content: block["content"],
-        ...(Object.hasOwn(block, "is_error")
-            ? { is_error: block["is_error"] }
-            : {}),
-    };
-}
-function optionalText(value) {
-    if (value === undefined || value === null)
-        return null;
-    if (typeof value !== "string")
-        throw new Error("unsupported-message");
-    return value;
-}
-function rawJsonApi(value) {
-    return "rawJSON" in value && typeof value.rawJSON === "function";
-}
-function preserveJson(...argumentsList) {
-    const [, value, context] = argumentsList;
-    if (typeof value !== "number")
-        return value;
-    if (!object(context) || typeof context["source"] !== "string")
-        throw new Error("unsupported-record");
-    if (JSON.stringify(value) === context["source"])
-        return value;
-    if (!rawJsonApi(JSON))
-        throw new Error("unsupported-record");
-    return JSON.rawJSON(context["source"]);
-}
-export function parsePreservedJson(text) {
-    return JSON.parse(text, preserveJson);
-}
-function blocks(content, omissions) {
-    if (typeof content === "string")
-        return [{ position: 0, type: "text", text: content }];
-    if (!Array.isArray(content))
-        throw new Error("unsupported-message");
-    return content.flatMap((block, position) => {
-        if (!object(block))
-            throw new Error("unsupported-block");
-        if (block["type"] === "thinking") {
-            omissions.thinkingBlocks++;
-            return [];
-        }
-        return [{ ...supportedBlock(block), position }];
-    });
-}
-function identity(value) {
-    const uuid = value["uuid"];
-    const parent = value["parentUuid"];
-    const sidechain = value["isSidechain"];
-    return {
-        uuid: typeof uuid === "string" && uuid !== "" ? uuid : null,
-        isSidechain: typeof sidechain === "boolean" ? sidechain : null,
-        ...(parent === null || (typeof parent === "string" && parent !== "")
-            ? { parentUuid: parent }
-            : {}),
-    };
-}
-function message(value, context) {
-    if (!object(value) || typeof value["type"] !== "string")
-        throw new Error("unsupported-record");
-    if (value["type"] !== "user" &&
-        value["type"] !== "assistant" &&
-        Object.hasOwn(value, "message"))
-        throw new Error("unsupported-message");
-    if (value["type"] !== "user" && value["type"] !== "assistant") {
-        context.omissions.metadataRecords++;
-        return null;
-    }
-    const envelope = value["message"];
-    if (!object(envelope) || envelope["role"] !== value["type"])
-        throw new Error("unsupported-message");
-    const normalized = blocks(envelope["content"], context.omissions);
-    return {
-        schemaVersion: 2,
-        ...identity(value),
-        sourceId: context.sourceId,
-        role: value["type"],
-        timestamp: optionalText(value["timestamp"]),
-        sessionId: optionalText(value["sessionId"]),
-        text: normalized
-            .filter((block) => block["type"] === "text")
-            .map((block) => block["text"])
-            .join("\n"),
-        blocks: normalized,
-    };
-}
-export function normalizeTranscript(text, digest) {
-    const omissions = { metadataRecords: 0, thinkingBlocks: 0 };
-    const records = [];
-    if (text === "")
-        return { status: "unsupported", reason: "empty-file", records, omissions };
-    let offset = 0;
-    const lines = text.split("\n");
-    if (lines.at(-1) === "")
-        lines.pop();
-    try {
-        for (const line of lines) {
-            const parsed = parsePreservedJson(line);
-            const record = message(parsed, {
-                sourceId: `${digest}:${String(offset)}`,
-                omissions,
-            });
-            if (record !== null)
-                records.push(record);
-            offset += new TextEncoder().encode(line).length + 1;
-        }
-    }
-    catch (error) {
-        let reason = "unsupported-record";
-        if (error instanceof Error)
-            reason = error.message;
-        if (error instanceof SyntaxError)
-            reason = "invalid-json";
-        return {
-            status: "unsupported",
-            reason,
-            records: [],
-            omissions,
-        };
-    }
-    return { status: "included", reason: "supported", records, omissions };
-}

package/dist/src/chats/previous-import.d.ts

@@ -1,5 +0,0 @@
-export declare function readPreviousImport(runtime: string, importId: string): Promise<{
-    importId: string;
-    entries: readonly import("./revision.ts").EvidenceSource[];
-    evidenceRevision: number;
-}>;

package/dist/src/chats/previous-import.js

@@ -1,34 +0,0 @@
-import { lstat } from "node:fs/promises";
-import { join } from "node:path";
-import { validateDestination } from "./destination.js";
-import { readInput } from "./input-file.js";
-import { parseEvidenceCoverage, nextEvidenceRevision, } from "./revision-input.js";
-async function optionalRevision(path) {
-    try {
-        await lstat(path);
-    }
-    catch (error) {
-        if (error instanceof Error && "code" in error && error.code === "ENOENT")
-            return;
-        throw error;
-    }
-    return readInput(path);
-}
-export async function readPreviousImport(runtime, importId) {
-    if (process.platform !== "darwin")
-        throw new Error("platform-unsupported");
-    try {
-        if (!/^[a-f\d]{8}-[a-f\d]{4}-4[a-f\d]{3}-[89ab][a-f\d]{3}-[a-f\d]{12}$/.test(importId))
-            throw new Error("invalid-import");
-        const directory = join(runtime, "chats", importId);
-        await validateDestination(directory);
-        if (!(await lstat(directory)).isDirectory())
-            throw new Error("invalid-import");
-        const entries = parseEvidenceCoverage(await readInput(join(directory, "coverage.json")), importId);
-        const evidenceRevision = nextEvidenceRevision(await optionalRevision(join(directory, "revision.json")), importId);
-        return { importId, entries, evidenceRevision };
-    }
-    catch {
-        throw new Error("previous-import-refused");
-    }
-}

package/dist/src/chats/reader.d.ts

@@ -1,21 +0,0 @@
-import { type ScanSource } from "./source.ts";
-import { type MessageRecord, type Omissions } from "./normalize.ts";
-export declare const transcriptLimit = 10485760;
-export interface SelectedTranscript {
-    readonly path: string;
-    readonly status: "included" | "unreadable" | "unsupported" | "too-large";
-    readonly reason: string;
-    readonly digest: string | null;
-    readonly byteSize?: number;
-    readonly recordCount: number;
-    readonly omissions: Omissions;
-}
-export interface ReadTranscript {
-    readonly coverage: SelectedTranscript;
-    readonly records: readonly MessageRecord[];
-    readonly prefixDigest?: string;
-}
-export declare function readTranscript(path: string, options?: {
-    readonly inspection?: ScanSource;
-    readonly prefixBytes?: number;
-}): Promise<ReadTranscript>;

package/dist/src/chats/reader.js

@@ -1,118 +0,0 @@
-import { createHash } from "node:crypto";
-import { constants } from "node:fs";
-import { lstat, open } from "node:fs/promises";
-import { sourceFailure, verifyScanSource, verifyOpenSource, } from "./source.js";
-import { normalizeTranscript, } from "./normalize.js";
-export const transcriptLimit = 10_485_760;
-function refused(path, reason) {
-    const status = reason === "source-unreadable" ? "unreadable" : "unsupported";
-    return {
-        coverage: {
-            path,
-            status: reason === "file-too-large" ? "too-large" : status,
-            reason,
-            digest: null,
-            recordCount: 0,
-            omissions: { metadataRecords: 0, thinkingBlocks: 0 },
-        },
-        records: [],
-    };
-}
-async function boundedBytes(handle) {
-    const stats = await handle.stat();
-    if (!stats.isFile())
-        return "nonregular-file";
-    if (stats.size > transcriptLimit)
-        return "file-too-large";
-    const chunks = [];
-    let total = 0;
-    while (total <= transcriptLimit) {
-        const buffer = Buffer.alloc(Math.min(65_536, transcriptLimit + 1 - total));
-        const { bytesRead } = await handle.read(buffer, 0, buffer.length, total);
-        if (bytesRead === 0)
-            break;
-        total += bytesRead;
-        chunks.push(buffer.subarray(0, bytesRead));
-    }
-    if (total > transcriptLimit || (await handle.stat()).size > transcriptLimit)
-        return "file-too-large";
-    return Buffer.concat(chunks, total);
-}
-function normalizeBytes(path, options) {
-    const { bytes } = options;
-    const digest = `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
-    let text;
-    try {
-        text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
-    }
-    catch {
-        const result = refused(path, "invalid-utf8");
-        return { ...result, coverage: { ...result.coverage, digest } };
-    }
-    const result = normalizeTranscript(text, digest);
-    return {
-        coverage: {
-            path,
-            status: result.status,
-            reason: result.reason,
-            digest,
-            ...(result.status === "included" ? { byteSize: bytes.length } : {}),
-            recordCount: result.records.length,
-            omissions: result.omissions,
-        },
-        records: result.records,
-        ...(options.prefixBytes !== undefined &&
-            options.prefixBytes >= 0 &&
-            options.prefixBytes < bytes.length
-            ? {
-                prefixDigest: `sha256:${createHash("sha256").update(bytes.subarray(0, options.prefixBytes)).digest("hex")}`,
-            }
-            : {}),
-    };
-}
-async function readSourceBytes(path, inspection) {
-    const handle = await open(path, constants.O_RDONLY |
-        constants.O_NONBLOCK |
-        (inspection === undefined ? constants.O_NOFOLLOW : 0x20000000));
-    try {
-        if (inspection !== undefined) {
-            verifyOpenSource(inspection, await handle.stat());
-            await verifyScanSource(inspection);
-        }
-        return await boundedBytes(handle);
-    }
-    finally {
-        await handle.close();
-    }
-}
-export async function readTranscript(path, options = {}) {
-    const { inspection } = options;
-    try {
-        if (inspection !== undefined)
-            await verifyScanSource(inspection);
-        const stats = await lstat(path);
-        if (stats.isSymbolicLink())
-            return refused(path, "symbolic-link");
-        if (!stats.isFile())
-            return refused(path, "nonregular-file");
-        const bytes = await readSourceBytes(path, inspection);
-        const result = typeof bytes === "string"
-            ? refused(path, bytes)
-            : normalizeBytes(path, { bytes, ...options });
-        return scannedResult(result, inspection);
-    }
-    catch (error) {
-        return readFailure(path, { error, inspection });
-    }
-}
-function scannedResult(result, inspection) {
-    if (inspection === undefined || result.coverage.status !== "too-large")
-        return result;
-    return { ...result, coverage: { ...result.coverage, status: "unsupported" } };
-}
-function readFailure(path, data) {
-    const { error, inspection } = data;
-    if (error instanceof Error && "code" in error && error.code === "ELOOP")
-        return refused(path, "symbolic-link");
-    return refused(path, inspection === undefined ? "source-unreadable" : sourceFailure(error));
-}

package/dist/src/chats/redact-import.d.ts

@@ -1,14 +0,0 @@
-import { type RedactionSettings } from "./redaction.ts";
-export interface RedactImportRequest {
-    readonly importId: string;
-    readonly workingDirectory: string;
-    readonly runtimeDirectory: string | undefined;
-    readonly settings: RedactionSettings;
-}
-export declare function redactImport(request: RedactImportRequest): Promise<{
-    code: string;
-    importId: string;
-    recordCount: number;
-    counts: import("./redaction.ts").RedactionCounts;
-    totalCount: number;
-}>;

package/dist/src/chats/redact-import.js

@@ -1,141 +0,0 @@
-import { openInput, readInput } from "./input-file.js";
-import { lstat, rm } from "node:fs/promises";
-import { join, resolve } from "node:path";
-import { createPrivateFile, validateDestination } from "./destination.js";
-import { parseCorpusRecord, validateCoverage } from "./corpus-input.js";
-import { emptyCounts, redactRecord, categories, } from "./redaction.js";
-async function absent(path) {
-    try {
-        await lstat(path);
-    }
-    catch (error) {
-        if (error instanceof Error && "code" in error && error.code === "ENOENT")
-            return;
-        throw error;
-    }
-    throw new Error("outputs-exist");
-}
-async function* corpusLines(handle) {
-    const buffer = Buffer.alloc(65_536);
-    const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
-    let pending = "";
-    let bytesRead = -1;
-    while (bytesRead !== 0) {
-        ({ bytesRead } = await handle.read({ buffer, position: null }));
-        pending += decoder.decode(buffer.subarray(0, bytesRead), {
-            stream: bytesRead !== 0,
-        });
-        let start = 0;
-        let end = pending.indexOf("\n");
-        while (end !== -1) {
-            yield pending.slice(start, end);
-            start = end + 1;
-            end = pending.indexOf("\n", start);
-        }
-        pending = pending.slice(start);
-    }
-    if (pending !== "")
-        yield pending;
-}
-async function redactCorpus(handles, settings) {
-    const counts = emptyCounts();
-    let recordCount = 0;
-    for await (const line of corpusLines(handles.input)) {
-        const result = redactRecord(parseCorpusRecord(line), settings);
-        for (const category of categories)
-            counts[category] += result.counts[category];
-        await handles.output.writeFile(JSON.stringify(result.record) + "\n");
-        recordCount++;
-    }
-    return {
-        counts,
-        recordCount,
-        totalCount: Object.values(counts).reduce((sum, count) => sum + count, 0),
-    };
-}
-async function writeRedactedCorpus(directory, settings) {
-    const path = join(directory, "redacted.jsonl");
-    let created = false;
-    try {
-        const input = await openInput(join(directory, "corpus.jsonl"));
-        try {
-            const output = await createPrivateFile(path);
-            created = true;
-            try {
-                return await redactCorpus({ input, output }, settings);
-            }
-            finally {
-                await output.close();
-            }
-        }
-        finally {
-            await input.close();
-        }
-    }
-    catch (error) {
-        if (created) {
-            await validateDestination(directory);
-            await rm(path, { force: true });
-        }
-        throw error;
-    }
-}
-async function writeManifest(directory, manifest) {
-    const path = join(directory, "redaction.json");
-    let created = false;
-    try {
-        const handle = await createPrivateFile(path);
-        created = true;
-        try {
-            await handle.writeFile(JSON.stringify(manifest) + "\n");
-        }
-        finally {
-            await handle.close();
-        }
-    }
-    catch (error) {
-        if (created) {
-            await validateDestination(directory);
-            await rm(path, { force: true });
-        }
-        throw error;
-    }
-}
-export async function redactImport(request) {
-    const empty = {
-        importId: request.importId,
-        recordCount: 0,
-        counts: emptyCounts(),
-        totalCount: 0,
-    };
-    if (process.platform !== "darwin")
-        return { ...empty, code: "platform-unsupported" };
-    if (request.runtimeDirectory === undefined)
-        return { ...empty, code: "runtime-directory-unknown" };
-    try {
-        if (!/^[a-f\d]{8}-[a-f\d]{4}-4[a-f\d]{3}-[89ab][a-f\d]{3}-[a-f\d]{12}$/.test(request.importId))
-            throw new Error("invalid-import");
-        const directory = resolve(request.workingDirectory, request.runtimeDirectory, "chats", request.importId);
-        await validateDestination(directory);
-        if (!(await lstat(directory)).isDirectory())
-            throw new Error("invalid-import");
-        await absent(join(directory, "redacted.jsonl"));
-        await absent(join(directory, "redaction.json"));
-        validateCoverage(await readInput(join(directory, "coverage.json")), request.importId);
-        const summary = await writeRedactedCorpus(directory, request.settings);
-        await writeManifest(directory, {
-            schemaVersion: 1,
-            importId: request.importId,
-            ruleVersion: 1,
-            ...summary,
-        });
-        return {
-            code: "redaction-complete",
-            importId: request.importId,
-            ...summary,
-        };
-    }
-    catch {
-        return { ...empty, code: "redaction-failed" };
-    }
-}

package/dist/src/chats/redaction.d.ts

@@ -1,41 +0,0 @@
-import type { MessageRecord } from "./normalize.ts";
-export interface RedactionSettings {
-    readonly home: string | null;
-    readonly privateTexts: readonly string[];
-}
-export declare const replacements: {
-    readonly privateKey: "[PRIVATE_KEY]";
-    readonly credentialUrl: "[CREDENTIAL_URL]";
-    readonly homePath: "[HOME_PATH]";
-    readonly token: "[TOKEN]";
-    readonly email: "[EMAIL]";
-    readonly privateText: "[PRIVATE_TEXT]";
-};
-export declare const categories: readonly ["privateKey", "credentialUrl", "homePath", "token", "email", "privateText"];
-export type Category = keyof typeof replacements;
-export type RedactionCounts = Record<Category, number>;
-export declare function emptyCounts(): RedactionCounts;
-export declare function redactRecord(record: MessageRecord, settings: RedactionSettings): {
-    record: {
-        text: string;
-        blocks: Record<string, unknown>[];
-        sourceId: string;
-        role: "user" | "assistant";
-        timestamp: string | null;
-        sessionId: string | null;
-        schemaVersion: 1;
-    } | {
-        text: string;
-        blocks: Record<string, unknown>[];
-        sourceId: string;
-        role: "user" | "assistant";
-        timestamp: string | null;
-        sessionId: string | null;
-        schemaVersion: 2;
-        uuid: string | null;
-        isSidechain: boolean | null;
-        parentUuid?: string | null;
-    };
-    counts: RedactionCounts;
-    totalCount: number;
-};