@bli-cockpit/cli 0.2.48 → 0.2.50

package/dist/adapters/raw-evidence-transcript-file.js

@@ -0,0 +1,182 @@
+/**
+ * One transcript, end to end: read it, mask it, name it, pay for it, stage it.
+ *
+ * The Codex reader, the Claude reader and the sidecar path all funnel through
+ * `collectOneEvidenceFile`, so the order of the guards is the contract: delivery
+ * backoff BEFORE the read (a held object costs nothing this cycle), the size cap
+ * before the buffer, masking before the content hash, and the reuse check before
+ * the budget — a file already durable remotely must not spend a budget it does
+ * not need.
+ *
+ * Every refusal pushes a named skip into the gap ledger first, so a file this
+ * module declines is never silently absent.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { describeError } from "../health-detail.js";
+import { DELIVERY_BACKOFF_HOLDING_REASON, evidenceSourceKey, } from "../raw-evidence-staging.js";
+import { isSecretLikePath, sha256, shortHash, } from "./raw-evidence-keys.js";
+import { maskSecretsInTranscript, } from "./raw-evidence-sanitize.js";
+import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
+import { evidenceEntry } from "./raw-evidence-manifest.js";
+import { admitToBudget, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
+/**
+ * Reads, secret-guards, content-addresses, budget-checks, and copies one
+ * attributed transcript into the pack.
+ *
+ * Returns whether this file's *content* is accounted for, which is not the same
+ * as "was staged": a transcript whose bytes are already durable remotely
+ * returns true without staging anything, and its images are still worth
+ * collecting. Every false return has pushed a named skip first.
+ */
+export async function collectOneEvidenceFile(collection, options) {
+    const names = nameEvidenceFile(options);
+    const skip = (reason) => {
+        collection.skipped.push({
+            kind: options.kind,
+            label: names.evidenceLabel,
+            reason,
+        });
+        return false;
+    };
+    // Delivery backoff is checked BEFORE the file is read. An object whose commit
+    // has failed repeatedly costs nothing at all this cycle — no read, no hash,
+    // no copy, no request — and the hold is a named, retryable gap so a held
+    // session cannot make the sync look clean (BLI-3066).
+    if (collection.heldSources.has(names.sourceKey)) {
+        collection.deliveryHeldCount += 1;
+        skip(DELIVERY_BACKOFF_HOLDING_REASON);
+        logDeliveryBackoffHold(options.kind, names.sourceKey);
+        return false;
+    }
+    const read = await readEvidenceFileWithinCap(options.filePath, options.maxFileBytes);
+    if (read.status === "read_failed")
+        return skip("file_read_failed");
+    if (read.status === "too_large") {
+        markCapApplied(collection, options.kind, "max_file_bytes");
+        return skip("file_too_large");
+    }
+    const sanitized = maskSecretsInTranscript(collection, {
+        raw: read.bytes,
+        kind: options.kind,
+        evidenceLabel: names.evidenceLabel,
+        secretLikeFileName: names.secretLikeFileName,
+    });
+    const contentHash = sha256(sanitized.bytes);
+    if (collection.skipContentHashes.has(contentHash)) {
+        collection.reused.push({
+            kind: options.kind,
+            label: names.evidenceLabel,
+            content_hash_sha256: contentHash,
+            codex_session_id: options.sessionId,
+        });
+        return true;
+    }
+    const deferReason = admitToBudget(collection.budget, sanitized.bytes.byteLength);
+    if (deferReason) {
+        markBudgetCapApplied(collection, deferReason);
+        return skip(deferReason);
+    }
+    await stageOneTranscript(collection, {
+        ...options,
+        names,
+        sanitized,
+        contentHash,
+    });
+    return true;
+}
+function nameEvidenceFile(options) {
+    const fileName = path.basename(options.filePath);
+    const secretLikeFileName = isSecretLikePath(fileName);
+    return {
+        secretLikeFileName,
+        evidenceLabel: secretLikeFileName ? "[REDACTED_FILE_NAME]" : fileName,
+        packedFileName: secretLikeFileName ? "redacted-file.jsonl" : fileName,
+        sourceKey: evidenceSourceKey({
+            kind: options.kind,
+            sessionId: options.sessionId,
+            sourcePath: options.filePath,
+        }),
+    };
+}
+function logDeliveryBackoffHold(kind, sourceKey) {
+    console.error("[raw-evidence] delivery backoff holding source", JSON.stringify({
+        reason: DELIVERY_BACKOFF_HOLDING_REASON,
+        kind,
+        source_key: sourceKey,
+    }));
+}
+/** Put an accepted transcript's bytes on disk and add its manifest entry. */
+async function stageOneTranscript(collection, file) {
+    collection.index.value += 1;
+    const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-${shortHash(file.filePath)}-${file.names.packedFileName}`);
+    const staged = await stageEvidenceBytes(collection, {
+        contentHash: file.contentHash,
+        bytes: file.sanitized.bytes,
+        fileName: path.basename(relativePath),
+        kind: file.kind,
+        sourceKey: file.names.sourceKey,
+    });
+    collection.entries.push(evidenceEntry({
+        context: collection.context,
+        kind: file.kind,
+        packId: collection.packId,
+        localPath: staged.local_path,
+        relativePath,
+        mediaType: file.mediaType,
+        redactedSummary: file.sanitized.status === "redacted"
+            ? `${file.redactedSummary} Secret-like values were deterministically redacted before upload.`
+            : file.redactedSummary,
+        // Both branches carry a record now (BLI-3277), so "was anything
+        // replaced?" is the status, never the presence of `redaction`.
+        redaction: file.sanitized.redaction,
+        bytes: file.sanitized.bytes,
+        codexSessionId: file.sessionId,
+        contentAddress: file.contentAddress(file.contentHash.slice(0, 16)),
+        stagedInPack: staged.staged_in_pack,
+        sourceKey: file.names.sourceKey,
+    }));
+}
+/**
+ * Size is checked twice on purpose: once by `stat` so an oversized transcript
+ * is never buffered at all, and once on the bytes actually read, because a live
+ * session can grow between the two.
+ */
+async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
+    if (maxFileBytes) {
+        let stat;
+        try {
+            stat = await fs.stat(filePath);
+        }
+        catch (error) {
+            // The caller turns this into the `file_read_failed` skip label, which is
+            // the one label an operator can do nothing with. The file was discovered
+            // moments ago, so a failure here is a rotated session, a permission
+            // problem or a dead symlink — three different answers (BLI-3238).
+            console.error("[raw-evidence] evidence file could not be stat'd", JSON.stringify({
+                reason: "file_read_failed",
+                stage: "stat",
+                ...describeError(error),
+            }));
+            return { status: "read_failed" };
+        }
+        if (stat.size > maxFileBytes)
+            return { status: "too_large" };
+    }
+    let bytes;
+    try {
+        bytes = await fs.readFile(filePath);
+    }
+    catch (error) {
+        console.error("[raw-evidence] evidence file could not be read", JSON.stringify({
+            reason: "file_read_failed",
+            stage: "read",
+            ...describeError(error),
+        }));
+        return { status: "read_failed" };
+    }
+    if (maxFileBytes && bytes.byteLength > maxFileBytes) {
+        return { status: "too_large" };
+    }
+    return { status: "ok", bytes };
+}