@bli-cockpit/cli 0.2.28 → 0.2.30

package/dist/adapters/raw-evidence-pack-store.js

@@ -0,0 +1,136 @@
+/**
+ * Where the bytes live on this disk.
+ *
+ * Staging first, promotion second (BLI-3066). A pack's id is derived from the
+ * content in it, so it cannot be known until collection is done: bytes land in
+ * a private `.staging-<pid>-<rand>` directory, and only then does the directory
+ * become its content-keyed name. Identical content on the next sync resolves to
+ * the identical directory instead of a 560th copy.
+ *
+ * Everything written here is mode `0600` inside a `0700` directory, on every
+ * platform except Windows, where the chmod is a no-op.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { makeManifest, manifestDescribesEntries, } from "./raw-evidence-manifest.js";
+import { writeRawEvidenceStagingState, } from "../raw-evidence-staging.js";
+/**
+ * Turn the staging directory into the content-keyed pack directory.
+ *
+ * Three outcomes, all of them named:
+ * - the pack does not exist yet: rename staging into place (`new`);
+ * - it exists and holds every file this pass staged: adopt it and delete the
+ *   staging copy (`reused`) — nothing is copied twice;
+ * - it exists but is missing files: fill the gaps from staging
+ *   (`restaged_incomplete`), because a half-written pack must not be trusted.
+ *
+ * Never deletes an existing pack directory wholesale: another pack's entries
+ * can point into it through the staged-object index.
+ */
+export async function promoteStagedPack(options) {
+    const evidenceDir = path.join(options.rawEvidenceRoot, options.packId);
+    const priorPackCount = await countPriorPacks(options.rawEvidenceRoot, options.workContextId, options.packId);
+    const existing = await fs.stat(evidenceDir).catch(() => null);
+    if (!existing?.isDirectory()) {
+        try {
+            await fs.rename(options.stagingDir, evidenceDir);
+            return {
+                state: "new",
+                evidenceDir,
+                priorPackCount,
+                refilledFileCount: 0,
+            };
+        }
+        catch {
+            // A concurrent sync can win the race to the same content-keyed name.
+            // Losing it is fine: the winner staged the identical bytes.
+        }
+    }
+    const refilledFileCount = await refillMissingPackFiles(evidenceDir, options.entries);
+    await fs
+        .rm(options.stagingDir, { recursive: true, force: true })
+        .catch(() => undefined);
+    return {
+        state: refilledFileCount > 0 ? "restaged_incomplete" : "reused",
+        evidenceDir,
+        priorPackCount,
+        refilledFileCount,
+    };
+}
+/** A file already there at the right size is left alone; anything else is recopied. */
+async function refillMissingPackFiles(evidenceDir, entries) {
+    let refilledFileCount = 0;
+    await ensurePrivateDir(path.join(evidenceDir, "files"));
+    for (const entry of entries) {
+        if (!entry.staged_in_pack)
+            continue;
+        const target = path.join(evidenceDir, "files", path.basename(entry.local_path));
+        const info = await fs.stat(target).catch(() => null);
+        if (info?.isFile() && info.size === entry.byte_size)
+            continue;
+        await fs.copyFile(entry.local_path, target).catch(() => undefined);
+        await chmodPrivate(target, 0o600);
+        refilledFileCount += 1;
+    }
+    return refilledFileCount;
+}
+async function countPriorPacks(rawEvidenceRoot, workContextId, packId) {
+    const entries = await fs
+        .readdir(rawEvidenceRoot, { withFileTypes: true })
+        .catch(() => []);
+    return entries.filter((entry) => entry.isDirectory() &&
+        entry.name !== packId &&
+        entry.name.startsWith(`${workContextId}-`)).length;
+}
+/**
+ * Write the manifest, or keep the one already in a reused pack.
+ *
+ * Byte-stability matters here: the manifest's object key is the only key that
+ * embeds the pack id, so rewriting it with a fresh `created_at` every sync
+ * would push different bytes at the same content-addressed key forever. A
+ * reused pack whose manifest already describes exactly these files keeps it.
+ */
+export async function stageManifest(options) {
+    if (options.reusePack) {
+        const existing = await fs.readFile(options.manifestPath).catch(() => null);
+        if (existing && manifestDescribesEntries(existing, options.entries)) {
+            return existing;
+        }
+    }
+    const manifestBytes = Buffer.from(`${JSON.stringify(makeManifest({
+        context: options.context,
+        packId: options.packId,
+        entries: options.entries,
+        skipped: options.skipped,
+        redacted: options.redacted,
+        reused: options.reused,
+    }), null, 2)}\n`, "utf8");
+    await fs.writeFile(options.manifestPath, manifestBytes, { mode: 0o600 });
+    await chmodPrivate(options.manifestPath, 0o600);
+    return manifestBytes;
+}
+/**
+ * Record which content hashes are staged where, so the next sync can adopt the
+ * copy instead of writing it again. Losing this file costs reuse and attempt
+ * counts, not evidence, so it must never fail a collection — but it must never
+ * be silent either.
+ */
+export async function persistStagingState(stateDir, staging, nowIso) {
+    staging.updated_at = nowIso;
+    await writeRawEvidenceStagingState(stateDir, staging).catch((error) => {
+        console.error("[raw-evidence] staging state write failed", JSON.stringify({
+            reason: "staging_state_write_failed",
+            detail: error instanceof Error ? error.name : typeof error,
+            staged_count: Object.keys(staging.staged).length,
+        }));
+    });
+}
+export async function ensurePrivateDir(dir) {
+    await fs.mkdir(dir, { recursive: true, mode: 0o700 });
+    await chmodPrivate(dir, 0o700);
+}
+export async function chmodPrivate(target, mode) {
+    if (process.platform === "win32")
+        return;
+    await fs.chmod(target, mode).catch(() => undefined);
+}

package/dist/adapters/raw-evidence-sanitize.js

@@ -0,0 +1,190 @@
+import { containsSecretLikeContent, redactSecretLikeContent, } from "@bli-cockpit/telemetry-core";
+import { sha256 } from "./raw-evidence-keys.js";
+export function sanitizeTextEvidenceForUpload(options) {
+    const originalBytes = options.originalBytes ?? Buffer.from(options.text, "utf8");
+    // Deliberately outside the try: the content guard is the cheap check that
+    // decides which masking strategy applies, and it is not the thing whose
+    // crash the stub below is for.
+    const secretLikeContent = containsSecretLikeContent(options.text);
+    try {
+        const redactionResult = (options.redact ?? redactSecretLikeContent)(options.text, {
+            appliedBy: "local_collector",
+            redactedFields: options.redactedFields,
+        });
+        if (options.secretLikeFileName) {
+            return maskFileWithSecretLikeName({
+                text: options.text,
+                originalBytes,
+                redactedFields: options.redactedFields,
+                redactionResult,
+            });
+        }
+        if (secretLikeContent) {
+            return maskSecretBearingLinesOnly({
+                text: options.text,
+                originalBytes,
+                redactedFields: options.redactedFields,
+                redactionResult,
+            });
+        }
+        if (redactionResult.redacted) {
+            return keepDeterministicRedaction({
+                text: options.text,
+                originalBytes,
+                redactedFields: options.redactedFields,
+                redactionResult,
+            });
+        }
+        return { status: "clean", bytes: originalBytes };
+    }
+    catch {
+        return stubForCrashedRedaction({
+            text: options.text,
+            originalBytes,
+            redactedFields: options.redactedFields,
+        });
+    }
+}
+/**
+ * The name itself (`.env.production`, `id_rsa`) is the signal, so nothing in
+ * this file is trusted: whatever the redactor left behind is re-checked, and a
+ * still-secret-looking result is replaced wholesale by one marker line.
+ */
+function maskFileWithSecretLikeName(input) {
+    const sanitizedText = input.redactionResult.redacted
+        ? input.redactionResult.text
+        : input.text;
+    const safeText = containsSecretLikeContent(sanitizedText)
+        ? "[REDACTED_LINE:secret_redaction_failed]\n"
+        : sanitizedText;
+    const sanitizedBytes = Buffer.from(safeText, "utf8");
+    return {
+        status: "redacted",
+        bytes: sanitizedBytes,
+        redaction: withRedactionContentMetadata(input.redactionResult.metadata ??
+            fallbackRedactionMetadata({
+                ruleId: "secret_like_file_name",
+                originalText: input.text,
+                redactedFields: input.redactedFields,
+                fullContentRedacted: false,
+            }), { originalBytes: input.originalBytes, sanitizedBytes }),
+        completenessLabel: "secret_like_name_masked",
+    };
+}
+/** Only the lines that matched are replaced; every other line uploads intact. */
+function maskSecretBearingLinesOnly(input) {
+    const redaction = input.redactionResult.metadata ??
+        fallbackRedactionMetadata({
+            ruleId: "secret_like_content_guard",
+            originalText: input.text,
+            redactedFields: input.redactedFields,
+        });
+    let maskedText = maskSecretBearingLines(input.text, redaction);
+    if (containsSecretLikeContent(maskedText)) {
+        maskedText = "[REDACTED_LINE:secret_redaction_failed]\n";
+    }
+    const sanitizedBytes = Buffer.from(maskedText, "utf8");
+    return {
+        status: "redacted",
+        bytes: sanitizedBytes,
+        redaction: withRedactionContentMetadata(redaction, {
+            originalBytes: input.originalBytes,
+            sanitizedBytes,
+        }),
+        completenessLabel: "secret_content_masked",
+    };
+}
+/** The redactor found and replaced something the content guard did not flag. */
+function keepDeterministicRedaction(input) {
+    const sanitizedBytes = Buffer.from(input.redactionResult.text, "utf8");
+    return {
+        status: "redacted",
+        bytes: sanitizedBytes,
+        redaction: withRedactionContentMetadata(input.redactionResult.metadata ??
+            fallbackRedactionMetadata({
+                ruleId: "secret_redaction_failed",
+                originalText: input.text,
+                redactedFields: input.redactedFields,
+            }), { originalBytes: input.originalBytes, sanitizedBytes }),
+        completenessLabel: "secret_content_masked",
+    };
+}
+function stubForCrashedRedaction(input) {
+    const sanitizedBytes = Buffer.from("[REDACTED_LINE:redaction_crashed_stubbed]\n", "utf8");
+    return {
+        status: "redacted",
+        bytes: sanitizedBytes,
+        redaction: withRedactionContentMetadata(fallbackRedactionMetadata({
+            ruleId: "redaction_crashed_stubbed",
+            originalText: input.text,
+            redactedFields: input.redactedFields,
+        }), { originalBytes: input.originalBytes, sanitizedBytes }),
+        completenessLabel: "redaction_crashed_stubbed",
+    };
+}
+function maskSecretBearingLines(text, redaction) {
+    if (redaction.redacted_ranges.length === 0) {
+        return "[REDACTED_LINE:secret_like_content_guard]\n";
+    }
+    const segments = text.match(/[^\n]*(?:\n|$)/gu)?.filter(Boolean) ?? [];
+    let offset = 0;
+    return segments
+        .map((segment) => {
+        const start = offset;
+        const end = offset + segment.length;
+        offset = end;
+        const matched = redaction.redacted_ranges.find((range) => range.start < end && start < range.end);
+        if (!matched)
+            return segment;
+        return `[REDACTED_LINE:${matched.rule_id}]${lineEndingOf(segment)}`;
+    })
+        .join("");
+}
+function lineEndingOf(segment) {
+    if (segment.endsWith("\r\n"))
+        return "\r\n";
+    if (segment.endsWith("\n"))
+        return "\n";
+    return "";
+}
+function fallbackRedactionMetadata(options) {
+    const fullContentRedacted = options.fullContentRedacted !== false;
+    return {
+        schema_version: "raw-evidence-redaction.v1",
+        status: "sanitized",
+        mode: "deterministic_text_replacement",
+        applied_by: ["local_collector"],
+        rule_counts: [
+            {
+                rule_id: options.ruleId,
+                match_count: 1,
+                redacted_char_count: fullContentRedacted
+                    ? options.originalText.length
+                    : 0,
+            },
+        ],
+        secret_like_match_count: 1,
+        redacted_fields: options.redactedFields,
+        redacted_ranges: fullContentRedacted && options.originalText.length > 0
+            ? [
+                {
+                    start: 0,
+                    end: options.originalText.length,
+                    rule_id: options.ruleId,
+                },
+            ]
+            : [],
+    };
+}
+function withRedactionContentMetadata(metadata, options) {
+    if (!metadata) {
+        throw new Error("redacted evidence is missing redaction metadata");
+    }
+    return {
+        ...metadata,
+        original_content_hash_sha256: sha256(options.originalBytes),
+        sanitized_content_hash_sha256: sha256(options.sanitizedBytes),
+        original_byte_size: options.originalBytes.byteLength,
+        sanitized_byte_size: options.sanitizedBytes.byteLength,
+    };
+}