@bli-cockpit/telemetry-core 0.1.42 → 0.1.44

package/dist/collector-heartbeat.d.ts

@@ -112,6 +112,10 @@ export declare const CollectorHeartbeatSchema: z.ZodObject<{
         hook_chars_p50: z.ZodOptional<z.ZodNumber>;
         hook_chars_p95: z.ZodOptional<z.ZodNumber>;
         hook_chars_samples: z.ZodOptional<z.ZodNumber>;
+        hook_stop_runs_24h: z.ZodOptional<z.ZodNumber>;
+        hook_stop_failed_24h: z.ZodOptional<z.ZodNumber>;
+        hook_session_start_runs_24h: z.ZodOptional<z.ZodNumber>;
+        hook_session_start_failed_24h: z.ZodOptional<z.ZodNumber>;
         hook_stats_reason: z.ZodOptional<z.ZodString>;
         hook_performance: z.ZodOptional<z.ZodObject<{
             schema_version: z.ZodLiteral<"memory-hook-performance.v1">;

package/dist/evidence-storage-encoding.d.ts

@@ -0,0 +1,8 @@
+/** In-object manifest. Legacy objects without this exact prefix remain raw. */
+export declare const EVIDENCE_ENCODING_MANIFEST: Buffer<ArrayBuffer>;
+export declare const MAX_DECODED_EVIDENCE_BYTES: number;
+export declare function encodeEvidenceBytes(bytes: Uint8Array): Buffer;
+export declare function decodeEvidenceBytes(bytes: Uint8Array): Buffer;
+export declare function encodeEvidenceBody(body: Buffer | ReadableStream<Uint8Array>): Buffer | ReadableStream<Uint8Array>;
+/** Lazy opening keeps read failures inside the caller's normal stream cleanup. */
+export declare function decodeEvidenceStream(body: ReadableStream<Uint8Array>): ReadableStream<Uint8Array>;

package/dist/evidence-storage-encoding.js

@@ -0,0 +1,97 @@
+import { gzipSync, gunzipSync } from "node:zlib";
+/** In-object manifest. Legacy objects without this exact prefix remain raw. */
+export const EVIDENCE_ENCODING_MANIFEST = Buffer.from('TOWER-EVIDENCE\n{"version":1,"encoding":"gzip"}\n');
+export const MAX_DECODED_EVIDENCE_BYTES = 512 * 1024 * 1024;
+export function encodeEvidenceBytes(bytes) {
+    return Buffer.concat([EVIDENCE_ENCODING_MANIFEST, gzipSync(bytes)]);
+}
+export function decodeEvidenceBytes(bytes) {
+    const buffer = Buffer.from(bytes);
+    return buffer.subarray(0, EVIDENCE_ENCODING_MANIFEST.length).equals(EVIDENCE_ENCODING_MANIFEST)
+        ? gunzipSync(buffer.subarray(EVIDENCE_ENCODING_MANIFEST.length), { maxOutputLength: MAX_DECODED_EVIDENCE_BYTES })
+        : buffer;
+}
+export function encodeEvidenceBody(body) {
+    if (Buffer.isBuffer(body))
+        return encodeEvidenceBytes(body);
+    const reader = body.pipeThrough(new CompressionStream("gzip")).getReader();
+    let prefixSent = false;
+    return new ReadableStream({
+        async pull(controller) {
+            if (!prefixSent) {
+                prefixSent = true;
+                controller.enqueue(EVIDENCE_ENCODING_MANIFEST);
+                return;
+            }
+            const next = await reader.read();
+            if (next.done)
+                controller.close();
+            else
+                controller.enqueue(next.value);
+        },
+        cancel(reason) { return reader.cancel(reason); },
+    });
+}
+/** Look ahead only as far as the manifest, then decode with bounded residency. */
+async function openDecodedStream(reader) {
+    let head = Buffer.alloc(0);
+    while (head.length < EVIDENCE_ENCODING_MANIFEST.length) {
+        const next = await reader.read();
+        if (next.done)
+            break;
+        head = Buffer.concat([head, next.value]);
+        const prefixLength = Math.min(head.length, EVIDENCE_ENCODING_MANIFEST.length);
+        if (!head.subarray(0, prefixLength).equals(EVIDENCE_ENCODING_MANIFEST.subarray(0, prefixLength)))
+            break;
+    }
+    const encoded = head.subarray(0, EVIDENCE_ENCODING_MANIFEST.length).equals(EVIDENCE_ENCODING_MANIFEST);
+    let first = encoded ? head.subarray(EVIDENCE_ENCODING_MANIFEST.length) : head;
+    const remainder = new ReadableStream({
+        async pull(controller) {
+            if (first !== null) {
+                const bytes = first;
+                first = null;
+                controller.enqueue(bytes);
+                return;
+            }
+            const next = await reader.read();
+            if (next.done)
+                controller.close();
+            else
+                controller.enqueue(next.value);
+        },
+        cancel(reason) { return reader.cancel(reason); },
+    });
+    if (!encoded)
+        return remainder;
+    let decodedBytes = 0;
+    return remainder.pipeThrough(new DecompressionStream("gzip")).pipeThrough(new TransformStream({
+        transform(chunk, controller) {
+            decodedBytes += chunk.byteLength;
+            if (decodedBytes > MAX_DECODED_EVIDENCE_BYTES)
+                throw new Error("evidence_decoded_size_exceeded");
+            controller.enqueue(chunk);
+        },
+    }));
+}
+/** Lazy opening keeps read failures inside the caller's normal stream cleanup. */
+export function decodeEvidenceStream(body) {
+    const sourceReader = body.getReader();
+    let reader;
+    return new ReadableStream({
+        async pull(controller) {
+            reader ??= (await openDecodedStream(sourceReader)).getReader();
+            const next = await reader.read();
+            if (next.done)
+                controller.close();
+            else
+                controller.enqueue(next.value);
+        },
+        async cancel(reason) {
+            if (reader)
+                await reader.cancel(reason);
+            else
+                await sourceReader.cancel(reason);
+        },
+    });
+}

package/dist/evidence-upload.d.ts

@@ -269,7 +269,11 @@ export declare const RawEvidenceUploadBeginResponseSchema: z.ZodObject<{
 }, z.core.$strict>;
 export type RawEvidenceUploadBeginResponse = z.infer<typeof RawEvidenceUploadBeginResponseSchema>;
 export declare const RawEvidenceUploadChunkRequestSchema: z.ZodObject<{
-    schema_version: z.ZodLiteral<"ambient-raw-evidence-upload-chunk.v1">;
+    schema_version: z.ZodEnum<{
+        "ambient-raw-evidence-upload-chunk.v1": "ambient-raw-evidence-upload-chunk.v1";
+        "ambient-raw-evidence-upload-chunk.v2": "ambient-raw-evidence-upload-chunk.v2";
+    }>;
+    encoding: z.ZodOptional<z.ZodLiteral<"gzip">>;
     generated_at: z.ZodString;
     provenance: z.ZodObject<{
         capture_source: z.ZodEnum<{

package/dist/evidence-upload.js

@@ -122,7 +122,8 @@ export const RawEvidenceUploadBeginResponseSchema = z
     .strict();
 export const RawEvidenceUploadChunkRequestSchema = z
     .object({
-    schema_version: z.literal("ambient-raw-evidence-upload-chunk.v1"),
+    schema_version: z.enum(["ambient-raw-evidence-upload-chunk.v1", "ambient-raw-evidence-upload-chunk.v2"]),
+    encoding: z.literal("gzip").optional(),
     generated_at: IsoDateTimeSchema,
     provenance: CaptureProvenanceSchema,
     upload_id: NonEmptyStringSchema,

package/dist/index.d.ts

@@ -14,6 +14,7 @@ export * from "./memory-hook-stats.js";
 export * from "./memory-extract-context.js";
 export * from "./memory-hook-performance.js";
 export * from "./memory-install-receipt.js";
+export * from "./outbound-guard.js";
 export * from "./paths.js";
 export * from "./privacy.js";
 export * from "./risk-flags.js";

package/dist/index.js

@@ -14,6 +14,7 @@ export * from "./memory-hook-stats.js";
 export * from "./memory-extract-context.js";
 export * from "./memory-hook-performance.js";
 export * from "./memory-install-receipt.js";
+export * from "./outbound-guard.js";
 export * from "./paths.js";
 export * from "./privacy.js";
 export * from "./risk-flags.js";

package/dist/memory-install-receipt.d.ts

@@ -120,6 +120,10 @@ export declare const MemoryInstallReceiptSchema: z.ZodObject<{
     hook_chars_p50: z.ZodOptional<z.ZodNumber>;
     hook_chars_p95: z.ZodOptional<z.ZodNumber>;
     hook_chars_samples: z.ZodOptional<z.ZodNumber>;
+    hook_stop_runs_24h: z.ZodOptional<z.ZodNumber>;
+    hook_stop_failed_24h: z.ZodOptional<z.ZodNumber>;
+    hook_session_start_runs_24h: z.ZodOptional<z.ZodNumber>;
+    hook_session_start_failed_24h: z.ZodOptional<z.ZodNumber>;
     hook_stats_reason: z.ZodOptional<z.ZodString>;
     hook_performance: z.ZodOptional<z.ZodObject<{
         schema_version: z.ZodLiteral<"memory-hook-performance.v1">;

package/dist/memory-install-receipt.js

@@ -180,6 +180,30 @@ export const MemoryInstallReceiptSchema = z
     hook_chars_p50: z.number().int().min(0).optional(),
     hook_chars_p95: z.number().int().min(0).optional(),
     hook_chars_samples: z.number().int().min(0).optional(),
+    /**
+     * THE OTHER TWO HOOKS (BLI-4057).
+     *
+     * Everything above counts the PROMPT hook only — the recall. Three hooks
+     * are installed on every machine and until this ticket the fleet heard
+     * from exactly one of them, so a SAVE hook that stopped firing (the one
+     * that writes memory at all) was invisible everywhere: the install
+     * receipt said `hooks ok`, the counts said 40 prompt recalls, and nothing
+     * anywhere said that not one memory had been written in ten days.
+     *
+     * `hook_stop_runs_24h` is the save hook; `hook_session_start_runs_24h` is
+     * the session-start recall. Each has its own failure count beside it,
+     * because a hook that runs and fails every time is a different fact from
+     * one that never ran — and the first is invisible in a run count alone.
+     *
+     * Optional like every count above, and ABSENT MEANS NOT REPORTED: a
+     * collector older than this ticket sends none, and a zero here would read
+     * as "the save hook ran zero times", which is exactly the alarm this
+     * ticket exists to raise. A reader may never substitute one for the other.
+     */
+    hook_stop_runs_24h: z.number().int().min(0).optional(),
+    hook_stop_failed_24h: z.number().int().min(0).optional(),
+    hook_session_start_runs_24h: z.number().int().min(0).optional(),
+    hook_session_start_failed_24h: z.number().int().min(0).optional(),
     /** Named when the counts are absent because the file could not be read. */
     hook_stats_reason: ReasonLabelSchema.optional(),
     /** Observed ordinary invocations, not legacy floors or a full host trace. */

package/dist/outbound-guard.d.ts

@@ -0,0 +1,77 @@
+/**
+ * The one seam every outbound write passes through (BLI-4086).
+ *
+ * The secret detector in this package was called at roughly fifty sites and
+ * every one of them was INGRESS — evidence upload, memory save, receipt
+ * parsing, notes intake, harvest extraction. Nothing ran on the way out. So
+ * the system could tell that a laptop had uploaded a credential and was blind
+ * when the assistant mailed one to somebody, and the assistant reaches every
+ * send path we own through MCP (`mail_send`, `msg_send`, `work_comment_issue`,
+ * `docs_update`, `notes_share`).
+ *
+ * **Why one function and not a guard per surface.** Five surfaces already
+ * disagreed about almost everything else — mail throws `MailDoorError`, msg
+ * throws `MsgDoorError`, Slack throws a bare `Error` — and a rule that has to
+ * be re-implemented five times is a rule that holds in four places within a
+ * quarter. The precedent is the mandatory-AI-SDK seam: make the single path
+ * the only path and let a census test fail when a sixth surface appears
+ * without it. `outbound-guard-census.test.ts` in the dashboard is that test.
+ *
+ * **It masks; it never refuses.** Consistent with the standing order that
+ * nothing is dropped — a refused send is a person's message deleted by a
+ * regex, and this detector's own header explains at length why coarse
+ * false positives are tolerable only because masking never costs the content.
+ * A masked send still sends, with `[REDACTED:<rule_id>]` where the value was.
+ *
+ * **It lives here, not in the dashboard**, because the worker's Linear mirror
+ * is an egress path too and cannot import from `apps/dashboard`. One choke
+ * point that stops at an app boundary is two choke points.
+ */
+/** The surfaces that can carry text out of Tower. Named so a log line can be grouped. */
+export declare const OUTBOUND_SURFACES: readonly ["mail", "msg", "slack", "work", "docs", "linear_mirror", "cal"];
+export type OutboundSurface = (typeof OUTBOUND_SURFACES)[number];
+/** What was masked, for the caller that wants to say so in its own log line. */
+export interface OutboundGuardReport {
+    /** True when at least one field changed. */
+    masked: boolean;
+    /** Field names that changed — names only, never values. */
+    maskedFields: string[];
+    /** Which rule caught what, and how much. The rule id IS the reason label. */
+    ruleCounts: {
+        rule_id: string;
+        match_count: number;
+        redacted_char_count: number;
+    }[];
+    /** Total matches across every field. */
+    matchCount: number;
+}
+export interface OutboundGuardResult<T> {
+    /** The same shape that came in, with secret-shaped spans replaced. */
+    fields: T;
+    report: OutboundGuardReport;
+}
+type GuardableFields = Record<string, string | null | undefined>;
+/**
+ * Masks every secret-shaped span in `fields` and logs once if anything changed.
+ *
+ * `null` and `undefined` pass through untouched, so a caller can hand over an
+ * optional field (`html`, a patch's absent `title`) without a branch at every
+ * call site — the branch is what gets forgotten.
+ *
+ * The log line carries the surface, the field NAMES, the rule ids and the
+ * counts. It deliberately carries no fragment of the matched text: a guard
+ * that prints what it caught has moved the secret from a message into a log,
+ * which is the failure this whole family exists to prevent
+ * (`~/.claude/rules/security.md`, "never log PII").
+ */
+export declare function guardOutboundFields<T extends GuardableFields>(surface: OutboundSurface, fields: T): OutboundGuardResult<T>;
+/**
+ * The single-field shorthand, for a writer that has exactly one body to guard.
+ *
+ * Prefer `guardOutboundFields` when a writer has two or more — it produces ONE
+ * log line for the send rather than one per field, and a reader counting
+ * masked sends should not have to know how many columns a surface happens to
+ * have.
+ */
+export declare function guardOutboundText(surface: OutboundSurface, field: string, text: string): string;
+export {};

package/dist/outbound-guard.js

@@ -0,0 +1,119 @@
+import { redactSecretLikeContent } from "./secret-guards-masking.js";
+/**
+ * The one seam every outbound write passes through (BLI-4086).
+ *
+ * The secret detector in this package was called at roughly fifty sites and
+ * every one of them was INGRESS — evidence upload, memory save, receipt
+ * parsing, notes intake, harvest extraction. Nothing ran on the way out. So
+ * the system could tell that a laptop had uploaded a credential and was blind
+ * when the assistant mailed one to somebody, and the assistant reaches every
+ * send path we own through MCP (`mail_send`, `msg_send`, `work_comment_issue`,
+ * `docs_update`, `notes_share`).
+ *
+ * **Why one function and not a guard per surface.** Five surfaces already
+ * disagreed about almost everything else — mail throws `MailDoorError`, msg
+ * throws `MsgDoorError`, Slack throws a bare `Error` — and a rule that has to
+ * be re-implemented five times is a rule that holds in four places within a
+ * quarter. The precedent is the mandatory-AI-SDK seam: make the single path
+ * the only path and let a census test fail when a sixth surface appears
+ * without it. `outbound-guard-census.test.ts` in the dashboard is that test.
+ *
+ * **It masks; it never refuses.** Consistent with the standing order that
+ * nothing is dropped — a refused send is a person's message deleted by a
+ * regex, and this detector's own header explains at length why coarse
+ * false positives are tolerable only because masking never costs the content.
+ * A masked send still sends, with `[REDACTED:<rule_id>]` where the value was.
+ *
+ * **It lives here, not in the dashboard**, because the worker's Linear mirror
+ * is an egress path too and cannot import from `apps/dashboard`. One choke
+ * point that stops at an app boundary is two choke points.
+ */
+/** The surfaces that can carry text out of Tower. Named so a log line can be grouped. */
+export const OUTBOUND_SURFACES = [
+    "mail",
+    "msg",
+    "slack",
+    "work",
+    "docs",
+    "linear_mirror",
+    // BLI-4167. A calendar write is the widest egress on this list: Google
+    // EMAILS every attendee the moment the event lands, so the summary,
+    // description and location leave for addresses Tower does not control and
+    // no later edit unsends them. `lib/cal/write.ts` is the writer.
+    "cal",
+];
+/**
+ * Masks every secret-shaped span in `fields` and logs once if anything changed.
+ *
+ * `null` and `undefined` pass through untouched, so a caller can hand over an
+ * optional field (`html`, a patch's absent `title`) without a branch at every
+ * call site — the branch is what gets forgotten.
+ *
+ * The log line carries the surface, the field NAMES, the rule ids and the
+ * counts. It deliberately carries no fragment of the matched text: a guard
+ * that prints what it caught has moved the secret from a message into a log,
+ * which is the failure this whole family exists to prevent
+ * (`~/.claude/rules/security.md`, "never log PII").
+ */
+export function guardOutboundFields(surface, fields) {
+    const guarded = {};
+    const maskedFields = [];
+    const ruleTotals = new Map();
+    let matchCount = 0;
+    for (const [name, value] of Object.entries(fields)) {
+        if (typeof value !== "string" || value.length === 0) {
+            guarded[name] = value;
+            continue;
+        }
+        // `server_commit` is the honest source: the mask is applied by the server
+        // at the moment it commits the send. The enum has no `outbound` member and
+        // adding one would relabel live redaction receipts that
+        // `secret-guards-lock.test.ts` pins — a schema change for a log line is a
+        // bad trade, and no outbound mask is persisted as a receipt anyway.
+        const result = redactSecretLikeContent(value, {
+            appliedBy: "server_commit",
+            redactedFields: [name],
+        });
+        guarded[name] = result.text;
+        if (!result.redacted)
+            continue;
+        maskedFields.push(name);
+        for (const count of result.metadata?.rule_counts ?? []) {
+            const existing = ruleTotals.get(count.rule_id) ?? {
+                rule_id: count.rule_id,
+                match_count: 0,
+                redacted_char_count: 0,
+            };
+            existing.match_count += count.match_count;
+            existing.redacted_char_count += count.redacted_char_count;
+            ruleTotals.set(count.rule_id, existing);
+            matchCount += count.match_count;
+        }
+    }
+    const report = {
+        masked: maskedFields.length > 0,
+        maskedFields,
+        ruleCounts: [...ruleTotals.values()],
+        matchCount,
+    };
+    if (report.masked) {
+        console.warn("[outbound guard] masked", JSON.stringify({
+            surface,
+            masked_fields: report.maskedFields,
+            rule_counts: report.ruleCounts,
+            secret_like_match_count: report.matchCount,
+        }));
+    }
+    return { fields: guarded, report };
+}
+/**
+ * The single-field shorthand, for a writer that has exactly one body to guard.
+ *
+ * Prefer `guardOutboundFields` when a writer has two or more — it produces ONE
+ * log line for the send rather than one per field, and a reader counting
+ * masked sends should not have to know how many columns a surface happens to
+ * have.
+ */
+export function guardOutboundText(surface, field, text) {
+    return guardOutboundFields(surface, { [field]: text }).fields[field];
+}

package/dist/secret-guards-masking.js

@@ -11,8 +11,42 @@ const SECRET_REDACTION_RULES = [
         secretGroup: 2,
     },
     {
+        // The body stops at `{`, and excludes nothing else (BLI-4168).
+        //
+        // `[\s\S]*?` used to sit here, which gave the rule no notion of where one
+        // record ends. Transcripts are JSONL, so a key whose `-----END-----` never
+        // arrived — a truncated write, a killed session — paired with the next
+        // `-----END-----` many records later and collapsed everything between into
+        // one `[REDACTED:private_key_block]`. That over-masks: nothing leaked, but
+        // a session's worth of legitimate records was destroyed silently, because
+        // a redaction looks exactly like the guard working.
+        //
+        // `{` is the JSONL record boundary: it is what opens the next record, and a
+        // PEM body is base64, so it cannot appear inside a key. This is JSONL-record
+        // scoping specifically, and JSONL is the ONLY shape it fixes. The same
+        // function also receives plain text — a raw `.pem`, a git diff, a log tail,
+        // YAML, CSV, a markdown note — where `{` is not a boundary at all. There it
+        // splits two ways. Where a `{` does happen to fall between the markers it
+        // merely ends the match early, leaving a narrow residue; that residue is the
+        // price of the scope. Where NO `{` falls between an unterminated BEGIN and a
+        // later stray END — the ordinary case for every one of those formats — the
+        // match is still unbounded and the original over-masking is FULLY ALIVE.
+        // Measured on six shapes, each an unterminated key, then a bystander record,
+        // then a stray END: JSONL preserved the bystander and the other five
+        // destroyed it. This rule does not fix that case. BLI-4216 carries it.
+        //
+        // It MUST stay a DENYLIST of that one character. An allowlist of the PEM
+        // alphabet was tried here and reverted the same day: one character outside
+        // the class anywhere between BEGIN and END — a `.`, a `[info] ` log prefix,
+        // an `N | ` gutter, an ANSI reset, a diff hunk header — makes the whole
+        // rule fail to match, and the body is then left in PLAINTEXT while
+        // `private_key_header` masks only the BEGIN line and
+        // `containsSecretLikeContent` reports the residue clean, so
+        // `secret_redaction_failed` never fires. That is a leak, and a leak is
+        // strictly worse than the over-masking this rule exists to fix. The
+        // leak-direction tests in `secret-guards.test.ts` pin it.
         ruleId: "private_key_block",
-        pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
+        pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[^{]*?-----END [A-Z ]*PRIVATE KEY-----/g,
     },
     {
         ruleId: "private_key_header",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/telemetry-core",
-  "version": "0.1.42",
+  "version": "0.1.44",
   "private": false,
   "type": "module",
   "main": "./dist/index.js",
@@ -18,6 +18,10 @@
     "./memory-experience": {
       "types": "./dist/memory-experience.d.ts",
       "import": "./dist/memory-experience.js"
+    },
+    "./evidence-storage-encoding": {
+      "types": "./dist/evidence-storage-encoding.d.ts",
+      "import": "./dist/evidence-storage-encoding.js"
     }
   },
   "publishConfig": {