@agentstrack/collector 0.2.1 → 0.4.1

package/dist/privacy/redact.js

@@ -1,3 +1,5 @@
+/** Org rules run on at most this many characters; built-ins see the whole value. */
+export const ORG_SUBJECT_CAP = 64 * 1024;
 export const BUILTIN_RULES = [
     { name: 'anthropic_key', pattern: /sk-ant-[A-Za-z0-9_-]{20,}/g, replacement: '[REDACTED:anthropic_key]' },
     { name: 'openai_key', pattern: /sk-(?:proj-)?[A-Za-z0-9_-]{20,}/g, replacement: '[REDACTED:openai_key]' },
@@ -18,30 +20,67 @@ export const BUILTIN_RULES = [
     { name: 'inline_password_flag', pattern: /(--password[= ]|(?<![\w-])-p)(?!\s)("[^"]*"|'[^']*'|\S+)/g, replacement: '$1[REDACTED]' },
     { name: 'generic_hex_secret', pattern: /\b[a-f0-9]{40,}\b/g, replacement: '[REDACTED:hex]' },
 ];
+/** The one kind reported for any rule that is not built in. */
+export const ORG_RULE_KIND = 'org_rule';
+const BUILTIN_NAMES = new Set(BUILTIN_RULES.map((rule) => rule.name));
+/**
+ * An org's own rule name can itself describe the shape of that org's secrets
+ * ("acme_prod_db_password"), which is their business and not ours to ship back
+ * to the server. Anything not built in therefore reports as one generic kind.
+ */
+const reportedKind = (name) => (BUILTIN_NAMES.has(name) ? name : ORG_RULE_KIND);
 export function redact(input, extraRules = []) {
     let text = input;
-    const redactions = [];
+    const counts = {};
     for (const rule of [...BUILTIN_RULES, ...extraRules]) {
+        // Org rules see only a bounded prefix; the untouched tail is re-appended.
+        const capped = rule.capSubject === true && text.length > ORG_SUBJECT_CAP;
+        const subject = capped ? text.slice(0, ORG_SUBJECT_CAP) : text;
         // Fresh lastIndex per call: these regexes are global and module-level, so
         // reusing them statefully across calls would skip matches.
         rule.pattern.lastIndex = 0;
-        if (!rule.pattern.test(text))
+        // `match` on a global regex returns every match, so the tally is a count of
+        // matches rather than of rules. Only the length is ever read.
+        const hits = subject.match(rule.pattern)?.length ?? 0;
+        if (hits === 0)
             continue;
         rule.pattern.lastIndex = 0;
-        text = text.replace(rule.pattern, rule.replacement);
-        redactions.push(rule.name);
+        const replaced = subject.replace(rule.pattern, rule.replacement);
+        text = capped ? replaced + text.slice(ORG_SUBJECT_CAP) : replaced;
+        const kind = reportedKind(rule.name);
+        counts[kind] = (counts[kind] ?? 0) + hits;
     }
-    return { text, redactions };
+    return { text, redactions: Object.keys(counts), counts };
 }
-/** Compiles org-supplied patterns, skipping any that do not compile. */
+/**
+ * An org rule is rejected before compilation when its source is longer than
+ * this, contains a nested quantifier, or uses a backreference — all shapes that
+ * invite catastrophic backtracking on V8's engine, which runs single-threaded
+ * on the daemon and would hang the whole collector.
+ */
+const MAX_ORG_PATTERN_LENGTH = 256;
+// A quantified group whose body holds a quantifier, an alternation or an
+// interval — directly or in one nested group. A heuristic: it rejects the
+// common catastrophic shapes, not every regex that can blow up.
+const NESTED_QUANTIFIER = /\((?:[^()]|\([^()]*\))*(?:[+*?|{]|\([^()]*[+*?|{][^()]*\))(?:[^()]|\([^()]*\))*\)[+*?{]/;
+const BACKREFERENCE = /\\[1-9]|\\k</;
+/** Compiles org-supplied patterns, skipping any unsafe or malformed rule. */
 export function compileRules(rules) {
     const compiled = [];
     for (const [index, rule] of rules.entries()) {
+        if (rule.pattern.length > MAX_ORG_PATTERN_LENGTH ||
+            NESTED_QUANTIFIER.test(rule.pattern) ||
+            BACKREFERENCE.test(rule.pattern)) {
+            // Same policy as a malformed rule: log-and-skip rather than stop the
+            // collector. (Logging happens at the call site, which holds the logger.)
+            continue;
+        }
         try {
             compiled.push({
                 name: `org_rule_${index}`,
                 pattern: new RegExp(rule.pattern, 'g'),
                 replacement: rule.replacement,
+                capSubject: true,
             });
         }
         catch {

package/dist/privacy/redact.js.map

@@ -1 +1 @@
-{"version":3,"file":"redact.js","sourceRoot":"","sources":["../../src/privacy/redact.ts"],"names":[],"mappings":"AAgBA,MAAM,CAAC,MAAM,aAAa,GAAoB;IAC5C,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,4BAA4B,EAAE,WAAW,EAAE,0BAA0B,EAAE;IACzG,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,kCAAkC,EAAE,WAAW,EAAE,uBAAuB,EAAE;IACzG,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,6BAA6B,EAAE,WAAW,EAAE,yBAAyB,EAAE;IACxG,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,+BAA+B,EAAE,WAAW,EAAE,uBAAuB,EAAE;IACtG,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,+BAA+B,EAAE,WAAW,EAAE,wBAAwB,EAAE;IACxG,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,uCAAuC,EAAE,WAAW,EAAE,uBAAuB,EAAE;IAC9G,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,gCAAgC,EAAE,WAAW,EAAE,2BAA2B,EAAE;IAC/G,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,4BAA4B,EAAE,WAAW,EAAE,2BAA2B,EAAE;IAC3G,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,qDAAqD,EAAE,WAAW,EAAE,4BAA4B,EAAE;IACtI,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,2EAA2E,EAAE,WAAW,EAAE,wBAAwB,EAAE;IACpJ,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,oEAAoE,EAAE,WAAW,EAAE,gBAAgB,EAAE;IAC7H,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,yCAAyC,EAAE,WAAW,EAAE,mBAAmB,EAAE;IAC/G,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,gDAAgD,EAAE,WAAW,EAAE,eAAe,EAAE;IACnH,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,uHAAuH,EAAE,WAAW,EAAE,eAAe,EAAE;IAC1L,8EAA8E;IAC9E,yEAAyE;IACzE,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,2DAA2D,EAAE,WAAW,EAAE,cAAc,EAAE;IACnI,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,gBAAgB,EAAE;CAC7F,CAAC;AAQF,MAAM,UAAU,MAAM,CAAC,KAAa,EAAE,aAA8B,EAAE;IACpE,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,aAAa,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;QACrD,0EAA0E;QAC1E,2DAA2D;QAC3D,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QACvC,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACpD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;AAC9B,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,YAAY,CAC1B,KAAiD;IAEjD,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,YAAY,KAAK,EAAE;gBACzB,OAAO,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC;gBACtC,WAAW,EAAE,IAAI,CAAC,WAAW;aAC9B,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,qEAAqE;QACvE,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,WAAW,CAAC,WAAmB;IAC7C,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACvD,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC;AACzC,CAAC"}
+{"version":3,"file":"redact.js","sourceRoot":"","sources":["../../src/privacy/redact.ts"],"names":[],"mappings":"AAuBA,oFAAoF;AACpF,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,GAAG,IAAI,CAAC;AAEzC,MAAM,CAAC,MAAM,aAAa,GAAoB;IAC5C,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,4BAA4B,EAAE,WAAW,EAAE,0BAA0B,EAAE;IACzG,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,kCAAkC,EAAE,WAAW,EAAE,uBAAuB,EAAE;IACzG,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,6BAA6B,EAAE,WAAW,EAAE,yBAAyB,EAAE;IACxG,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,+BAA+B,EAAE,WAAW,EAAE,uBAAuB,EAAE;IACtG,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,+BAA+B,EAAE,WAAW,EAAE,wBAAwB,EAAE;IACxG,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,uCAAuC,EAAE,WAAW,EAAE,uBAAuB,EAAE;IAC9G,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,gCAAgC,EAAE,WAAW,EAAE,2BAA2B,EAAE;IAC/G,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,4BAA4B,EAAE,WAAW,EAAE,2BAA2B,EAAE;IAC3G,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,qDAAqD,EAAE,WAAW,EAAE,4BAA4B,EAAE;IACtI,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,2EAA2E,EAAE,WAAW,EAAE,wBAAwB,EAAE;IACpJ,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,oEAAoE,EAAE,WAAW,EAAE,gBAAgB,EAAE;IAC7H,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,yCAAyC,EAAE,WAAW,EAAE,mBAAmB,EAAE;IAC/G,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,gDAAgD,EAAE,WAAW,EAAE,eAAe,EAAE;IACnH,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,uHAAuH,EAAE,WAAW,EAAE,eAAe,EAAE;IAC1L,8EAA8E;IAC9E,yEAAyE;IACzE,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,2DAA2D,EAAE,WAAW,EAAE,cAAc,EAAE;IACnI,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,gBAAgB,EAAE;CAC7F,CAAC;AAUF,+DAA+D;AAC/D,MAAM,CAAC,MAAM,aAAa,GAAG,UAAU,CAAC;AAExC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAEtE;;;;GAIG;AACH,MAAM,YAAY,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;AAEhG,MAAM,UAAU,MAAM,CAAC,KAAa,EAAE,aAA8B,EAAE;IACpE,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,MAAM,MAAM,GAA2B,EAAE,CAAC;IAE1C,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,aAAa,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;QACrD,0EAA0E;QAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC;QACzE,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/D,0EAA0E;QAC1E,2DAA2D;QAC3D,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QAC3B,4EAA4E;QAC5E,8DAA8D;QAC9D,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;QACtD,IAAI,IAAI,KAAK,CAAC;YAAE,SAAS;QACzB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QAC3B,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACjE,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QAClE,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;IAC5C,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;AAC3D,CAAC;AAED;;;;;GAKG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,yEAAyE;AACzE,0EAA0E;AAC1E,gEAAgE;AAChE,MAAM,iBAAiB,GACrB,yFAAyF,CAAC;AAC5F,MAAM,aAAa,GAAG,cAAc,CAAC;AAErC,6EAA6E;AAC7E,MAAM,UAAU,YAAY,CAC1B,KAAiD;IAEjD,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,IACE,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,sBAAsB;YAC5C,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;YACpC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAChC,CAAC;YACD,qEAAqE;YACrE,yEAAyE;YACzE,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,YAAY,KAAK,EAAE;gBACzB,OAAO,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC;gBACtC,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,UAAU,EAAE,IAAI;aACjB,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,qEAAqE;QACvE,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,WAAW,CAAC,WAAmB;IAC7C,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACvD,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC;AACzC,CAAC"}

package/dist/queue/event-id.d.ts

@@ -0,0 +1,9 @@
+/**
+ * A stable event_id from a stable seed, so re-reading a transcript (rotated
+ * inode, purged spool, second collector on the same files) produces the same
+ * id and the server's dedupe absorbs it instead of double-counting.
+ *
+ * sha256, truncated and stamped as a UUID: version nibble 8 (RFC 9562
+ * "custom"), variant bits 10xx — the shape the server's `uuid()` check wants.
+ */
+export declare function deterministicEventId(seed: string): string;

package/dist/queue/event-id.js

@@ -0,0 +1,15 @@
+import { createHash } from 'node:crypto';
+/**
+ * A stable event_id from a stable seed, so re-reading a transcript (rotated
+ * inode, purged spool, second collector on the same files) produces the same
+ * id and the server's dedupe absorbs it instead of double-counting.
+ *
+ * sha256, truncated and stamped as a UUID: version nibble 8 (RFC 9562
+ * "custom"), variant bits 10xx — the shape the server's `uuid()` check wants.
+ */
+export function deterministicEventId(seed) {
+    const hex = createHash('sha256').update(seed).digest('hex');
+    const variant = ((parseInt(hex.charAt(16), 16) & 0x3) | 0x8).toString(16);
+    return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-8${hex.slice(13, 16)}-${variant}${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
+}
+//# sourceMappingURL=event-id.js.map

package/dist/queue/event-id.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"event-id.js","sourceRoot":"","sources":["../../src/queue/event-id.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC1E,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;AAC5H,CAAC"}

package/dist/queue/spool.d.ts

@@ -1,4 +1,9 @@
 import type { EventEnvelope } from '../schema.js';
+export interface Checkpoint {
+    inode: string;
+    offset: number;
+    size: number;
+}
 /**
  * Durable local spool.
  *
@@ -12,8 +17,25 @@ import type { EventEnvelope } from '../schema.js';
  */
 export declare class Spool {
     private readonly db;
+    private readonly stmt;
+    /**
+     * Every checkpoint, in memory. A steady-state scan touches every tracked
+     * file every 5s and almost none of them changed; answering that from a Map
+     * costs nothing, and only a change is written through.
+     */
+    private readonly checkpoints;
     constructor(path: string);
+    private loadCheckpoints;
     private migrate;
+    /**
+     * A drained backfill leaves the file at its high-water mark, nearly all of
+     * it free pages (a 206MB spool holding 78 events was observed). VACUUM once
+     * at open when the freelist is both large and the majority of the file —
+     * never on a small db, where it is pure cost.
+     */
+    private vacuumIfBloated;
+    /** Runs `fn` atomically. A throw rolls back everything written inside it. */
+    transaction<T>(fn: () => T): T;
     enqueue(events: EventEnvelope[]): number;
     /** Oldest-first so a backlog drains in the order it happened. */
     peek(limit: number): {
@@ -33,14 +55,11 @@ export declare class Spool {
      */
     fail(eventIds: string[], maxAttempts: number): number;
     depth(): number;
-    getCheckpoint(path: string): {
-        inode: string;
-        offset: number;
-        size: number;
-    } | null;
+    getCheckpoint(path: string): Checkpoint | null;
     setCheckpoint(path: string, inode: string, offset: number, size: number): void;
     getMeta(key: string): string | null;
     setMeta(key: string, value: string): void;
+    deleteMeta(key: string): void;
     /** Stable per-install id, generated once. */
     installId(): string;
     close(): void;

package/dist/queue/spool.js

@@ -15,22 +15,54 @@ import { randomUUID } from 'node:crypto';
  */
 export class Spool {
     db;
+    stmt;
+    /**
+     * Every checkpoint, in memory. A steady-state scan touches every tracked
+     * file every 5s and almost none of them changed; answering that from a Map
+     * costs nothing, and only a change is written through.
+     */
+    checkpoints = new Map();
     constructor(path) {
         mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
         this.db = new Database(path);
-        // The spool holds un-uploaded telemetry; default umask makes it world-readable.
+        // WAL so a crash mid-write cannot corrupt the queue.
+        this.db.pragma('journal_mode = WAL');
+        this.db.pragma('synchronous = NORMAL');
+        // The WAL is otherwise only trimmed on close; a backfill leaves it at its
+        // high-water mark forever.
+        this.db.pragma('journal_size_limit = 8388608');
+        // The spool holds un-uploaded telemetry; default umask makes it
+        // world-readable. WAL and SHM exist only after the pragma above.
         for (const suffix of ['', '-wal', '-shm']) {
             try {
                 chmodSync(`${path}${suffix}`, 0o600);
             }
             catch {
-                // WAL/SHM may not exist yet; the main db is the one that matters.
+                // The main db is the one that matters.
             }
         }
-        // WAL so a crash mid-write cannot corrupt the queue.
-        this.db.pragma('journal_mode = WAL');
-        this.db.pragma('synchronous = NORMAL');
         this.migrate();
+        this.vacuumIfBloated();
+        this.stmt = {
+            insert: this.db.prepare('INSERT OR IGNORE INTO events (event_id, body, created_at) VALUES (?, ?, ?)'),
+            peek: this.db.prepare('SELECT event_id, body FROM events ORDER BY created_at ASC, rowid ASC LIMIT ?'),
+            del: this.db.prepare('DELETE FROM events WHERE event_id = ?'),
+            bump: this.db.prepare('UPDATE events SET attempts = attempts + 1 WHERE event_id = ?'),
+            drop: this.db.prepare('DELETE FROM events WHERE event_id = ? AND attempts >= ?'),
+            count: this.db.prepare('SELECT COUNT(*) AS n FROM events'),
+            checkpoint: this.db.prepare(`INSERT INTO checkpoints (path, inode, offset, size) VALUES (?, ?, ?, ?)
+         ON CONFLICT(path) DO UPDATE SET inode = excluded.inode, offset = excluded.offset, size = excluded.size`),
+            getMeta: this.db.prepare('SELECT value FROM meta WHERE key = ?'),
+            setMeta: this.db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value'),
+            delMeta: this.db.prepare('DELETE FROM meta WHERE key = ?'),
+        };
+        this.loadCheckpoints();
+    }
+    loadCheckpoints() {
+        const rows = this.db.prepare('SELECT path, inode, offset, size FROM checkpoints').all();
+        this.checkpoints.clear();
+        for (const { path: p, ...cp } of rows)
+            this.checkpoints.set(p, cp);
     }
     migrate() {
         this.db.exec(`
@@ -55,24 +87,50 @@ export class Spool {
       );
     `);
     }
+    /**
+     * A drained backfill leaves the file at its high-water mark, nearly all of
+     * it free pages (a 206MB spool holding 78 events was observed). VACUUM once
+     * at open when the freelist is both large and the majority of the file —
+     * never on a small db, where it is pure cost.
+     */
+    vacuumIfBloated() {
+        const free = Number(this.db.pragma('freelist_count', { simple: true }));
+        const pages = Number(this.db.pragma('page_count', { simple: true }));
+        if (free > 2048 && free > pages / 2) {
+            try {
+                this.db.exec('VACUUM');
+            }
+            catch {
+                // Out of disk or a concurrent reader: the queue still works, just fat.
+            }
+        }
+    }
+    /** Runs `fn` atomically. A throw rolls back everything written inside it. */
+    transaction(fn) {
+        try {
+            return this.db.transaction(fn)();
+        }
+        catch (error) {
+            // setCheckpoint() updated the cache before the rollback (or a failed
+            // COMMIT — SQLITE_FULL) undid the row; bring it back in line with disk.
+            this.loadCheckpoints();
+            throw error;
+        }
+    }
     enqueue(events) {
         if (events.length === 0)
             return 0;
-        const stmt = this.db.prepare('INSERT OR IGNORE INTO events (event_id, body, created_at) VALUES (?, ?, ?)');
         const now = Date.now();
-        const insertAll = this.db.transaction((batch) => {
+        return this.transaction(() => {
             let written = 0;
-            for (const event of batch)
-                written += stmt.run(event.event_id, JSON.stringify(event), now).changes;
+            for (const event of events)
+                written += this.stmt.insert.run(event.event_id, JSON.stringify(event), now).changes;
             return written;
         });
-        return insertAll(events);
     }
     /** Oldest-first so a backlog drains in the order it happened. */
     peek(limit) {
-        const rows = this.db
-            .prepare('SELECT event_id, body FROM events ORDER BY created_at ASC, rowid ASC LIMIT ?')
-            .all(limit);
+        const rows = this.stmt.peek.all(limit);
         return rows.flatMap((row) => {
             try {
                 return [{ eventId: row.event_id, event: JSON.parse(row.body) }];
@@ -87,8 +145,7 @@ export class Spool {
     ack(eventIds) {
         if (eventIds.length === 0)
             return;
-        const stmt = this.db.prepare('DELETE FROM events WHERE event_id = ?');
-        this.db.transaction((ids) => ids.forEach((id) => stmt.run(id)))(eventIds);
+        this.transaction(() => eventIds.forEach((id) => this.stmt.del.run(id)));
     }
     /**
      * Records a failed attempt so poison events can be dropped eventually.
@@ -104,37 +161,36 @@ export class Spool {
         if (eventIds.length === 0)
             return 0;
         const limit = Math.max(1, maxAttempts);
-        const bump = this.db.prepare('UPDATE events SET attempts = attempts + 1 WHERE event_id = ?');
-        const drop = this.db.prepare('DELETE FROM events WHERE event_id = ? AND attempts >= ?');
-        return this.db.transaction((ids) => {
+        return this.transaction(() => {
             let dropped = 0;
-            for (const id of ids) {
-                bump.run(id);
-                dropped += drop.run(id, limit).changes;
+            for (const id of eventIds) {
+                this.stmt.bump.run(id);
+                dropped += this.stmt.drop.run(id, limit).changes;
             }
             return dropped;
-        })(eventIds);
+        });
     }
     depth() {
-        return this.db.prepare('SELECT COUNT(*) AS n FROM events').get().n;
+        return this.stmt.count.get().n;
     }
     getCheckpoint(path) {
-        return (this.db.prepare('SELECT inode, offset, size FROM checkpoints WHERE path = ?').get(path) ?? null);
+        return this.checkpoints.get(path) ?? null;
     }
     setCheckpoint(path, inode, offset, size) {
-        this.db
-            .prepare(`INSERT INTO checkpoints (path, inode, offset, size) VALUES (?, ?, ?, ?)
-         ON CONFLICT(path) DO UPDATE SET inode = excluded.inode, offset = excluded.offset, size = excluded.size`)
-            .run(path, inode, offset, size);
+        const current = this.checkpoints.get(path);
+        if (current && current.inode === inode && current.offset === offset && current.size === size)
+            return;
+        this.stmt.checkpoint.run(path, inode, offset, size);
+        this.checkpoints.set(path, { inode, offset, size });
     }
     getMeta(key) {
-        return (this.db.prepare('SELECT value FROM meta WHERE key = ?').get(key)?.value ??
-            null);
+        return this.stmt.getMeta.get(key)?.value ?? null;
     }
     setMeta(key, value) {
-        this.db
-            .prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value')
-            .run(key, value);
+        this.stmt.setMeta.run(key, value);
+    }
+    deleteMeta(key) {
+        this.stmt.delMeta.run(key);
     }
     /** Stable per-install id, generated once. */
     installId() {

package/dist/queue/spool.js.map

@@ -1 +1 @@
-{"version":3,"file":"spool.js","sourceRoot":"","sources":["../../src/queue/spool.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC;;;;;;;;;;GAUG;AACH,MAAM,OAAO,KAAK;IACC,EAAE,CAAoB;IAEvC,YAAY,IAAY;QACtB,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC7B,gFAAgF;QAChF,KAAK,MAAM,MAAM,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACH,SAAS,CAAC,GAAG,IAAI,GAAG,MAAM,EAAE,EAAE,KAAK,CAAC,CAAC;YACvC,CAAC;YAAC,MAAM,CAAC;gBACP,kEAAkE;YACpE,CAAC;QACH,CAAC;QACD,qDAAqD;QACrD,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,EAAE,CAAC;IACjB,CAAC;IAEO,OAAO;QACb,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;KAoBZ,CAAC,CAAC;IACL,CAAC;IAED,OAAO,CAAC,MAAuB;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAC1B,4EAA4E,CAC7E,CAAC;QACF,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,KAAsB,EAAE,EAAE;YAC/D,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,KAAK,MAAM,KAAK,IAAI,KAAK;gBAAE,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC;YACnG,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC,CAAC;QACH,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,iEAAiE;IACjE,IAAI,CAAC,KAAa;QAChB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CAAC,8EAA8E,CAAC;aACvF,GAAG,CAAC,KAAK,CAAyC,CAAC;QAEtD,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAC1B,IAAI,CAAC;gBACH,OAAO,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAkB,EAAE,CAAC,CAAC;YACnF,CAAC;YAAC,MAAM,CAAC;gBACP,0DAA0D;gBAC1D,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACzB,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,GAAG,CAAC,QAAkB;QACpB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC;QACtE,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,GAAa,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACtF,CAAC;IAED;;;;;;;;;OASG;IACH,IAAI,CAAC,QAAkB,EAAE,WAAmB;QAC1C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QAEvC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,8DAA8D,CAAC,CAAC;QAC7F,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yDAAyD,CAAC,CAAC;QAExF,OAAO,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,GAAa,EAAE,EAAE;YAC3C,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;gBACrB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACb,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC;YACzC,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACf,CAAC;IAED,KAAK;QACH,OAAQ,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;IACxF,CAAC;IAED,aAAa,CAAC,IAAY;QACxB,OAAO,CACJ,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,4DAA4D,CAAC,CAAC,GAAG,CAAC,IAAI,CAEzE,IAAI,IAAI,CACvB,CAAC;IACJ,CAAC;IAED,aAAa,CAAC,IAAY,EAAE,KAAa,EAAE,MAAc,EAAE,IAAY;QACrE,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;gHACwG,CACzG;aACA,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,OAAO,CAAC,GAAW;QACjB,OAAO,CACJ,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAmC,EAAE,KAAK;YAC1G,IAAI,CACL,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,GAAW,EAAE,KAAa;QAChC,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC,mGAAmG,CAAC;aAC5G,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACrB,CAAC;IAED,6CAA6C;IAC7C,SAAS;QACP,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACpC,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,EAAE,GAAG,UAAU,EAAE,CAAC;YAClB,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF"}
+{"version":3,"file":"spool.js","sourceRoot":"","sources":["../../src/queue/spool.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AASzC;;;;;;;;;;GAUG;AACH,MAAM,OAAO,KAAK;IACC,EAAE,CAAoB;IACtB,IAAI,CAWnB;IACF;;;;OAIG;IACc,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;IAE7D,YAAY,IAAY;QACtB,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC7B,qDAAqD;QACrD,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACvC,0EAA0E;QAC1E,2BAA2B;QAC3B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,8BAA8B,CAAC,CAAC;QAC/C,gEAAgE;QAChE,iEAAiE;QACjE,KAAK,MAAM,MAAM,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACH,SAAS,CAAC,GAAG,IAAI,GAAG,MAAM,EAAE,EAAE,KAAK,CAAC,CAAC;YACvC,CAAC;YAAC,MAAM,CAAC;gBACP,uCAAuC;YACzC,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC,eAAe,EAAE,CAAC;QAEvB,IAAI,CAAC,IAAI,GAAG;YACV,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,4EAA4E,CAAC;YACrG,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,8EAA8E,CAAC;YACrG,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,uCAAuC,CAAC;YAC7D,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,8DAA8D,CAAC;YACrF,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yDAAyD,CAAC;YAChF,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kCAAkC,CAAC;YAC1D,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CACzB;gHACwG,CACzG;YACD,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC;YAChE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CACtB,mGAAmG,CACpG;YACD,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gCAAgC,CAAC;SAC3D,CAAC;QAEF,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC;IAEO,eAAe;QACrB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,mDAAmD,CAAC,CAAC,GAAG,EAEjF,CAAC;QACL,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,KAAK,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,IAAI,IAAI;YAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACrE,CAAC;IAEO,OAAO;QACb,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;KAoBZ,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACK,eAAe;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,gBAAgB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACrE,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACzB,CAAC;YAAC,MAAM,CAAC;gBACP,uEAAuE;YACzE,CAAC;QACH,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,WAAW,CAAI,EAAW;QACxB,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,qEAAqE;YACrE,wEAAwE;YACxE,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,CAAC,MAAuB;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAClC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE;YAC3B,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,KAAK,MAAM,KAAK,IAAI,MAAM;gBAAE,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC;YAChH,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,iEAAiE;IACjE,IAAI,CAAC,KAAa;QAChB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAyC,CAAC;QAE/E,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAC1B,IAAI,CAAC;gBACH,OAAO,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAkB,EAAE,CAAC,CAAC;YACnF,CAAC;YAAC,MAAM,CAAC;gBACP,0DAA0D;gBAC1D,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACzB,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,GAAG,CAAC,QAAkB;QACpB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAClC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED;;;;;;;;;OASG;IACH,IAAI,CAAC,QAAkB,EAAE,WAAmB;QAC1C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QAEvC,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE;YAC3B,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;gBAC1B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACvB,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC;YACnD,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK;QACH,OAAQ,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,aAAa,CAAC,IAAY;QACxB,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;IAC5C,CAAC;IAED,aAAa,CAAC,IAAY,EAAE,KAAa,EAAE,MAAc,EAAE,IAAY;QACrE,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,OAAO,CAAC,IAAI,KAAK,IAAI;YAAE,OAAO;QACrG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,CAAC,GAAW;QACjB,OAAQ,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAmC,EAAE,KAAK,IAAI,IAAI,CAAC;IACtF,CAAC;IAED,OAAO,CAAC,GAAW,EAAE,KAAa;QAChC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACpC,CAAC;IAED,UAAU,CAAC,GAAW;QACpB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,6CAA6C;IAC7C,SAAS;QACP,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACpC,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,EAAE,GAAG,UAAU,EAAE,CAAC;YAClB,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF"}

package/dist/queue/tailer.d.ts

@@ -2,8 +2,11 @@ import type { Spool } from './spool.js';
 /**
  * Resumable line tailer.
  *
- * Checkpoints (path, inode, offset) after every read so a restart continues
- * exactly where it stopped. Three failure modes it must survive:
+ * Reads in fixed-size chunks and hands every complete line to `onLines`; the
+ * checkpoint (path, inode, offset) for a chunk is written in the SAME
+ * transaction as whatever `onLines` spooled, so a throw or a crash between
+ * "read" and "queued" re-reads those lines instead of losing them. Failure
+ * modes it must survive:
  *
  *  - rotation/replacement: the inode changes, so the old offset is meaningless
  *    and we start from zero.
@@ -14,12 +17,32 @@ import type { Spool } from './spool.js';
  *    remainder arrives on the next pass as an orphaned fragment and both
  *    halves fail to parse. So the checkpoint stops at the last complete
  *    newline and the partial line is re-read whole next time.
+ *  - a pathological line: anything over `maxLineBytes` is skipped to the next
+ *    newline and counted, never buffered — one 512MiB line used to throw and
+ *    abort every later file on every scan.
+ *  - a huge backlog: at most `maxBytesPerScan` per file per call, so a first
+ *    import yields to the upload loop instead of reading 27MB files back to
+ *    back.
  *
  * Offsets are byte offsets, computed from the buffer rather than from decoded
  * strings — a multi-byte character would otherwise desynchronise the position.
  */
+export interface TailedLine {
+    text: string;
+    /** Byte offset of the line's first byte. Stable for an append-only file. */
+    offset: number;
+}
 export interface TailResult {
-    lines: string[];
+    lines: number;
     bytesRead: number;
+    /** Lines over `maxLineBytes`, discarded. */
+    skipped: number;
+    /** True when `maxBytesPerScan` stopped the read before EOF. */
+    more: boolean;
+}
+export interface TailOptions {
+    chunkBytes?: number;
+    maxLineBytes?: number;
+    maxBytesPerScan?: number;
 }
-export declare function tailFile(path: string, spool: Spool): Promise<TailResult>;
+export declare function tailFile(path: string, spool: Spool, onLines: (lines: TailedLine[]) => void, options?: TailOptions): Promise<TailResult>;

package/dist/queue/tailer.js

@@ -1,10 +1,16 @@
-import { existsSync, statSync } from 'node:fs';
+import { statSync } from 'node:fs';
 import { open } from 'node:fs/promises';
 const NEWLINE = 0x0a;
-export async function tailFile(path, spool) {
-    if (!existsSync(path))
-        return { lines: [], bytesRead: 0 };
-    const stats = statSync(path);
+const CR = 0x0d;
+const EMPTY = Buffer.alloc(0);
+export async function tailFile(path, spool, onLines, options = {}) {
+    const chunkBytes = options.chunkBytes ?? 4 * 1024 * 1024;
+    const maxLineBytes = options.maxLineBytes ?? 8 * 1024 * 1024;
+    const maxBytesPerScan = options.maxBytesPerScan ?? 64 * 1024 * 1024;
+    const result = { lines: 0, bytesRead: 0, skipped: 0, more: false };
+    const stats = statSync(path, { throwIfNoEntry: false });
+    if (!stats)
+        return result;
     const inode = String(stats.ino);
     const checkpoint = spool.getCheckpoint(path);
     let start = 0;
@@ -14,35 +20,90 @@ export async function tailFile(path, spool) {
     // else: rotated (inode differs) or truncated (offset > size) — reread from 0.
     if (start >= stats.size) {
         spool.setCheckpoint(path, inode, stats.size, stats.size);
-        return { lines: [], bytesRead: 0 };
+        return result;
     }
-    const length = stats.size - start;
-    const buffer = Buffer.allocUnsafe(length);
     const handle = await open(path, 'r');
     try {
-        await handle.read(buffer, 0, length, start);
+        let position = start;
+        let carry = EMPTY;
+        // Inside a line that already exceeded maxLineBytes: drop bytes up to and
+        // including the next newline.
+        let skipping = false;
+        while (position < stats.size && result.bytesRead < maxBytesPerScan) {
+            const want = Math.min(chunkBytes, stats.size - position);
+            const chunk = Buffer.allocUnsafe(want);
+            let got = 0;
+            // read() may return short; loop until the chunk is full or the file
+            // turned out shorter than stat said (truncated under us).
+            while (got < want) {
+                const { bytesRead } = await handle.read(chunk, got, want - got, position + got);
+                if (bytesRead === 0)
+                    break;
+                got += bytesRead;
+            }
+            if (got === 0)
+                break;
+            result.bytesRead += got;
+            const data = carry.length ? Buffer.concat([carry, chunk.subarray(0, got)]) : chunk.subarray(0, got);
+            const dataStart = position - carry.length;
+            position += got;
+            const lastNewline = data.lastIndexOf(NEWLINE);
+            if (lastNewline === -1) {
+                // No complete line in what we have. Either keep waiting for the
+                // newline or, past the cap, give up on this line entirely.
+                if (skipping || data.length > maxLineBytes) {
+                    if (!skipping)
+                        result.skipped += 1;
+                    skipping = true;
+                    carry = EMPTY;
+                    spool.setCheckpoint(path, inode, position, stats.size);
+                }
+                else {
+                    carry = Buffer.from(data);
+                }
+                continue;
+            }
+            const lines = [];
+            let lineStart = 0;
+            while (lineStart <= lastNewline) {
+                const nl = data.indexOf(NEWLINE, lineStart);
+                const end = nl > lineStart && data[nl - 1] === CR ? nl - 1 : nl;
+                if (skipping) {
+                    skipping = false; // this newline terminates the oversized line
+                }
+                else if (nl - lineStart > maxLineBytes) {
+                    result.skipped += 1;
+                }
+                else if (end > lineStart) {
+                    lines.push({ text: data.toString('utf8', lineStart, end), offset: dataStart + lineStart });
+                }
+                lineStart = nl + 1;
+            }
+            const consumed = dataStart + lastNewline + 1;
+            // Lines and their checkpoint commit together, or not at all.
+            spool.transaction(() => {
+                if (lines.length > 0)
+                    onLines(lines);
+                spool.setCheckpoint(path, inode, consumed, stats.size);
+            });
+            result.lines += lines.length;
+            const rest = data.subarray(lastNewline + 1);
+            if (rest.length > maxLineBytes) {
+                result.skipped += 1;
+                skipping = true;
+                carry = EMPTY;
+                spool.setCheckpoint(path, inode, position, stats.size);
+            }
+            else {
+                // Copy, so the 4MB chunk behind the slice can be collected.
+                carry = rest.length ? Buffer.from(rest) : EMPTY;
+            }
+        }
+        result.more = position < stats.size;
     }
     finally {
         await handle.close();
     }
-    // Consume only up to the final newline. Anything after it is a line the
-    // agent has not finished writing.
-    const lastNewline = buffer.lastIndexOf(NEWLINE);
-    if (lastNewline === -1) {
-        // No complete line yet — leave the checkpoint untouched so the whole
-        // partial line is re-read once it is terminated.
-        return { lines: [], bytesRead: 0 };
-    }
-    const consumable = buffer.subarray(0, lastNewline + 1);
-    const lines = consumable
-        .toString('utf8')
-        .split('\n')
-        // The trailing element after the final \n is always '' — drop it, and drop
-        // any blank lines rather than handing '' to a parser.
-        .filter((line) => line.length > 0)
-        .map((line) => (line.endsWith('\r') ? line.slice(0, -1) : line));
-    const consumed = start + consumable.length;
-    spool.setCheckpoint(path, inode, consumed, stats.size);
-    return { lines, bytesRead: consumable.length };
+    return result;
 }
 //# sourceMappingURL=tailer.js.map

package/dist/queue/tailer.js.map

@@ -1 +1 @@
-{"version":3,"file":"tailer.js","sourceRoot":"","sources":["../../src/queue/tailer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AA2BxC,MAAM,OAAO,GAAG,IAAI,CAAC;AAErB,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAY,EAAE,KAAY;IACvD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IAE1D,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAE7C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,UAAU,IAAI,UAAU,CAAC,KAAK,KAAK,KAAK,IAAI,UAAU,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAChF,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC;IAC5B,CAAC;IACD,8EAA8E;IAE9E,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QACxB,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACzD,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IACrC,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC;IAClC,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAE1C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAC9C,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED,wEAAwE;IACxE,kCAAkC;IAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAChD,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;QACvB,qEAAqE;QACrE,iDAAiD;QACjD,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IACrC,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,UAAU;SACrB,QAAQ,CAAC,MAAM,CAAC;SAChB,KAAK,CAAC,IAAI,CAAC;QACZ,2EAA2E;QAC3E,sDAAsD;SACrD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;SACjC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEnE,MAAM,QAAQ,GAAG,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC;IAC3C,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IACvD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;AACjD,CAAC"}
+{"version":3,"file":"tailer.js","sourceRoot":"","sources":["../../src/queue/tailer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAoDxC,MAAM,OAAO,GAAG,IAAI,CAAC;AACrB,MAAM,EAAE,GAAG,IAAI,CAAC;AAChB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAE9B,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAY,EACZ,KAAY,EACZ,OAAsC,EACtC,UAAuB,EAAE;IAEzB,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;IACzD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;IAC7D,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;IACpE,MAAM,MAAM,GAAe,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IAE/E,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IACxD,IAAI,CAAC,KAAK;QAAE,OAAO,MAAM,CAAC;IAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAE7C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,UAAU,IAAI,UAAU,CAAC,KAAK,KAAK,KAAK,IAAI,UAAU,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAChF,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC;IAC5B,CAAC;IACD,8EAA8E;IAE9E,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QACxB,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACzD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACrC,IAAI,CAAC;QACH,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,yEAAyE;QACzE,8BAA8B;QAC9B,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,OAAO,QAAQ,GAAG,KAAK,CAAC,IAAI,IAAI,MAAM,CAAC,SAAS,GAAG,eAAe,EAAE,CAAC;YACnE,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC;YACzD,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,GAAG,GAAG,CAAC,CAAC;YACZ,oEAAoE;YACpE,0DAA0D;YAC1D,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;gBAClB,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,GAAG,GAAG,EAAE,QAAQ,GAAG,GAAG,CAAC,CAAC;gBAChF,IAAI,SAAS,KAAK,CAAC;oBAAE,MAAM;gBAC3B,GAAG,IAAI,SAAS,CAAC;YACnB,CAAC;YACD,IAAI,GAAG,KAAK,CAAC;gBAAE,MAAM;YACrB,MAAM,CAAC,SAAS,IAAI,GAAG,CAAC;YAExB,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACpG,MAAM,SAAS,GAAG,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC;YAC1C,QAAQ,IAAI,GAAG,CAAC;YAEhB,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAC9C,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;gBACvB,gEAAgE;gBAChE,2DAA2D;gBAC3D,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;oBAC3C,IAAI,CAAC,QAAQ;wBAAE,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;oBACnC,QAAQ,GAAG,IAAI,CAAC;oBAChB,KAAK,GAAG,KAAK,CAAC;oBACd,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;gBACzD,CAAC;qBAAM,CAAC;oBACN,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC5B,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,KAAK,GAAiB,EAAE,CAAC;YAC/B,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,OAAO,SAAS,IAAI,WAAW,EAAE,CAAC;gBAChC,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;gBAC5C,MAAM,GAAG,GAAG,EAAE,GAAG,SAAS,IAAI,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChE,IAAI,QAAQ,EAAE,CAAC;oBACb,QAAQ,GAAG,KAAK,CAAC,CAAC,6CAA6C;gBACjE,CAAC;qBAAM,IAAI,EAAE,GAAG,SAAS,GAAG,YAAY,EAAE,CAAC;oBACzC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;gBACtB,CAAC;qBAAM,IAAI,GAAG,GAAG,SAAS,EAAE,CAAC;oBAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC,CAAC;gBAC7F,CAAC;gBACD,SAAS,GAAG,EAAE,GAAG,CAAC,CAAC;YACrB,CAAC;YAED,MAAM,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,CAAC,CAAC;YAC7C,6DAA6D;YAC7D,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE;gBACrB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;oBAAE,OAAO,CAAC,KAAK,CAAC,CAAC;gBACrC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACzD,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YAE7B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;YAC5C,IAAI,IAAI,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;gBAC/B,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;gBACpB,QAAQ,GAAG,IAAI,CAAC;gBAChB,KAAK,GAAG,KAAK,CAAC;gBACd,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACzD,CAAC;iBAAM,CAAC;gBACN,4DAA4D;gBAC5D,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YAClD,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,GAAG,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC;IACtC,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/sessions/title.d.ts

@@ -1,2 +1,14 @@
-/** First meaningful line of a prompt, trimmed to a usable title length. */
-export declare function deriveTitle(text: string, maxLength?: number): string;
+/**
+ * First meaningful line of a prompt, secret-redacted and NOT truncated.
+ *
+ * Truncation deliberately does not happen here. Cutting the line first slices a
+ * straddling secret in half, neither half matches its pattern any more, and the
+ * fragment ships inside the title. Only the built-in rules are reachable from an
+ * adapter, so doing the cut here could only ever be safe for those — an
+ * org-supplied pattern would still be bisected.
+ *
+ * The privacy pipeline redacts this field again with the org's own rules and
+ * only then truncates it (see TITLE_MAX_LENGTH in privacy/pipeline.ts), so the
+ * cut always lands on already-redacted text whatever the rule's source.
+ */
+export declare function deriveTitle(text: string): string;