@phigon/observer 8.1.0 → 8.3.0

package/dist/src/event-queue.js

@@ -1,53 +1,94 @@
+/**
+ * Collection pauses once a byte-budgeted queue holds this share of its
+ * budget. Events can be up to 1 MiB each, so the event count alone lets a
+ * slow server fill the budget first; the rest is room for a scan already
+ * under way.
+ */
+const byteBackpressureRatio = 0.75;
+/** Whether a queue still has room by bytes for another collection pass. */
+export function hasByteHeadroom(metrics) {
+    return (metrics.bytes === undefined ||
+        metrics.byteCapacity === undefined ||
+        metrics.bytes < metrics.byteCapacity * byteBackpressureRatio);
+}
 export class BoundedEventQueue {
     #capacity;
+    // Queued events are #events[#head..], ordered by sequence. The stream
+    // acknowledges every event and pumps after each acknowledgement, so
+    // acknowledge, metrics and pendingAfter work from the ends of the queue
+    // and acknowledged events are released by advancing #head; a full queue
+    // drains in linear rather than quadratic time.
     #events = [];
+    #head = 0;
     #dropped = 0;
     #coalesced = 0;
     #acknowledged = 0n;
+    // Sum of the `bytes` of the queued events (#events[#head..]).
+    #bytes = 0;
     constructor(capacity = 2048) {
         if (!Number.isSafeInteger(capacity) || capacity < 1) {
             throw new Error("Event queue capacity must be a positive integer.");
         }
         this.#capacity = capacity;
     }
-    enqueue(value, sequence, coalesceKey) {
+    enqueue(value, sequence, coalesceKey, bytes = 0) {
         if (sequence <= this.#acknowledged)
             return;
+        const event = {
+            value,
+            sequence,
+            ...(coalesceKey === undefined ? {} : { coalesceKey }),
+            ...(bytes === 0 ? {} : { bytes }),
+        };
+        if (coalesceKey === undefined && this.#size() < this.#capacity) {
+            const last = this.#last();
+            if (last === undefined || last.sequence <= sequence) {
+                this.#events.push(event);
+                this.#bytes += bytes;
+                return;
+            }
+        }
+        this.#compact();
         if (coalesceKey !== undefined) {
-            const index = this.#events.findIndex((event) => event.coalesceKey === coalesceKey);
+            const index = this.#events.findIndex((candidate) => candidate.coalesceKey === coalesceKey);
             if (index >= 0) {
-                this.#events[index] = { value, sequence, coalesceKey };
-                this.#events.sort((left, right) => left.sequence < right.sequence ? -1 : 1);
+                this.#bytes += bytes - (this.#events[index]?.bytes ?? 0);
+                this.#events[index] = event;
+                this.#events.sort((left, right) => left.sequence < right.sequence ? -1 : left.sequence > right.sequence ? 1 : 0);
                 this.#coalesced++;
                 return;
             }
         }
         if (this.#events.length >= this.#capacity) {
-            this.#events.shift();
+            this.#bytes -= this.#events.shift()?.bytes ?? 0;
             this.#dropped++;
         }
-        this.#events.push({
-            value,
-            sequence,
-            ...(coalesceKey === undefined ? {} : { coalesceKey }),
-        });
+        this.#events.splice(this.#firstAfter(sequence), 0, event);
+        this.#bytes += bytes;
     }
     canEnqueue(coalesceKey) {
-        return (this.#events.length < this.#capacity ||
-            (coalesceKey !== undefined &&
-                this.#events.some((event) => event.coalesceKey === coalesceKey)));
+        if (this.#size() < this.#capacity)
+            return true;
+        if (coalesceKey === undefined)
+            return false;
+        for (let index = this.#head; index < this.#events.length; index++) {
+            if (this.#events[index]?.coalesceKey === coalesceKey)
+                return true;
+        }
+        return false;
     }
     acknowledge(throughSequence) {
         if (throughSequence <= this.#acknowledged)
             return;
         this.#acknowledged = throughSequence;
-        this.#events = this.#events.filter((event) => event.sequence > throughSequence);
+        this.#removeThrough(throughSequence);
     }
     discard(sequence) {
+        this.#compact();
         const index = this.#events.findIndex((event) => event.sequence === sequence);
         if (index < 0)
             return false;
-        this.#events.splice(index, 1);
+        this.#bytes -= this.#events.splice(index, 1)[0]?.bytes ?? 0;
         this.#dropped++;
         return true;
     }
@@ -56,15 +97,18 @@ export class BoundedEventQueue {
             throw new Error("Acknowledged sequence must not be negative.");
         }
         this.#acknowledged = throughSequence;
-        this.#events = this.#events.filter((event) => event.sequence > throughSequence);
+        this.#removeThrough(throughSequence);
     }
     pending() {
-        return this.#events.map((event) => event.value);
+        return this.#events.slice(this.#head).map((event) => event.value);
     }
     entries() {
-        return this.#events.map((event) => Object.freeze({ ...event }));
+        return this.#events
+            .slice(this.#head)
+            .map((event) => Object.freeze({ ...event }));
     }
     replacePending(replace) {
+        this.#compact();
         this.#events = this.#events.map((event) => ({
             ...event,
             value: replace(event.value),
@@ -74,46 +118,58 @@ export class BoundedEventQueue {
         if (source.#capacity !== this.#capacity) {
             throw new Error("Event queue replacement capacity does not match.");
         }
-        this.#events = source.#events.map((event) => ({ ...event }));
+        this.#events = source.#events
+            .slice(source.#head)
+            .map((event) => ({ ...event }));
+        this.#head = 0;
         this.#dropped = source.#dropped;
         this.#coalesced = source.#coalesced;
         this.#acknowledged = source.#acknowledged;
+        this.#bytes = source.#bytes;
     }
-    latestSequence() {
-        let latest;
-        for (const event of this.#events) {
-            if (latest === undefined || event.sequence > latest) {
-                latest = event.sequence;
-            }
+    /** Stored size of the queued events. */
+    queuedBytes() {
+        return this.#bytes;
+    }
+    /** Replace the stored sizes of the queued events, oldest first. */
+    reweigh(bytes) {
+        this.#compact();
+        if (bytes.length !== this.#events.length) {
+            throw new Error("Event queue sizes do not match the queued events.");
         }
-        return latest;
+        this.#events = this.#events.map((event, index) => {
+            const size = bytes[index] ?? 0;
+            return {
+                value: event.value,
+                sequence: event.sequence,
+                ...(event.coalesceKey === undefined ? {} : { coalesceKey: event.coalesceKey }),
+                ...(size === 0 ? {} : { bytes: size }),
+            };
+        });
+        this.#bytes = bytes.reduce((total, size) => total + size, 0);
+    }
+    latestSequence() {
+        return this.#last()?.sequence;
     }
     pendingAfter(sequence, limit = this.#capacity) {
         if (!Number.isSafeInteger(limit) || limit < 1) {
             return [];
         }
+        const start = this.#firstAfter(sequence);
         return this.#events
-            .filter((event) => event.sequence > sequence)
-            .slice(0, limit)
+            .slice(start, start + limit)
             .map((event) => event.value);
     }
     metrics() {
-        let oldestSequence;
-        let latestSequence;
-        for (const event of this.#events) {
-            if (oldestSequence === undefined ||
-                event.sequence < oldestSequence) {
-                oldestSequence = event.sequence;
-            }
-            if (latestSequence === undefined ||
-                event.sequence > latestSequence) {
-                latestSequence = event.sequence;
-            }
-        }
+        const depth = this.#size();
+        const oldestSequence = depth === 0
+            ? undefined
+            : this.#events[this.#head]?.sequence;
+        const latestSequence = this.#last()?.sequence;
         return Object.freeze({
             capacity: this.#capacity,
-            depth: this.#events.length,
-            utilization: this.#events.length / this.#capacity,
+            depth,
+            utilization: depth / this.#capacity,
             dropped: this.#dropped,
             coalesced: this.#coalesced,
             acknowledged: this.#acknowledged,
@@ -121,5 +177,44 @@ export class BoundedEventQueue {
             ...(latestSequence === undefined ? {} : { latestSequence }),
         });
     }
+    #size() {
+        return this.#events.length - this.#head;
+    }
+    #last() {
+        return this.#size() === 0 ? undefined : this.#events.at(-1);
+    }
+    /** Index of the first queued event with a sequence above `sequence`. */
+    #firstAfter(sequence) {
+        let low = this.#head;
+        let high = this.#events.length;
+        while (low < high) {
+            const middle = (low + high) >>> 1;
+            if ((this.#events[middle]?.sequence ?? 0n) <= sequence) {
+                low = middle + 1;
+            }
+            else {
+                high = middle;
+            }
+        }
+        return low;
+    }
+    #removeThrough(sequence) {
+        const head = this.#firstAfter(sequence);
+        for (let index = this.#head; index < head; index++) {
+            this.#bytes -= this.#events[index]?.bytes ?? 0;
+        }
+        this.#head = head;
+        // Release acknowledged events once they are half of the backing array,
+        // which keeps the copy amortized to constant work per event.
+        if (this.#head > 0 && this.#head * 2 >= this.#events.length) {
+            this.#compact();
+        }
+    }
+    #compact() {
+        if (this.#head === 0)
+            return;
+        this.#events = this.#events.slice(this.#head);
+        this.#head = 0;
+    }
 }
 //# sourceMappingURL=event-queue.js.map

package/dist/src/event-queue.js.map

@@ -1 +1 @@
-{"version":3,"file":"event-queue.js","sourceRoot":"","sources":["../../src/event-queue.ts"],"names":[],"mappings":"AAiBA,MAAM,OAAO,iBAAiB;IACnB,SAAS,CAAS;IAC3B,OAAO,GAA0B,EAAE,CAAC;IACpC,QAAQ,GAAG,CAAC,CAAC;IACb,UAAU,GAAG,CAAC,CAAC;IACf,aAAa,GAAG,EAAE,CAAC;IAEnB,YAAY,QAAQ,GAAG,IAAI;QACzB,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,OAAO,CACL,KAAQ,EACR,QAAgB,EAChB,WAAoB;QAEpB,IAAI,QAAQ,IAAI,IAAI,CAAC,aAAa;YAAE,OAAO;QAC3C,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAClC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,WAAW,CAC7C,CAAC;YACF,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACf,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;gBACvD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAChC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACxC,CAAC;gBACF,IAAI,CAAC,UAAU,EAAE,CAAC;gBAClB,OAAO;YACT,CAAC;QACH,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1C,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACrB,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,KAAK;YACL,QAAQ;YACR,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC;SACtD,CAAC,CAAC;IACL,CAAC;IAED,UAAU,CAAC,WAAoB;QAC7B,OAAO,CACL,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS;YACpC,CAAC,WAAW,KAAK,SAAS;gBACxB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,WAAW,CAAC,CAAC,CACnE,CAAC;IACJ,CAAC;IAED,WAAW,CAAC,eAAuB;QACjC,IAAI,eAAe,IAAI,IAAI,CAAC,aAAa;YAAE,OAAO;QAClD,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC;QACrC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAChC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,GAAG,eAAe,CAC5C,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,QAAgB;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAClC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,CACvC,CAAC;QACF,IAAI,KAAK,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC9B,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,qBAAqB,CAAC,eAAuB;QAC3C,IAAI,eAAe,GAAG,EAAE,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC;QACrC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAChC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,GAAG,eAAe,CAC5C,CAAC;IACJ,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAClD,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;IAClE,CAAC;IAED,cAAc,CAAC,OAAwB;QACrC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC1C,GAAG,KAAK;YACR,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;SAC5B,CAAC,CAAC,CAAC;IACN,CAAC;IAES,YAAY,CAAC,MAA4B;QACjD,IAAI,MAAM,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAChC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;IAC5C,CAAC;IAED,cAAc;QACZ,IAAI,MAA0B,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjC,IAAI,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,GAAG,MAAM,EAAE,CAAC;gBACpD,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC;YAC1B,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,YAAY,CACV,QAAgB,EAChB,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC9C,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,OAAO,IAAI,CAAC,OAAO;aAChB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC;aAC5C,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;aACf,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;IAED,OAAO;QACL,IAAI,cAAkC,CAAC;QACvC,IAAI,cAAkC,CAAC;QACvC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjC,IACE,cAAc,KAAK,SAAS;gBAC5B,KAAK,CAAC,QAAQ,GAAG,cAAc,EAC/B,CAAC;gBACD,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAC;YAClC,CAAC;YACD,IACE,cAAc,KAAK,SAAS;gBAC5B,KAAK,CAAC,QAAQ,GAAG,cAAc,EAC/B,CAAC;gBACD,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAC;YAClC,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC;YACnB,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;YAC1B,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS;YACjD,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,SAAS,EAAE,IAAI,CAAC,UAAU;YAC1B,YAAY,EAAE,IAAI,CAAC,aAAa;YAChC,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC;YAC3D,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC;SAC5D,CAAC,CAAC;IACL,CAAC;CACF"}
+{"version":3,"file":"event-queue.js","sourceRoot":"","sources":["../../src/event-queue.ts"],"names":[],"mappings":"AAeA;;;;;GAKG;AACH,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAEnC,2EAA2E;AAC3E,MAAM,UAAU,eAAe,CAAC,OAAqB;IACnD,OAAO,CACL,OAAO,CAAC,KAAK,KAAK,SAAS;QAC3B,OAAO,CAAC,YAAY,KAAK,SAAS;QAClC,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,YAAY,GAAG,qBAAqB,CAC7D,CAAC;AACJ,CAAC;AAUD,MAAM,OAAO,iBAAiB;IACnB,SAAS,CAAS;IAC3B,sEAAsE;IACtE,oEAAoE;IACpE,wEAAwE;IACxE,wEAAwE;IACxE,+CAA+C;IAC/C,OAAO,GAA0B,EAAE,CAAC;IACpC,KAAK,GAAG,CAAC,CAAC;IACV,QAAQ,GAAG,CAAC,CAAC;IACb,UAAU,GAAG,CAAC,CAAC;IACf,aAAa,GAAG,EAAE,CAAC;IACnB,8DAA8D;IAC9D,MAAM,GAAG,CAAC,CAAC;IAEX,YAAY,QAAQ,GAAG,IAAI;QACzB,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,OAAO,CACL,KAAQ,EACR,QAAgB,EAChB,WAAoB,EACpB,KAAK,GAAG,CAAC;QAET,IAAI,QAAQ,IAAI,IAAI,CAAC,aAAa;YAAE,OAAO;QAC3C,MAAM,KAAK,GAAG;YACZ,KAAK;YACL,QAAQ;YACR,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC;YACrD,GAAG,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;SAClC,CAAC;QACF,IAAI,WAAW,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YAC/D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,QAAQ,IAAI,QAAQ,EAAE,CAAC;gBACpD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACzB,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;gBACrB,OAAO;YACT,CAAC;QACH,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAClC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,KAAK,WAAW,CACrD,CAAC;YACF,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;gBACzD,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;gBAC5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAChC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC7E,CAAC;gBACF,IAAI,CAAC,UAAU,EAAE,CAAC;gBAClB,OAAO;YACT,CAAC;QACH,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,KAAK,IAAI,CAAC,CAAC;YAChD,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;IACvB,CAAC;IAED,UAAU,CAAC,WAAoB;QAC7B,IAAI,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAC/C,IAAI,WAAW,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QAC5C,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YAClE,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,WAAW,KAAK,WAAW;gBAAE,OAAO,IAAI,CAAC;QACpE,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,WAAW,CAAC,eAAuB;QACjC,IAAI,eAAe,IAAI,IAAI,CAAC,aAAa;YAAE,OAAO;QAClD,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC;QACrC,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,CAAC,QAAgB;QACtB,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAClC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,CACvC,CAAC;QACF,IAAI,KAAK,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAC5B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,qBAAqB,CAAC,eAAuB;QAC3C,IAAI,eAAe,GAAG,EAAE,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC;QACrC,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IACvC,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACpE,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,OAAO;aAChB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;aACjB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,cAAc,CAAC,OAAwB;QACrC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC1C,GAAG,KAAK;YACR,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;SAC5B,CAAC,CAAC,CAAC;IACN,CAAC;IAES,YAAY,CAAC,MAA4B;QACjD,IAAI,MAAM,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO;aAC1B,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC;aACnB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;QACf,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAChC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC9B,CAAC;IAED,wCAAwC;IAC9B,WAAW;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,mEAAmE;IACzD,OAAO,CAAC,KAAwB;QACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACvE,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YAC/C,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC/B,OAAO;gBACL,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,GAAG,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;gBAC9E,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;aACvC,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,KAAK,EAAE,EAAE,QAAQ,CAAC;IAChC,CAAC;IAED,YAAY,CACV,QAAgB,EAChB,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC9C,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACzC,OAAO,IAAI,CAAC,OAAO;aAChB,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK,CAAC;aAC3B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;IAED,OAAO;QACL,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,MAAM,cAAc,GAAG,KAAK,KAAK,CAAC;YAChC,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC;QACvC,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,EAAE,EAAE,QAAQ,CAAC;QAC9C,OAAO,MAAM,CAAC,MAAM,CAAC;YACnB,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,KAAK;YACL,WAAW,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;YACnC,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,SAAS,EAAE,IAAI,CAAC,UAAU;YAC1B,YAAY,EAAE,IAAI,CAAC,aAAa;YAChC,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC;YAC3D,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC;SAC5D,CAAC,CAAC;IACL,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;IAC1C,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,wEAAwE;IACxE,WAAW,CAAC,QAAgB;QAC1B,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC;QACrB,IAAI,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QAC/B,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;YAClB,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,QAAQ,IAAI,EAAE,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACvD,GAAG,GAAG,MAAM,GAAG,CAAC,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,IAAI,GAAG,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,cAAc,CAAC,QAAgB;QAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACxC,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACnD,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,uEAAuE;QACvE,6DAA6D;QAC7D,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAC5D,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,CAAC;IACH,CAAC;IAED,QAAQ;QACN,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC;YAAE,OAAO;QAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9C,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;IACjB,CAAC;CACF"}

package/dist/src/event-spool.d.ts

@@ -1,14 +1,62 @@
 import { type EventEnvelope } from "./generated/phigon/v1/telemetry_pb.js";
-import { BoundedEventQueue } from "./event-queue.js";
+import { BoundedEventQueue, type QueueMetrics } from "./event-queue.js";
+/**
+ * What `restore` did instead of loading the spool as it was:
+ * - `other_machine`: an earlier pairing's spool, moved to `directory`.
+ * - `damaged`: a spool that could not be read, moved to `directory`.
+ * - `over_capacity`: more events than the queue holds; the newest were kept
+ *   and `dropped` of the oldest were left out.
+ */
+export type SpoolRestoreNotice = Readonly<{
+    reason: "other_machine" | "damaged";
+    directory: string;
+}> | Readonly<{
+    reason: "over_capacity";
+    dropped: number;
+}>;
 /**
  * An encrypted append-only delivery spool. The in-memory queue remains the hot
  * path, while this log makes unacknowledged events recoverable after an
  * observer restart without writing prompt or tool content as plaintext.
+ *
+ * Records are group-committed: they are buffered and written with one
+ * flushed append at most 100 ms later, or at once by `flush`. An event is
+ * durable only after a flush, so a source checkpoint may move past an event
+ * only after `flush` returns (the collector's `flushDelivery`). Events the
+ * server already acknowledged are safe either way, and a lost
+ * acknowledgement or discard record only sends an event again.
  */
 export declare class FileBackedEventQueue extends BoundedEventQueue<EventEnvelope> {
     #private;
-    constructor(dataDirectory: string, capacity?: number);
-    restore(tenantID: string, machineID: string, credential: string): void;
+    constructor(dataDirectory: string, capacity?: number, maxBytes?: number);
+    /**
+     * Writes every buffered record with one flushed append. Throws when the
+     * spool cannot be written; the records stay buffered and the next write
+     * retries them.
+     */
+    flush(): void;
+    /**
+     * Adds the spool's size by bytes: the stored size of the unacknowledged
+     * events, which is what a compaction keeps, and the budget it must fit.
+     */
+    metrics(): QueueMetrics;
+    /**
+     * Loads the spool for this machine. None of these stops the observer, and
+     * the returned notice says which one happened:
+     * - A spool that an earlier pairing of this data directory left for
+     *   another machine cannot be delivered as this one: it is moved aside
+     *   with its stream cursor.
+     * - A spool that is too large to read safely, or holds a record that does
+     *   not verify before its last line, is moved aside on its own and the
+     *   queue starts empty. Its events were written before their sources'
+     *   checkpoints moved, so they are not read again; the moved copy keeps
+     *   them for an administrator.
+     * - A spool with more events than the queue holds, as after lowering
+     *   PHIGON_ACTIVITY_QUEUE_CAPACITY, keeps the newest events, as a full
+     *   queue does while running. The control plane records the skipped
+     *   sequence numbers as a gap.
+     */
+    restore(tenantID: string, machineID: string, credential: string): SpoolRestoreNotice | undefined;
     enqueue(value: EventEnvelope, sequence: bigint, coalesceKey?: string): void;
     acknowledge(throughSequence: bigint): void;
     discard(sequence: bigint): boolean;
@@ -16,4 +64,14 @@ export declare class FileBackedEventQueue extends BoundedEventQueue<EventEnvelop
     replacePending(replace: (value: EventEnvelope) => EventEnvelope): void;
 }
 export declare function clearEventSpool(dataDirectory: string): void;
+/**
+ * Moves what an earlier pairing of this data directory left behind out of
+ * the way, so a computer paired again as a new machine starts with an empty
+ * queue instead of refusing to read the old one. The encrypted spool, the
+ * stream cursor and `extraFiles` (such as the old credential, the only key
+ * that can still decrypt that spool) go to `retired-pairings/<time>-<machine>/`.
+ * Only the newest few pairings are kept. Returns that directory, or
+ * undefined when there was nothing to move.
+ */
+export declare function retirePairingState(dataDirectory: string, machineID: string | undefined, extraFiles?: readonly string[]): string | undefined;
 //# sourceMappingURL=event-spool.d.ts.map

package/dist/src/event-spool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"event-spool.d.ts","sourceRoot":"","sources":["../../src/event-spool.ts"],"names":[],"mappings":"AAoBA,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAoCrD;;;;GAIG;AACH,qBAAa,oBAAqB,SAAQ,iBAAiB,CAAC,aAAa,CAAC;;gBAO5D,aAAa,EAAE,MAAM,EAAE,QAAQ,SAAO;IAMlD,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAgD7D,OAAO,CACd,KAAK,EAAE,aAAa,EACpB,QAAQ,EAAE,MAAM,EAChB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI;IAeE,WAAW,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI;IAO1C,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAUlC,qBAAqB,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI;IAMpD,cAAc,CACrB,OAAO,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,aAAa,GAC/C,IAAI;CA+KR;AAED,wBAAgB,eAAe,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAkB3D"}
+{"version":3,"file":"event-spool.d.ts","sourceRoot":"","sources":["../../src/event-spool.ts"],"names":[],"mappings":"AAuBA,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,KAAK,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAoBxE;;;;;;GAMG;AACH,MAAM,MAAM,kBAAkB,GAC1B,QAAQ,CAAC;IAAE,MAAM,EAAE,eAAe,GAAG,SAAS,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,GACpE,QAAQ,CAAC;IAAE,MAAM,EAAE,eAAe,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AA8B3D;;;;;;;;;;;GAWG;AACH,qBAAa,oBAAqB,SAAQ,iBAAiB,CAAC,aAAa,CAAC;;gBAqBtE,aAAa,EAAE,MAAM,EACrB,QAAQ,SAAO,EACf,QAAQ,SAAuB;IAejC;;;;OAIG;IACH,KAAK,IAAI,IAAI;IAuBb;;;OAGG;IACM,OAAO,IAAI,YAAY;IAQhC;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CACL,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,kBAAkB,GAAG,SAAS;IAwExB,OAAO,CACd,KAAK,EAAE,aAAa,EACpB,QAAQ,EAAE,MAAM,EAChB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI;IAuBE,WAAW,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI;IAO1C,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAUlC,qBAAqB,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI;IAMpD,cAAc,CACrB,OAAO,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,aAAa,GAC/C,IAAI;CA6RR;AAED,wBAAgB,eAAe,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAE3D;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAChC,aAAa,EAAE,MAAM,EACrB,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,UAAU,GAAE,SAAS,MAAM,EAAO,GACjC,MAAM,GAAG,SAAS,CAQpB"}