@etzelm/openclaw-memory-hindsight 0.2.0 → 0.4.0

package/dist/metrics.js

@@ -0,0 +1,197 @@
+import { normalizeDiagnosticValue } from "openclaw/plugin-sdk/diagnostic-runtime";
+import { getPluginRuntimeGatewayRequestScope } from "openclaw/plugin-sdk/plugin-runtime";
+import { normalizeAgentIdStrict } from "openclaw/plugin-sdk/routing";
+/**
+ * Gateway-side metrics for auto-recall, auto-retain and the retain queue, rendered as Prometheus
+ * text on a gateway-auth plugin route. diagnostics-prometheus renders only core diagnostic events
+ * and has no plugin metric surface, so the plugin serves its own series on the gateway port.
+ * Labels are agent and bank plus fixed enums; never session ids, document ids or text.
+ */
+export const METRICS_ROUTE_PATH = "/plugins/hindsight-native/metrics";
+const PREFIX = "openclaw_hindsight_";
+// Seconds. Recall times out at 10 s, retain at 30 s.
+export const LATENCY_BUCKETS = [0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30];
+// Label sets per family before new agent and bank values fold into "other"; agents and banks come
+// from config. Outcome and reason keep their value, so the bound grows by the enum size at most.
+export const MAX_SERIES_PER_FAMILY = 200;
+const OTHER = "other";
+const FOLDED_LABELS = new Set(["agent", "bank"]);
+/** Agent label: the normalized agent id, or "unknown" when it is missing or not label-safe. */
+export function agentLabel(agentId) {
+    // The guard runs first: normalizing would turn a session-shaped value into a safe-looking id.
+    // The strict form never falls back to the default agent, so an odd id reads unknown, not main.
+    const safe = normalizeDiagnosticValue(agentId);
+    if (safe === "unknown") {
+        return safe;
+    }
+    const normalized = normalizeAgentIdStrict(safe);
+    return normalized.ok ? normalized.value : "unknown";
+}
+/**
+ * Bank label. A conversation bank (`<owner>::conv::<digest>`, one per root session) folds into
+ * `<owner>::conv`, so the label set stays as small as the configured agents and bank map.
+ */
+export function bankLabel(bankId) {
+    if (!bankId) {
+        return "unknown";
+    }
+    const conv = bankId.indexOf("::conv::");
+    return normalizeDiagnosticValue(conv >= 0 ? `${bankId.slice(0, conv)}::conv` : bankId);
+}
+/** True for the error an `AbortSignal.timeout` fetch rejects with. */
+export function isTimeoutError(error) {
+    return error?.name === "TimeoutError";
+}
+function labelKey(labels) {
+    return JSON.stringify(Object.entries(labels));
+}
+function escapeLabelValue(value) {
+    return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
+}
+function formatLabels(labels, extra) {
+    const pairs = Object.entries(labels);
+    if (extra) {
+        pairs.push(extra);
+    }
+    if (pairs.length === 0) {
+        return "";
+    }
+    return `{${pairs.map(([k, v]) => `${k}="${escapeLabelValue(v)}"`).join(",")}}`;
+}
+/** Bounded map of label set to state; past the bound every new set lands on the "other" set. */
+class Series {
+    create;
+    entries = new Map();
+    constructor(create) {
+        this.create = create;
+    }
+    get(labels) {
+        let key = labelKey(labels);
+        let entry = this.entries.get(key);
+        if (!entry) {
+            if (this.entries.size >= MAX_SERIES_PER_FAMILY) {
+                labels = Object.fromEntries(Object.entries(labels).map(([k, v]) => [k, FOLDED_LABELS.has(k) ? OTHER : v]));
+                key = labelKey(labels);
+                entry = this.entries.get(key);
+            }
+            if (!entry) {
+                entry = { labels, value: this.create() };
+                this.entries.set(key, entry);
+            }
+        }
+        return entry.value;
+    }
+    values() {
+        return [...this.entries.values()];
+    }
+}
+function newHistogram() {
+    return { buckets: LATENCY_BUCKETS.map(() => 0), sum: 0, count: 0 };
+}
+function formatNumber(value) {
+    return Number.isFinite(value) ? String(value) : value > 0 ? "+Inf" : "-Inf";
+}
+export class HindsightMetrics {
+    recalls = new Series(() => ({ n: 0 }));
+    recallLatency = new Series(newHistogram);
+    retains = new Series(() => ({ n: 0 }));
+    retainLatency = new Series(newHistogram);
+    replays = new Series(() => ({ n: 0 }));
+    drops = new Series(() => ({ n: 0 }));
+    // Kept across scrapes, so a bank whose queue drained reads 0 instead of disappearing.
+    depth = new Series(() => ({ n: 0 }));
+    recordRecall(agentId, bankId, outcome, ms) {
+        const base = { agent: agentLabel(agentId), bank: bankLabel(bankId) };
+        this.recalls.get({ ...base, outcome }).n += 1;
+        observe(this.recallLatency.get(base), ms);
+    }
+    recordRetain(agentId, bankId, outcome, ms) {
+        const base = { agent: agentLabel(agentId), bank: bankLabel(bankId) };
+        this.retains.get({ ...base, outcome }).n += 1;
+        observe(this.retainLatency.get(base), ms);
+    }
+    recordReplay(bankId, outcome) {
+        this.replays.get({ bank: bankLabel(bankId), outcome }).n += 1;
+    }
+    recordDrop(bankId, reason, count = 1) {
+        if (count > 0) {
+            this.drops.get({ bank: bankLabel(bankId), reason }).n += count;
+        }
+    }
+    /** Prometheus text exposition (version 0.0.4). Families with no samples print only HELP/TYPE. */
+    render(queueDepth) {
+        const lines = [];
+        const counter = (name, help, series) => {
+            lines.push(`# HELP ${PREFIX}${name} ${help}`, `# TYPE ${PREFIX}${name} counter`);
+            for (const { labels, value } of series.values()) {
+                lines.push(`${PREFIX}${name}${formatLabels(labels)} ${value.n}`);
+            }
+        };
+        const histogram = (name, help, series) => {
+            lines.push(`# HELP ${PREFIX}${name} ${help}`, `# TYPE ${PREFIX}${name} histogram`);
+            for (const { labels, value } of series.values()) {
+                LATENCY_BUCKETS.forEach((le, i) => {
+                    lines.push(`${PREFIX}${name}_bucket${formatLabels(labels, ["le", String(le)])} ${value.buckets[i]}`);
+                });
+                lines.push(`${PREFIX}${name}_bucket${formatLabels(labels, ["le", "+Inf"])} ${value.count}`, `${PREFIX}${name}_sum${formatLabels(labels)} ${formatNumber(value.sum)}`, `${PREFIX}${name}_count${formatLabels(labels)} ${value.count}`);
+            }
+        };
+        counter("recall_total", "Auto-recall attempts by outcome.", this.recalls);
+        histogram("recall_duration_seconds", "Auto-recall latency.", this.recallLatency);
+        counter("retain_total", "Auto-retain attempts by outcome.", this.retains);
+        histogram("retain_duration_seconds", "Auto-retain latency.", this.retainLatency);
+        lines.push(`# HELP ${PREFIX}retain_queue_depth Retains waiting in the offline queue.`, `# TYPE ${PREFIX}retain_queue_depth gauge`);
+        for (const { value } of this.depth.values()) {
+            value.n = 0;
+        }
+        for (const [bankId, n] of queueDepth?.() ?? []) {
+            this.depth.get({ bank: bankLabel(bankId) }).n += n;
+        }
+        for (const { labels, value } of this.depth.values()) {
+            lines.push(`${PREFIX}retain_queue_depth${formatLabels(labels)} ${value.n}`);
+        }
+        counter("retain_queue_replays_total", "Queued retain replay attempts by outcome.", this.replays);
+        counter("retain_queue_drops_total", "Retains dropped from the queue by reason.", this.drops);
+        return `${lines.join("\n")}\n`;
+    }
+}
+function observe(h, ms) {
+    const seconds = Math.max(0, ms) / 1000;
+    LATENCY_BUCKETS.forEach((le, i) => {
+        if (seconds <= le) {
+            h.buckets[i] += 1;
+        }
+    });
+    h.sum += seconds;
+    h.count += 1;
+}
+// Mirrors diagnostics-prometheus: the route is gateway-auth, and read access is checked here so a
+// scope-narrowed operator identity cannot read it.
+const READ_SCOPES = ["operator.read", "operator.write", "operator.admin"];
+function hasReadScope() {
+    const scopes = getPluginRuntimeGatewayRequestScope()?.client?.connect?.scopes;
+    return Array.isArray(scopes) && READ_SCOPES.some((scope) => scopes.includes(scope));
+}
+export function createMetricsRouteHandler(metrics, queueDepth, readScope = hasReadScope) {
+    return (req, res) => {
+        res.setHeader("Cache-Control", "no-store");
+        if (!readScope()) {
+            res.statusCode = 403;
+            res.setHeader("Content-Type", "text/plain; charset=utf-8");
+            res.end("missing scope: operator.read");
+            return true;
+        }
+        if (req.method !== "GET" && req.method !== "HEAD") {
+            res.statusCode = 405;
+            res.setHeader("Allow", "GET, HEAD");
+            res.end("Method Not Allowed");
+            return true;
+        }
+        const body = metrics.render(queueDepth);
+        res.statusCode = 200;
+        res.setHeader("Content-Type", "text/plain; version=0.0.4; charset=utf-8");
+        res.setHeader("Content-Length", String(Buffer.byteLength(body, "utf8")));
+        res.end(req.method === "HEAD" ? undefined : body);
+        return true;
+    };
+}

package/dist/provider-runtime.js

@@ -36,6 +36,7 @@ function sliceLines(text, from, lines) {
 }
 export function createHindsightProviderRuntime(params) {
     const { providerId, service } = params;
+    const reference = (id) => ({ providerId, id });
     return {
         async open({ agentId: rawAgentId, context }) {
             if (!rawAgentId?.trim()) {
@@ -57,7 +58,6 @@ export function createHindsightProviderRuntime(params) {
                 assertCurrent();
                 return grant.bankId;
             };
-            const reference = (id) => ({ providerId, id });
             const provider = {
                 capabilities: {
                     sources: ["memory"],

package/dist/retain-queue.js

@@ -80,6 +80,10 @@ function isQueuedRetain(value) {
         v.item !== null &&
         typeof v.item.content === "string");
 }
+/** One queued copy per bank and document id; JSON keeps ids with any characters apart. */
+function queueKey(key) {
+    return JSON.stringify([key.bankId, key.documentId]);
+}
 /** The key of a tombstone line, written when a rewrite failed to drop an out-of-date copy. */
 function tombstoneKey(value) {
     const t = value?.tombstone;
@@ -87,6 +91,9 @@ function tombstoneKey(value) {
         ? { bankId: t.bankId, documentId: t.documentId }
         : undefined;
 }
+function tombstoneLine(key) {
+    return JSON.stringify({ tombstone: { bankId: key.bankId, documentId: key.documentId } });
+}
 /** True when the file is empty or its last byte is a newline. */
 function endsWithNewline(filePath) {
     const fd = openSync(filePath, "r");
@@ -109,7 +116,10 @@ function stored(entry, line = JSON.stringify(entry)) {
 /**
  * Bounded, file-backed FIFO of failed auto-retains, one JSON line per item. One entry per bank and
  * document id: a newer retain of the same document replaces the queued one in place. A new item is
- * appended to the file; replacing, pruning and replay rewrite it (mode 600, atomic rename).
+ * appended to the file; replacing, pruning and replay rewrite it (mode 600, atomic rename). When a
+ * rewrite fails, the item is appended instead (`load()` keeps the newest copy of each document). An
+ * append that would push the file past the size bound goes through a rewrite; when both fail, the
+ * caller sends the item live.
  */
 export class RetainQueue {
     options;
@@ -117,12 +127,16 @@ export class RetainQueue {
     dropWarned = false;
     /** Removals not yet written (replay batches its writes). */
     dirty = false;
+    /** Bytes in the file as this queue last read or wrote it; appends grow it, rewrites reset it. */
+    fileBytes = 0;
     limits;
     logger;
+    onDrop;
     constructor(options) {
         this.options = options;
         this.limits = options.limits;
         this.logger = options.logger;
+        this.onDrop = options.onDrop;
     }
     get filePath() {
         return this.options.filePath;
@@ -131,6 +145,7 @@ export class RetainQueue {
     configure(params) {
         this.limits = params.limits;
         this.logger = params.logger;
+        this.onDrop = params.onDrop;
     }
     now() {
         return this.options.now?.() ?? Date.now();
@@ -150,13 +165,18 @@ export class RetainQueue {
         catch (error) {
             if (error.code === "ENOENT") {
                 this.stored = [];
+                this.fileBytes = 0;
                 return;
             }
             throw error;
         }
-        let loaded = [];
+        this.fileBytes = Buffer.byteLength(text, "utf8");
+        // Keyed by bank and document id in file order. A later copy wins and takes the later place:
+        // a crash between replay's batched saves leaves a delivered copy before a newer append.
+        const loaded = new Map();
         let skipped = 0;
         let tombstones = 0;
+        let duplicates = 0;
         for (const line of text.split("\n")) {
             if (!line.trim()) {
                 continue;
@@ -164,13 +184,17 @@ export class RetainQueue {
             try {
                 const entry = JSON.parse(line);
                 if (isQueuedRetain(entry)) {
-                    loaded.push(stored(entry, line));
+                    const key = queueKey(entry);
+                    if (loaded.delete(key)) {
+                        duplicates += 1;
+                    }
+                    loaded.set(key, stored(entry, line));
                     continue;
                 }
                 const dead = tombstoneKey(entry);
                 if (dead) {
-                    // Drops the earlier copies the tombstone was written for.
-                    loaded = loaded.filter((s) => s.entry.bankId !== dead.bankId || s.entry.documentId !== dead.documentId);
+                    // Drops the earlier copy the tombstone was written for.
+                    loaded.delete(queueKey(dead));
                     tombstones += 1;
                     continue;
                 }
@@ -180,11 +204,11 @@ export class RetainQueue {
             }
             skipped += 1;
         }
-        this.stored = loaded;
+        this.stored = [...loaded.values()];
         if (skipped > 0) {
             this.logger.warn?.(`memory-hindsight: retain queue skipped ${skipped} unreadable line(s)`);
         }
-        if (this.prune(false) > 0 || skipped > 0 || tombstones > 0) {
+        if (this.prune(false) > 0 || skipped > 0 || tombstones > 0 || duplicates > 0) {
             try {
                 this.persist();
             }
@@ -203,6 +227,21 @@ export class RetainQueue {
     peek() {
         return this.stored[0]?.entry;
     }
+    /**
+     * True when the item alone is over the size bound, so `enqueue` refuses it. Not reported as a
+     * drop here: the caller may still send it live, and reports it only when it is lost.
+     */
+    isTooLarge(entry) {
+        return stored({ ...entry, queuedAt: this.now() }).size > this.maxBytes();
+    }
+    /** Queued items per bank id. */
+    depthByBank() {
+        const depth = new Map();
+        for (const { entry } of this.stored) {
+            depth.set(entry.bankId, (depth.get(entry.bankId) ?? 0) + 1);
+        }
+        return depth;
+    }
     /** Bank and document id of every queued item, oldest first. */
     list() {
         return this.stored.map(({ entry }) => ({ bankId: entry.bankId, documentId: entry.documentId }));
@@ -241,24 +280,65 @@ export class RetainQueue {
         else {
             this.stored.push(next);
         }
+        let rewriteError;
         try {
-            if (this.prune(false) === 0 && index < 0 && existsSync(this.filePath)) {
+            // Drops are reported only once the write that makes them stick succeeds; the fallback below
+            // restores memory and prunes again, and reports then.
+            const pruned = this.pruneStored();
+            if (pruned.length === 0 &&
+                index < 0 &&
+                existsSync(this.filePath) &&
+                this.canAppend(next.size)) {
                 this.append(next.line);
                 return true;
             }
             this.persist();
+            this.reportDrops(pruned);
+            // False when the bound dropped the item just added or replaced.
+            return this.stored.includes(next);
         }
         catch (error) {
-            // A queued copy of the same document is out of date now (the caller sends the newer one
-            // live), so it must not be replayed over it: not from memory, and not after a restart.
-            this.stored = index >= 0 ? before.filter((_, i) => i !== index) : before;
-            if (index >= 0) {
-                this.markStale(before[index].entry);
+            rewriteError = error;
+        }
+        // The rewrite failed. load() keeps the newest copy per bank and document id, so appending the
+        // new copy also replaces an older one in the file; the byte check keeps the file bounded.
+        const kept = index >= 0 ? before.filter((_, i) => i !== index) : before;
+        // Memory becomes what load() gives for the file plus the appended text: the new copy after the
+        // older one, and a tombstone for every item the bounds drop, so no dropped line can come back
+        // on a restart (say, after a later tombstone brings the count under maxItems).
+        this.stored = [...kept, next];
+        // Reported only if the append below sticks; otherwise memory goes back to `kept`.
+        const prunedAgain = this.pruneStored();
+        const keepsNext = this.stored.includes(next);
+        const dead = kept.filter((s) => !this.stored.includes(s)).map((s) => s.entry);
+        if (!keepsNext && index >= 0) {
+            dead.push(before[index].entry);
+        }
+        const lines = dead.map((e) => tombstoneLine(e));
+        if (keepsNext) {
+            lines.push(next.line);
+        }
+        const text = lines.join("\n");
+        // One more byte for the trailing newline, as Stored.size counts it.
+        if (existsSync(this.filePath) && this.canAppend(Buffer.byteLength(text, "utf8") + 1)) {
+            try {
+                if (text) {
+                    this.append(text);
+                }
+                this.reportDrops(prunedAgain);
+                return keepsNext;
             }
-            throw error;
+            catch {
+                // Neither write worked; handled below.
+            }
+        }
+        // A queued copy of the same document is out of date now (the caller sends the newer one live),
+        // so it must not be replayed over it: not from memory, and not after a restart.
+        this.stored = kept;
+        if (index >= 0) {
+            this.markStale(before[index].entry);
         }
-        // False when the bound dropped the item just added or replaced.
-        return this.stored.includes(next);
+        throw rewriteError;
     }
     /**
      * Removes the entry when it is still the queued copy (a newer enqueue replaced it otherwise).
@@ -289,9 +369,17 @@ export class RetainQueue {
      * stays quiet until the queue is empty again.
      */
     prune(persist = true) {
+        const dropped = this.pruneStored();
+        this.reportDrops(dropped);
+        if (dropped.length > 0 && persist) {
+            this.persist();
+        }
+        return dropped.length;
+    }
+    /** The bounds step of `prune()`: updates memory and warns, but neither reports nor writes. */
+    pruneStored() {
         const { maxItems, maxAgeMs } = this.limits();
         const cutoff = this.now() - maxAgeMs;
-        const before = this.stored.length;
         let kept = this.stored.filter((s) => s.entry.queuedAt >= cutoff);
         if (kept.length > maxItems) {
             kept = kept.slice(kept.length - maxItems);
@@ -300,29 +388,34 @@ export class RetainQueue {
         while (kept.length > 0 && bytes > this.maxBytes()) {
             bytes -= kept.shift().size;
         }
+        const keptSet = new Set(kept);
+        const dropped = this.stored.filter((s) => !keptSet.has(s));
         this.stored = kept;
-        const dropped = before - kept.length;
-        if (dropped > 0) {
-            if (!this.dropWarned) {
-                this.dropWarned = true;
-                this.logger.warn?.(`memory-hindsight: retain queue dropped ${dropped} item(s) over retainQueueMaxItems (${maxItems}), retainQueueMaxAgeMs (${maxAgeMs}) or the ${this.maxBytes()}-byte size bound; further drops are not logged until the queue is empty`);
-            }
-            if (persist) {
-                this.persist();
-            }
+        if (dropped.length > 0 && !this.dropWarned) {
+            this.dropWarned = true;
+            this.logger.warn?.(`memory-hindsight: retain queue dropped ${dropped.length} item(s) over retainQueueMaxItems (${maxItems}), retainQueueMaxAgeMs (${maxAgeMs}) or the ${this.maxBytes()}-byte size bound; further drops are not logged until the queue is empty`);
         }
         return dropped;
     }
+    reportDrops(dropped) {
+        for (const s of dropped) {
+            this.onDrop?.(s.entry.bankId, "limit");
+        }
+    }
     ensureDir() {
         const dir = dirname(this.filePath);
         mkdirSync(dir, { recursive: true, mode: 0o700 });
         chmodSync(dir, 0o700);
     }
+    /** No append that would take the file past the size bound (one byte for a torn line's newline). */
+    canAppend(size) {
+        return this.fileBytes + size + 1 <= this.maxBytes();
+    }
     append(line) {
         // A torn last line (crash mid-append) must not swallow this item: start on a fresh line.
-        appendFileSync(this.filePath, `${endsWithNewline(this.filePath) ? "" : "\n"}${line}\n`, {
-            mode: 0o600,
-        });
+        const text = `${endsWithNewline(this.filePath) ? "" : "\n"}${line}\n`;
+        appendFileSync(this.filePath, text, { mode: 0o600 });
+        this.fileBytes += Buffer.byteLength(text, "utf8");
         try {
             chmodSync(this.filePath, 0o600);
         }
@@ -336,25 +429,32 @@ export class RetainQueue {
      */
     markStale(entry) {
         try {
-            this.append(JSON.stringify({ tombstone: { bankId: entry.bankId, documentId: entry.documentId } }));
+            this.append(tombstoneLine(entry));
         }
         catch (error) {
             this.logger.warn?.(`memory-hindsight: could not mark an out-of-date queued retain (${formatErrorMessage(error)}); a restart may replay it before the newer copy is written again`);
         }
     }
     persist() {
+        this.rewrite();
+        // Cleared only once written, so a failed replay save is retried by the next save().
         this.dirty = false;
+    }
+    rewrite() {
         if (this.stored.length === 0) {
             rmSync(this.filePath, { force: true });
+            this.fileBytes = 0;
             this.dropWarned = false;
             return;
         }
         this.ensureDir();
         const tmp = `${this.filePath}.tmp`;
-        writeFileSync(tmp, `${this.stored.map((s) => s.line).join("\n")}\n`, { mode: 0o600 });
+        const text = `${this.stored.map((s) => s.line).join("\n")}\n`;
+        writeFileSync(tmp, text, { mode: 0o600 });
         // writeFileSync keeps the mode of an existing file; force it.
         chmodSync(tmp, 0o600);
         renameSync(tmp, this.filePath);
+        this.fileBytes = Buffer.byteLength(text, "utf8");
     }
 }
 /**
@@ -459,6 +559,7 @@ export class RetainQueueRunner {
                     return delivered;
                 }
                 if (!isDroppingRejection(error)) {
+                    this.options.onReplay?.(entry.bankId, "deferred");
                     // Not an answer about this item (outage, auth, rate limit, ambiguous 404, refused
                     // redirect, or an unexpected error): keep it and back off.
                     if (!isTransportFailure(error) && !this.rejectionWarned) {
@@ -473,6 +574,7 @@ export class RetainQueueRunner {
                 }
                 // Hindsight answered 4xx about this item: retrying cannot help, so it leaves the queue.
                 this.queue.remove(entry, { persist: false });
+                this.options.onDrop?.(entry.bankId, "rejected");
                 logger.warn?.(`memory-hindsight: queued retain rejected and dropped: ${formatErrorMessage(error)}`);
                 continue;
             }
@@ -481,6 +583,7 @@ export class RetainQueueRunner {
                 return delivered;
             }
             this.queue.remove(entry, { persist: false });
+            this.options.onReplay?.(entry.bankId, "delivered");
             this.rejectionWarned = false;
             delivered += 1;
             if (delivered % SAVE_EVERY === 0) {

package/dist/service.js

@@ -3,6 +3,12 @@ import { boundedSignal, HindsightClient } from "./client.js";
 import { RETAIN_TIMEOUT_MS } from "./config.js";
 import { BankSetupError, isDroppingRejection, isTransportFailure, RetainQueueRunner, retainQueuePath, sharedRetainQueue, } from "./retain-queue.js";
 export function createHindsightService(params) {
+    const { metrics } = params;
+    // Queue and replay observers feed the gateway metrics; none when the caller passes no metrics.
+    const queueObserver = metrics ? { onDrop: metrics.recordDrop.bind(metrics) } : {};
+    const runnerObserver = metrics
+        ? { onDrop: metrics.recordDrop.bind(metrics), onReplay: metrics.recordReplay.bind(metrics) }
+        : {};
     const preparedBanks = new Map();
     let stopped = false;
     let runner;
@@ -71,6 +77,10 @@ export function createHindsightService(params) {
                 // Read again after the await: the service may have stopped or restarted meanwhile.
                 const target = queue;
                 if (!target || refused || !isTransportFailure(error)) {
+                    // Queue-first path: the queue refused the item and the live write failed, so it is lost.
+                    if (target && refused && isTransportFailure(error) && target.isTooLarge(entry)) {
+                        metrics?.recordDrop(bankId, "too_large");
+                    }
                     throw error;
                 }
                 let queued;
@@ -82,6 +92,11 @@ export function createHindsightService(params) {
                     throw error;
                 }
                 if (!queued) {
+                    // Lost: the live write failed and the queue refused it. A bound that dropped the new item
+                    // is already counted as `limit` by the queue.
+                    if (target.isTooLarge(entry)) {
+                        metrics?.recordDrop(bankId, "too_large");
+                    }
                     throw error;
                 }
                 params.logger.warn?.(`memory-hindsight: Hindsight unreachable, retain queued (${target.size()} pending): ${formatErrorMessage(error)}`);
@@ -109,7 +124,11 @@ export function createHindsightService(params) {
                 return { maxItems: config.retainQueueMaxItems, maxAgeMs: config.retainQueueMaxAgeMs };
             };
             try {
-                queue = sharedRetainQueue(retainQueuePath(stateDir), { limits, logger: params.logger });
+                queue = sharedRetainQueue(retainQueuePath(stateDir), {
+                    limits,
+                    logger: params.logger,
+                    ...queueObserver,
+                });
             }
             catch (error) {
                 queue = undefined;
@@ -126,9 +145,12 @@ export function createHindsightService(params) {
                         // A setup 4xx is about the bank config, not this item: keep it (see BankSetupError).
                         throw isDroppingRejection(error) ? new BankSetupError(error) : error;
                     }
-                    await client.retain(entry.bankId, entry.item, { signal });
+                    // Replay may repeat a delivered item (a crash before the batched save); a replace of
+                    // the same document id makes the repeat a no-op on the server.
+                    await client.retain(entry.bankId, { ...entry.item, document_id: entry.documentId, update_mode: "replace" }, { signal });
                 },
                 logger: params.logger,
+                ...runnerObserver,
                 ...params.retainQueueRetry,
             });
             if (queue.size() > 0) {
@@ -137,6 +159,7 @@ export function createHindsightService(params) {
             }
         },
         retainQueue: () => runner,
+        retainQueueDepth: () => queue?.depthByBank(),
         stopped: () => stopped,
         stop() {
             stopped = true;