@joe-broadhead/opencode-mem 0.9.5 → 0.9.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/CHANGELOG.md +1408 -0
  2. package/README.md +120 -41
  3. package/docs/agent-instructions.md +17 -0
  4. package/docs/agent-protocol.md +28 -27
  5. package/docs/permissions.md +150 -0
  6. package/package.json +386 -17
  7. package/skills/mem.pages/SKILL.md +40 -16
  8. package/skills/mem.sharing/SKILL.md +39 -6
  9. package/skills/mem.sources/SKILL.md +41 -10
  10. package/skills/mem.usage/SKILL.md +37 -16
  11. package/src/backend/records.js +8 -0
  12. package/src/backend/v3.js +15 -10
  13. package/src/call-shapes.js +18 -25
  14. package/src/capture-pause.js +136 -0
  15. package/src/capture.js +7 -5
  16. package/src/collector-service.js +212 -35
  17. package/src/collector.js +187 -47
  18. package/src/commands.js +37 -0
  19. package/src/compaction.js +4 -2
  20. package/src/config.js +151 -1
  21. package/src/curation.js +6 -10
  22. package/src/events.js +144 -36
  23. package/src/forget-purge.js +377 -0
  24. package/src/forgotten.js +152 -0
  25. package/src/hooks.js +9 -4
  26. package/src/index.js +5 -3
  27. package/src/permissions.js +116 -38
  28. package/src/protocol.js +32 -11
  29. package/src/recall.js +13 -1
  30. package/src/records.js +4 -1
  31. package/src/runtime.js +202 -15
  32. package/src/shapes.js +63 -0
  33. package/src/sources.js +205 -12
  34. package/src/spaces.js +58 -1
  35. package/src/spool.js +442 -64
  36. package/src/stated.js +334 -0
  37. package/src/storable.js +477 -0
  38. package/src/tools/aliases.js +36 -147
  39. package/src/tools/commitments.js +79 -13
  40. package/src/tools/held-write.js +5 -1
  41. package/src/tools/index.js +25 -43
  42. package/src/tools/ingest.js +33 -128
  43. package/src/tools/memory.js +525 -911
  44. package/src/tools/page-sections.js +147 -0
  45. package/src/tools/page.js +314 -82
  46. package/src/tools/recall-browse.js +173 -0
  47. package/src/tools/recall.js +36 -8
  48. package/src/tools/records-kit.js +2 -1
  49. package/src/tools/share.js +494 -131
  50. package/src/tools/shared.js +97 -19
  51. package/src/tools/spaces.js +194 -101
  52. package/src/tools/status.js +31 -3
  53. package/src/tools/wiki.js +5 -13
  54. package/src/util.js +10 -59
  55. package/src/views/contracts.js +258 -20
  56. package/src/views/overdue.js +1 -1
  57. package/src/views/profile.js +2 -3
  58. package/src/views/receipt.js +81 -0
  59. package/src/views/records-reads.js +545 -25
  60. package/src/views/rpc.js +41 -3
  61. package/views/memory.html +16 -3
  62. package/views/remembered.html +130 -0
  63. package/src/erase.js +0 -36
package/src/spool.js CHANGED
@@ -14,11 +14,36 @@ const BACKOFF_MAX_MS = 60 * 60_000
14
14
  const CLAIM_STALE_MS = 5 * 60_000
15
15
  /** Stop a flush after this many consecutive "backend unreachable" failures. */
16
16
  const UNREACHABLE_STOP = 3
17
+ /** A quarantined entry is deleted this long after it was quarantined. */
18
+ export const QUARANTINE_TTL_MS = 14 * 86_400_000
19
+ /** A temp file older than this was left by a crashed write. */
20
+ const TEMP_STALE_MS = CLAIM_STALE_MS
17
21
 
18
22
  const PENDING = ".json"
19
23
  const INFLIGHT = ".inflight"
20
24
  const FAILED = ".failed"
25
+ /** `<name>.json.state`: an entry's retry and quarantine state, beside its payload. */
26
+ const STATE = ".state"
27
+ const TEMP = ".tmp"
21
28
  const CLAIM_SUFFIX = /\.(\d+)-(\d+)\.inflight$/
29
+ /** The retry fields a 0.9.7 or older entry kept inside its payload. */
30
+ const LEGACY_STATE_FIELDS = ["attempts", "lastError", "lastStatus", "lastAttemptAt", "nextAttemptAt", "quarantine"]
31
+
32
+ /**
33
+ * A captured turn's idempotency key: its session, its message, and a hash of
34
+ * the payload. The live send and every replay of one spooled turn send the
35
+ * same key, and the gateway stores a key once (docs/capture-delivery.md).
36
+ * Only characters the gateway accepts are kept.
37
+ *
38
+ * @param {string} sessionID
39
+ * @param {string | null | undefined} messageID
40
+ * @param {unknown} payload
41
+ */
42
+ export function turnKey(sessionID, messageID, payload) {
43
+ const safe = (value) => String(value ?? "").replace(/[^A-Za-z0-9._:-]/g, "_").slice(0, 190)
44
+ const session = safe(sessionID) || "session"
45
+ return `${/^[A-Za-z0-9]/.test(session) ? "" : "s"}${session}:${safe(messageID) || "-"}:${shortHash(JSON.stringify(payload ?? null), 16)}`
46
+ }
22
47
 
23
48
  /**
24
49
  * Classify a failed send.
@@ -65,9 +90,27 @@ function backoffMs(attempts) {
65
90
  * atomic rename, so two processes, or a flush racing a live capture, never
66
91
  * send the same entry twice. A claim older than five minutes is returned to
67
92
  * pending.
68
- * - `<name>.json.failed` is quarantined. It holds the entry plus
69
- * `quarantine: { reason, at }`. Rename it back to `.json` to retry it; the
70
- * attempt count restarts.
93
+ * - `<name>.json.failed` is quarantined; the reason is in its state file.
94
+ * Rename it back to `.json` to retry it; the attempt count restarts. A
95
+ * quarantined entry is deleted 14 days after it was quarantined
96
+ * (`QUARANTINE_TTL_MS`), and `spoolStats` lists the reasons for status.
97
+ * - `<name>.json.state` holds the entry's retry state (attempts, next attempt,
98
+ * last error) and, once quarantined, `quarantine: { reason, at }`.
99
+ *
100
+ * **Durability.** A payload file is written once and then only renamed: to
101
+ * a claim, back to pending, or to quarantine. Retry state never touches it.
102
+ * Every write (a payload or a state file) goes to a temp file that is fsynced
103
+ * and then renamed into place, and the directory is fsynced, so a crash leaves
104
+ * the old file or the new one, never a torn one. A temp file a crash left
105
+ * behind is ignored and later deleted.
106
+ *
107
+ * **Delivery.** At least once: a send whose answer is lost (a timeout, a
108
+ * crash, a claim recovered from a sleeping process) is sent again. A turn
109
+ * carries an idempotency key (`turnKey`), so the gateway answers a repeat
110
+ * from its receipt instead of storing it twice. After a successful send the
111
+ * entry is deleted in its own step: a delete that fails (another process
112
+ * already recovered the claim) never puts the entry back in the queue
113
+ * (docs/capture-delivery.md).
71
114
  *
72
115
  * In control mode a turn entry records the session, agent, task, and the
73
116
  * visibility it was captured with, never a container. Its replay goes through
@@ -91,7 +134,10 @@ function backoffMs(attempts) {
91
134
  * replay (another person spoke there since), it is kept for later, never sent
92
135
  * with that token.
93
136
  *
94
- * @param {{ config: any, log: any, backend: any, captureTurn?: Function | null, now?: () => number }} input
137
+ * **Forget.** `purgeMatching` deletes entries that hold forgotten text, and
138
+ * `sendable` (src/forget-purge.js) is asked again just before each replay.
139
+ *
140
+ * @param {{ config: any, log: any, backend: any, captureTurn?: Function | null, now?: () => number, sendable?: (entry: any) => boolean | "hold" }} input
95
141
  */
96
142
  export function createSpool(input) {
97
143
  const { config, log, backend } = input
@@ -101,6 +147,12 @@ export function createSpool(input) {
101
147
  const subject = typeof input.subject === "function" ? input.subject : () => ""
102
148
  /** The account the current session's token names now, asking Jelly when needed: checked before a replay. */
103
149
  const speaker = typeof input.speaker === "function" ? input.speaker : async () => ""
150
+ /**
151
+ * Whether an entry may be sent: false when it holds text the person forgot
152
+ * (src/forget-purge.js), deleted, never sent; "hold" when its space has a
153
+ * forget pending, kept for a later replay. Checked just before each replay.
154
+ */
155
+ const sendable = typeof input.sendable === "function" ? input.sendable : () => true
104
156
  const spoolDir = path.join(config.stateDir, "spool")
105
157
  const claimsDir = path.join(config.stateDir, "claims")
106
158
  let sequence = 0
@@ -114,12 +166,97 @@ export function createSpool(input) {
114
166
  }
115
167
  }
116
168
 
117
- function writePrivate(file, record, flag = "w") {
118
- fs.writeFileSync(file, JSON.stringify(record, null, 2), { mode: 0o600, flag })
169
+ /** Make a rename or a new file in the spool survive a crash. Best effort. */
170
+ function syncDir() {
171
+ let fd
119
172
  try {
120
- fs.chmodSync(file, 0o600)
173
+ fd = fs.openSync(spoolDir, "r")
174
+ fs.fsyncSync(fd)
121
175
  } catch {
122
- /* best effort on platforms without POSIX modes */
176
+ /* not supported on every platform (Windows) */
177
+ } finally {
178
+ if (fd !== undefined) {
179
+ try {
180
+ fs.closeSync(fd)
181
+ } catch {
182
+ /* ignore */
183
+ }
184
+ }
185
+ }
186
+ }
187
+
188
+ /**
189
+ * Write a file so a crash leaves either the old content or the new, never
190
+ * a torn file: a private temp file in the same directory, written in full,
191
+ * fsynced, then renamed over the target, then the directory fsynced.
192
+ */
193
+ function writeDurable(file, record) {
194
+ const bytes = Buffer.from(JSON.stringify(record, null, 2))
195
+ const temp = path.join(path.dirname(file), `.${path.basename(file)}.${process.pid}-${Math.random().toString(36).slice(2, 10)}${TEMP}`)
196
+ let fd
197
+ try {
198
+ fd = fs.openSync(temp, "wx", 0o600)
199
+ try {
200
+ fs.fchmodSync(fd, 0o600)
201
+ } catch {
202
+ /* best effort on platforms without POSIX modes */
203
+ }
204
+ let offset = 0
205
+ while (offset < bytes.length) offset += fs.writeSync(fd, bytes, offset, bytes.length - offset)
206
+ fs.fsyncSync(fd)
207
+ fs.closeSync(fd)
208
+ fd = undefined
209
+ fs.renameSync(temp, file)
210
+ } catch (error) {
211
+ if (fd !== undefined) {
212
+ try {
213
+ fs.closeSync(fd)
214
+ } catch {
215
+ /* ignore */
216
+ }
217
+ }
218
+ try {
219
+ fs.unlinkSync(temp)
220
+ } catch {
221
+ /* never created, or already renamed */
222
+ }
223
+ throw error
224
+ }
225
+ syncDir()
226
+ }
227
+
228
+ function stateFile(pending) {
229
+ return `${pending}${STATE}`
230
+ }
231
+
232
+ /** An entry's sidecar state, or null when it has none (or it is unreadable). */
233
+ function readState(pending) {
234
+ try {
235
+ const state = JSON.parse(fs.readFileSync(stateFile(pending), "utf8"))
236
+ return state && typeof state === "object" && !Array.isArray(state) ? state : null
237
+ } catch {
238
+ return null
239
+ }
240
+ }
241
+
242
+ /** The retry fields an entry from 0.9.7 or older kept in its payload. */
243
+ function legacyState(entry) {
244
+ const state = {}
245
+ if (!entry || typeof entry !== "object") return state
246
+ for (const field of LEGACY_STATE_FIELDS) if (entry[field] !== undefined) state[field] = entry[field]
247
+ return state
248
+ }
249
+
250
+ /** The state that counts: the sidecar when there is one, else the legacy fields. */
251
+ function stateOf(pending, entry) {
252
+ return readState(pending) ?? legacyState(entry)
253
+ }
254
+
255
+ function removeState(pending) {
256
+ try {
257
+ fs.unlinkSync(stateFile(pending))
258
+ } catch {
259
+ /* none */
123
260
  }
124
261
  }
125
262
 
@@ -153,7 +290,7 @@ export function createSpool(input) {
153
290
  try {
154
291
  ensureDir()
155
292
  const file = path.join(spoolDir, newName(data))
156
- writePrivate(file, entryOf(kind, data, scope, meta), "wx")
293
+ writeDurable(file, entryOf(kind, data, scope, meta))
157
294
  return file
158
295
  } catch (error) {
159
296
  log.error("spool write failed", errorMessage(error))
@@ -187,7 +324,7 @@ export function createSpool(input) {
187
324
  try {
188
325
  ensureDir()
189
326
  const file = claimName(path.join(spoolDir, newName(data)))
190
- writePrivate(file, entryOf(kind, data, scope, meta), "wx")
327
+ writeDurable(file, entryOf(kind, data, scope, meta))
191
328
  return file
192
329
  } catch (error) {
193
330
  log.error("spool write failed", errorMessage(error))
@@ -195,13 +332,31 @@ export function createSpool(input) {
195
332
  }
196
333
  }
197
334
 
335
+ /**
336
+ * Delete a sent entry and its state. A missing file is not an error: another
337
+ * process may have recovered the claim (and will send it again, which the
338
+ * gateway dedupes by key). Never re-creates anything.
339
+ *
340
+ * @returns {boolean} whether the entry is gone
341
+ */
198
342
  function removeSpool(file) {
199
- if (!file) return
343
+ if (!file) return true
344
+ let gone = true
200
345
  try {
201
346
  fs.unlinkSync(file)
202
- } catch {
203
- /* ignore */
347
+ } catch (error) {
348
+ if (error?.code === "ENOENT") {
349
+ log.debug("sent spool entry was already gone: another process recovered its claim", { file: path.basename(pendingOf(file)) })
350
+ } else {
351
+ gone = false
352
+ log.warn("sent spool entry could not be deleted; a later replay is deduplicated by its key", {
353
+ file: path.basename(pendingOf(file)),
354
+ error: error?.code ?? errorMessage(error),
355
+ })
356
+ }
204
357
  }
358
+ if (gone) removeState(pendingOf(file))
359
+ return gone
205
360
  }
206
361
 
207
362
  /** Atomically take a pending entry. Null when another sender got it first. */
@@ -216,23 +371,53 @@ export function createSpool(input) {
216
371
  }
217
372
  }
218
373
 
219
- /** Move an entry aside with the reason so the rest of the queue drains. */
220
- function quarantine(claimed, entry, reason, raw) {
221
- const target = `${pendingOf(claimed)}${FAILED}`
374
+ /**
375
+ * Move an entry aside with the reason so the rest of the queue drains. The
376
+ * payload is renamed as it is (an unreadable one keeps its bytes); the
377
+ * reason goes to its state file. An entry that is no longer ours (another
378
+ * process recovered the claim) is left alone.
379
+ *
380
+ * **The order matters.** The payload moves first and the reason follows. A
381
+ * `quarantine` field in an entry's state means the entry was in quarantine,
382
+ * and an operator who renames it back to `.json` restarts its attempt count.
383
+ * Written the other way round, a crash between the two steps leaves a claim
384
+ * whose state says "quarantined": the next start recovers the claim, sees
385
+ * that field, and takes the entry for a manual retry, resetting its attempts.
386
+ * In this order, a crash between the steps leaves a `.failed` entry without a
387
+ * recorded reason (status says so), and a crash before them leaves a claim
388
+ * with the state it had, which resumes where it was.
389
+ */
390
+ function quarantine(claimed, reason, state = {}) {
391
+ const pending = pendingOf(claimed)
392
+ const target = `${pending}${FAILED}`
393
+ if (!fs.existsSync(claimed)) {
394
+ log.debug("spool entry not quarantined: another process holds it now", { file: path.basename(pending) })
395
+ return false
396
+ }
222
397
  const at = new Date(now()).toISOString()
223
- const record =
224
- entry && typeof entry === "object" && !Array.isArray(entry)
225
- ? { ...entry, quarantine: { reason, at } }
226
- : { quarantine: { reason, at }, raw: String(raw ?? "") }
227
398
  try {
228
- writePrivate(claimed, record)
229
399
  fs.renameSync(claimed, target)
230
- log.warn("spool entry quarantined", { file: path.basename(target), reason })
231
- return true
232
400
  } catch (error) {
233
401
  log.warn("spool quarantine failed", { file: path.basename(claimed), error: errorMessage(error) })
234
402
  return false
235
403
  }
404
+ try {
405
+ // Permission and time changes, not rewrites: the entry holds memory
406
+ // text, and its age in quarantine is its file's time if its reason
407
+ // never gets written.
408
+ fs.chmodSync(target, 0o600)
409
+ fs.utimesSync(target, now() / 1000, now() / 1000)
410
+ } catch {
411
+ /* best effort on platforms without POSIX modes */
412
+ }
413
+ try {
414
+ writeDurable(stateFile(pending), { ...state, quarantine: { reason, at } })
415
+ } catch (error) {
416
+ log.warn("spool quarantine reason not written", { file: path.basename(pending), error: errorMessage(error) })
417
+ }
418
+ syncDir()
419
+ log.warn("spool entry quarantined", { file: path.basename(target), reason })
420
+ return true
236
421
  }
237
422
 
238
423
  /**
@@ -240,17 +425,20 @@ export function createSpool(input) {
240
425
  * an exhausted entry; otherwise schedule the next attempt and return the
241
426
  * entry to the queue.
242
427
  *
243
- * @returns {"quarantined" | "kept"}
428
+ * The payload is never rewritten: the new state goes to the entry's state
429
+ * file (written atomically), then the claim is renamed back to pending. A
430
+ * claim that is gone (another process recovered it) is not re-created.
431
+ *
432
+ * @returns {"quarantined" | "kept" | "gone"}
244
433
  */
245
- function recordFailure(claimed, entry, error) {
434
+ function recordFailure(claimed, entry, error, state = {}) {
246
435
  const kind = classifyFailure(error)
247
436
  const status = Number(error?.status) || 0
248
437
  const message = oneLine(redactSecrets(errorMessage(error)), 300)
249
- const attempts = (Number(entry.attempts) || 0) + 1
438
+ const attempts = (Number(state.attempts) || 0) + 1
250
439
  const queuedAt = Date.parse(entry.queuedAt ?? "")
251
440
  const age = Number.isFinite(queuedAt) ? now() - queuedAt : 0
252
441
  const updated = {
253
- ...entry,
254
442
  attempts,
255
443
  lastError: message,
256
444
  lastStatus: status,
@@ -262,12 +450,26 @@ export function createSpool(input) {
262
450
  if (kind === "permanent") reason = `rejected (${status}): ${message}`
263
451
  else if (attempts >= MAX_ATTEMPTS) reason = `gave up after ${attempts} attempts: ${message}`
264
452
  else if (age >= MAX_AGE_MS) reason = `gave up after ${Math.floor(age / 86_400_000)} days: ${message}`
265
- if (reason && quarantine(claimed, updated, reason)) return "quarantined"
453
+ if (reason && quarantine(claimed, reason, updated)) return "quarantined"
266
454
 
455
+ const pending = pendingOf(claimed)
456
+ if (!fs.existsSync(claimed)) {
457
+ log.warn("spool send failed; another process holds the entry now, so it is not re-queued here", { kind, status })
458
+ return "gone"
459
+ }
267
460
  try {
268
- writePrivate(claimed, updated)
269
- fs.renameSync(claimed, pendingOf(claimed))
461
+ writeDurable(stateFile(pending), updated)
462
+ } catch (stateError) {
463
+ // The entry is still returned to the queue, only without its backoff.
464
+ log.warn("spool retry state not written", { file: path.basename(pending), error: errorMessage(stateError) })
465
+ }
466
+ try {
467
+ fs.renameSync(claimed, pending)
270
468
  } catch (releaseError) {
469
+ if (releaseError?.code === "ENOENT") {
470
+ log.warn("spool send failed; another process holds the entry now, so it is not re-queued here", { kind, status })
471
+ return "gone"
472
+ }
271
473
  log.warn("spool release failed", { file: path.basename(claimed), error: errorMessage(releaseError) })
272
474
  }
273
475
  log.warn("spool send failed; kept", { kind, status, attempts, error: message })
@@ -284,6 +486,7 @@ export function createSpool(input) {
284
486
  } catch {
285
487
  /* already gone */
286
488
  }
489
+ removeState(pendingOf(file))
287
490
  log.warn("spooled entry dropped: the person speaking in its session hasn't linked a Jelly account on this host, and no other credential may send it", {
288
491
  file: path.basename(pendingOf(file)),
289
492
  kind: entry?.kind ?? null,
@@ -305,7 +508,7 @@ export function createSpool(input) {
305
508
  }
306
509
  try {
307
510
  const entry = JSON.parse(fs.readFileSync(claimed, "utf8"))
308
- return recordFailure(claimed, entry, error)
511
+ return recordFailure(claimed, entry, error, stateOf(pendingOf(claimed), entry))
309
512
  } catch (readError) {
310
513
  log.warn("spool release failed", { file: path.basename(claimed), error: errorMessage(readError) })
311
514
  return null
@@ -437,6 +640,113 @@ export function createSpool(input) {
437
640
  }
438
641
  }
439
642
 
643
+ /** When a quarantined entry was quarantined: its state, a legacy record, else the file's time. */
644
+ function quarantinedAt(failed) {
645
+ const pending = pendingOf(failed)
646
+ let at = Date.parse(readState(pending)?.quarantine?.at ?? "")
647
+ if (!Number.isFinite(at)) {
648
+ try {
649
+ at = Date.parse(JSON.parse(fs.readFileSync(failed, "utf8"))?.quarantine?.at ?? "")
650
+ } catch {
651
+ at = Number.NaN
652
+ }
653
+ }
654
+ if (!Number.isFinite(at)) {
655
+ try {
656
+ at = fs.statSync(failed).mtimeMs
657
+ } catch {
658
+ at = now()
659
+ }
660
+ }
661
+ return at
662
+ }
663
+
664
+ /**
665
+ * Delete what nothing will read again: quarantined entries past
666
+ * `QUARANTINE_TTL_MS`, state files whose entry is gone, and temp files a
667
+ * crashed write left behind.
668
+ */
669
+ function housekeep(names) {
670
+ const present = new Set()
671
+ for (const name of names) {
672
+ if (name.endsWith(PENDING) || name.endsWith(FAILED) || name.endsWith(INFLIGHT)) present.add(path.basename(pendingOf(name)))
673
+ }
674
+ let expired = 0
675
+ for (const name of names) {
676
+ const file = path.join(spoolDir, name)
677
+ try {
678
+ if (name.endsWith(FAILED)) {
679
+ if (now() - quarantinedAt(file) < QUARANTINE_TTL_MS) continue
680
+ fs.unlinkSync(file)
681
+ removeState(pendingOf(file))
682
+ expired += 1
683
+ } else if (name.endsWith(STATE)) {
684
+ if (!present.has(name.slice(0, -STATE.length))) fs.unlinkSync(file)
685
+ } else if (name.startsWith(".") && name.endsWith(TEMP)) {
686
+ if (now() - fs.statSync(file).mtimeMs >= TEMP_STALE_MS) fs.unlinkSync(file)
687
+ }
688
+ } catch {
689
+ /* another process got there first */
690
+ }
691
+ }
692
+ if (expired > 0) log.info("expired quarantined spool entries deleted", { count: expired, afterDays: QUARANTINE_TTL_MS / 86_400_000 })
693
+ return expired
694
+ }
695
+
696
+ /**
697
+ * Delete every queued, in-flight, or quarantined entry the predicate
698
+ * matches, with its state. The hook for forget (slice 6b): it decides what
699
+ * matches; this only finds and deletes. An entry that is being sent is
700
+ * deleted too; its sender then neither re-queues nor quarantines it.
701
+ *
702
+ * @param {(entry: any, info: { state: "pending" | "inflight" | "quarantined", file: string, raw: string }) => boolean} predicate
703
+ * `entry` is null when the file is not readable JSON; `raw` is its text.
704
+ * @returns {{ purged: number, failed: number }}
705
+ */
706
+ function purgeMatching(predicate) {
707
+ const result = { purged: 0, failed: 0 }
708
+ if (typeof predicate !== "function") return result
709
+ let names = []
710
+ try {
711
+ names = fs.readdirSync(spoolDir)
712
+ } catch {
713
+ return result
714
+ }
715
+ for (const name of names) {
716
+ const state = name.endsWith(PENDING) ? "pending" : name.endsWith(INFLIGHT) ? "inflight" : name.endsWith(FAILED) ? "quarantined" : null
717
+ if (!state) continue
718
+ const file = path.join(spoolDir, name)
719
+ let raw = ""
720
+ let entry = null
721
+ try {
722
+ raw = fs.readFileSync(file, "utf8")
723
+ entry = JSON.parse(raw)
724
+ } catch (error) {
725
+ if (error?.code === "ENOENT") continue
726
+ }
727
+ let matched = false
728
+ try {
729
+ matched = Boolean(predicate(entry, { state, file: name, raw }))
730
+ } catch (error) {
731
+ result.failed += 1
732
+ log.warn("spool purge predicate failed", { file: path.basename(pendingOf(name)), error: errorMessage(error) })
733
+ continue
734
+ }
735
+ if (!matched) continue
736
+ try {
737
+ fs.unlinkSync(file)
738
+ removeState(pendingOf(file))
739
+ result.purged += 1
740
+ } catch (error) {
741
+ if (error?.code === "ENOENT") continue
742
+ result.failed += 1
743
+ log.warn("spool purge failed", { file: path.basename(pendingOf(name)), error: error?.code ?? errorMessage(error) })
744
+ }
745
+ }
746
+ if (result.purged > 0) syncDir()
747
+ return result
748
+ }
749
+
440
750
  async function send(entry) {
441
751
  // Someone else may speak in the session now. An entry captured under one
442
752
  // person's token is never sent with another's: it waits, with backoff,
@@ -456,12 +766,15 @@ export function createSpool(input) {
456
766
  // was captured with. An entry without one (written before visibility
457
767
  // was recorded) replays as private, the narrower choice. A container
458
768
  // in an older entry's scope is never sent.
769
+ // The key the turn was captured with; an entry from before 0.9.8
770
+ // gets one from its content, the same on every replay.
459
771
  await captureTurn({
460
772
  sessionID: turnSessionOf(entry),
461
773
  agent: entry.agent ?? entry.scope?.agent_id,
462
774
  visibility: entry.visibility === "team" ? "team" : "private",
463
775
  task: entry.task ?? entry.scope?.task_id,
464
776
  payload: entry.data,
777
+ key: typeof entry.key === "string" && entry.key ? entry.key : turnKey(turnSessionOf(entry), null, entry.data),
465
778
  })
466
779
  return
467
780
  }
@@ -488,6 +801,7 @@ export function createSpool(input) {
488
801
  return result
489
802
  }
490
803
  recoverStaleClaims(names)
804
+ housekeep(names)
491
805
  try {
492
806
  names = fs.readdirSync(spoolDir).filter((name) => name.endsWith(PENDING)).sort()
493
807
  } catch {
@@ -507,8 +821,10 @@ export function createSpool(input) {
507
821
  } catch (error) {
508
822
  if (error?.code === "ENOENT") continue
509
823
  }
510
- const requeued = Boolean(peeked?.quarantine)
511
- if (!requeued && !options.force && peeked?.nextAttemptAt && Date.parse(peeked.nextAttemptAt) > now()) {
824
+ let state = stateOf(file, peeked)
825
+ // Quarantine state on a pending entry: an operator renamed it back.
826
+ const requeued = Boolean(state.quarantine)
827
+ if (!requeued && !options.force && state.nextAttemptAt && Date.parse(state.nextAttemptAt) > now()) {
512
828
  result.deferred += 1
513
829
  continue
514
830
  }
@@ -516,61 +832,102 @@ export function createSpool(input) {
516
832
  const claimed = claim(file)
517
833
  if (!claimed) continue
518
834
 
519
- let raw = ""
520
835
  let entry
521
836
  try {
522
- raw = fs.readFileSync(claimed, "utf8")
523
- entry = JSON.parse(raw)
524
- if (entry?.quarantine) {
525
- // An operator renamed a quarantined entry back: start it afresh.
526
- entry = { ...entry, attempts: 0 }
527
- delete entry.quarantine
528
- delete entry.nextAttemptAt
529
- }
837
+ entry = JSON.parse(fs.readFileSync(claimed, "utf8"))
530
838
  } catch (error) {
531
839
  result.failed += 1
532
- if (quarantine(claimed, null, `unreadable: ${errorMessage(error)}`, raw)) result.quarantined += 1
840
+ if (quarantine(claimed, `unreadable: ${errorMessage(error)}`, state)) result.quarantined += 1
533
841
  continue
534
842
  }
843
+ // A requeued entry starts afresh.
844
+ if (requeued) state = { attempts: 0 }
535
845
  const invalid = invalidEntry(entry)
536
846
  if (invalid) {
537
847
  result.failed += 1
538
- if (quarantine(claimed, entry, `invalid: ${invalid}`)) result.quarantined += 1
848
+ if (quarantine(claimed, `invalid: ${invalid}`, state)) result.quarantined += 1
849
+ continue
850
+ }
851
+
852
+ let keep = true
853
+ try {
854
+ keep = sendable(entry)
855
+ } catch (error) {
856
+ log.warn("spool forget check failed; the entry is kept for later", { file: path.basename(pendingOf(claimed)), error: errorMessage(error) })
857
+ recordFailure(claimed, entry, Object.assign(new Error("forget check failed"), { status: 0 }), state)
858
+ continue
859
+ }
860
+ if (keep === "hold") {
861
+ // Its space has a forget whose entries are not read yet: it waits,
862
+ // with no attempt counted, and the rest of the queue drains.
863
+ try {
864
+ fs.renameSync(claimed, pendingOf(claimed))
865
+ } catch {
866
+ /* another process holds it now */
867
+ }
868
+ result.deferred += 1
869
+ continue
870
+ }
871
+ if (keep === false) {
872
+ // Text the person forgot after it was queued: deleted, never sent.
873
+ removeSpool(claimed)
874
+ result.purged = (result.purged ?? 0) + 1
875
+ log.info("spooled entry holding forgotten text deleted before replay", { file: path.basename(pendingOf(claimed)) })
539
876
  continue
540
877
  }
541
878
 
542
879
  sends += 1
880
+ let failure = null
543
881
  try {
544
882
  // As the session the entry was captured in: its token, no other.
545
883
  await asSession(credentialSessionOf(entry), () => send(entry))
546
- fs.unlinkSync(claimed)
884
+ } catch (error) {
885
+ failure = error ?? new Error("send failed")
886
+ }
887
+
888
+ if (!failure) {
889
+ // The send succeeded. The delete is its own step: if it fails (another
890
+ // process recovered the claim) the entry is not re-queued or rewritten
891
+ // here; a later replay is deduplicated by its key.
892
+ removeSpool(claimed)
547
893
  result.flushed += 1
548
894
  unreachable = 0
549
- } catch (error) {
550
- if (isUnlinked(error)) {
551
- dropUnlinked(claimed, entry)
552
- result.dropped = (result.dropped ?? 0) + 1
553
- continue
554
- }
555
- result.failed += 1
556
- if (recordFailure(claimed, entry, error) === "quarantined") result.quarantined += 1
557
- if (classifyFailure(error) === "unreachable") {
558
- unreachable += 1
559
- if (unreachable >= UNREACHABLE_STOP) {
560
- log.warn("spool flush stopped: backend unreachable", { sent: sends })
561
- break
562
- }
563
- } else {
564
- unreachable = 0
895
+ continue
896
+ }
897
+
898
+ if (isUnlinked(failure)) {
899
+ dropUnlinked(claimed, entry)
900
+ result.dropped = (result.dropped ?? 0) + 1
901
+ continue
902
+ }
903
+ result.failed += 1
904
+ if (recordFailure(claimed, entry, failure, state) === "quarantined") result.quarantined += 1
905
+ if (classifyFailure(failure) === "unreachable") {
906
+ unreachable += 1
907
+ if (unreachable >= UNREACHABLE_STOP) {
908
+ log.warn("spool flush stopped: backend unreachable", { sent: sends })
909
+ break
565
910
  }
911
+ } else {
912
+ unreachable = 0
566
913
  }
567
914
  }
568
915
  return result
569
916
  }
570
917
 
571
- /** Queue depth by state, for mem_status and /mem-sync. */
918
+ /**
919
+ * Queue depth by state, for mem_status and /mem-sync. `reasons` lists the
920
+ * newest quarantine reasons (at most five), each with when it expires.
921
+ */
572
922
  function spoolStats() {
573
- const stats = { pending: 0, inflight: 0, quarantined: 0, oldest: null }
923
+ const stats = {
924
+ pending: 0,
925
+ inflight: 0,
926
+ quarantined: 0,
927
+ oldest: null,
928
+ reasons: [],
929
+ quarantineDays: QUARANTINE_TTL_MS / 86_400_000,
930
+ }
574
931
  let names = []
575
932
  try {
576
933
  names = fs.readdirSync(spoolDir)
@@ -578,15 +935,35 @@ export function createSpool(input) {
578
935
  return stats
579
936
  }
580
937
  let oldest = Number.POSITIVE_INFINITY
938
+ const quarantined = []
581
939
  for (const name of names) {
582
940
  if (name.endsWith(PENDING)) {
583
941
  stats.pending += 1
584
942
  const written = Number.parseInt(name, 10)
585
943
  if (Number.isFinite(written) && written < oldest) oldest = written
586
944
  } else if (name.endsWith(INFLIGHT)) stats.inflight += 1
587
- else if (name.endsWith(FAILED)) stats.quarantined += 1
945
+ else if (name.endsWith(FAILED)) {
946
+ const file = path.join(spoolDir, name)
947
+ const at = quarantinedAt(file)
948
+ if (now() - at >= QUARANTINE_TTL_MS) continue
949
+ stats.quarantined += 1
950
+ let reason = readState(pendingOf(file))?.quarantine?.reason
951
+ if (typeof reason !== "string") {
952
+ try {
953
+ reason = JSON.parse(fs.readFileSync(file, "utf8"))?.quarantine?.reason
954
+ } catch {
955
+ reason = undefined
956
+ }
957
+ }
958
+ quarantined.push({
959
+ at: new Date(at).toISOString(),
960
+ reason: oneLine(typeof reason === "string" ? reason : "reason not recorded (the process stopped while quarantining it)", 160),
961
+ expires: new Date(at + QUARANTINE_TTL_MS).toISOString(),
962
+ })
963
+ }
588
964
  }
589
965
  if (Number.isFinite(oldest)) stats.oldest = new Date(oldest).toISOString()
966
+ stats.reasons = quarantined.sort((a, b) => b.at.localeCompare(a.at)).slice(0, 5)
590
967
  return stats
591
968
  }
592
969
 
@@ -603,5 +980,6 @@ export function createSpool(input) {
603
980
  enqueueSpool,
604
981
  flushSpool,
605
982
  spoolStats,
983
+ purgeMatching,
606
984
  }
607
985
  }