@joe-broadhead/opencode-mem 0.9.2 → 0.9.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/CHANGELOG.md +1605 -0
  2. package/README.md +135 -43
  3. package/docs/agent-protocol.md +29 -28
  4. package/docs/permissions.md +148 -0
  5. package/package.json +277 -16
  6. package/skills/mem.pages/SKILL.md +35 -16
  7. package/skills/mem.sharing/SKILL.md +39 -6
  8. package/skills/mem.sources/SKILL.md +41 -10
  9. package/skills/mem.usage/SKILL.md +47 -16
  10. package/src/backend/records.js +413 -50
  11. package/src/backend/v3.js +15 -10
  12. package/src/call-shapes.js +18 -25
  13. package/src/capture-pause.js +136 -0
  14. package/src/capture.js +7 -4
  15. package/src/collector-service.js +212 -35
  16. package/src/collector.js +186 -47
  17. package/src/commands.js +37 -0
  18. package/src/compaction.js +4 -2
  19. package/src/config.js +162 -1
  20. package/src/connect.js +353 -0
  21. package/src/copies.js +28 -3
  22. package/src/credentials.js +35 -7
  23. package/src/curation.js +33 -14
  24. package/src/events.js +144 -36
  25. package/src/forget-purge.js +377 -0
  26. package/src/forgotten.js +356 -0
  27. package/src/hooks.js +9 -4
  28. package/src/index.js +25 -6
  29. package/src/permissions.js +116 -38
  30. package/src/protocol.js +21 -11
  31. package/src/records.js +4 -1
  32. package/src/runtime.js +207 -16
  33. package/src/shapes.js +63 -0
  34. package/src/sources.js +183 -12
  35. package/src/spaces.js +58 -1
  36. package/src/spool.js +442 -64
  37. package/src/stated.js +334 -0
  38. package/src/storable.js +477 -0
  39. package/src/tools/aliases.js +36 -147
  40. package/src/tools/commitments.js +69 -12
  41. package/src/tools/held-write.js +9 -1
  42. package/src/tools/index.js +29 -44
  43. package/src/tools/ingest.js +33 -128
  44. package/src/tools/memory.js +594 -901
  45. package/src/tools/page-sections.js +147 -0
  46. package/src/tools/page.js +314 -82
  47. package/src/tools/recall-browse.js +173 -0
  48. package/src/tools/recall.js +79 -19
  49. package/src/tools/share.js +494 -131
  50. package/src/tools/shared.js +97 -19
  51. package/src/tools/spaces.js +215 -104
  52. package/src/tools/status.js +31 -3
  53. package/src/tools/wiki.js +5 -13
  54. package/src/util.js +10 -20
  55. package/src/views/contracts.js +137 -6
  56. package/src/views/overdue.js +125 -0
  57. package/src/views/profile.js +154 -0
  58. package/src/views/records-reads.js +132 -33
  59. package/src/views/rpc.js +63 -12
  60. package/src/views/shape.js +49 -0
  61. package/views/commitments.html +31 -4
  62. package/views/home.html +22 -3
  63. package/views/memory.html +126 -24
  64. package/src/erase.js +0 -36
package/src/events.js CHANGED
@@ -2,7 +2,9 @@ import { buildSessionPayload, extractCompactionSummary, newTurn } from "./captur
2
2
  import { UNLINKED_SENTENCE, asSession } from "./credentials.js"
3
3
  import { CAPTURE_MESSAGE_CHARS, MAX_MESSAGES, splitContent } from "./limits.js"
4
4
  import { parentIDOf, readSession } from "./sessions.js"
5
- import { errorMessage, redactSecrets, rememberBounded, shortHash } from "./util.js"
5
+ import { turnKey } from "./spool.js"
6
+ import { storable } from "./storable.js"
7
+ import { errorMessage, rememberBounded, shortHash } from "./util.js"
6
8
 
7
9
  /** Bounds on the per-process maps; old sessions fall out in insertion order. */
8
10
  const COMPACTED_MAX = 200
@@ -18,12 +20,13 @@ const SUMMARY_PART_CHARS = CAPTURE_MESSAGE_CHARS - 80
18
20
  * A host compaction summary as L0 user messages. A summary longer than the
19
21
  * upstream per-message limit is split, not truncated: it is the one record of
20
22
  * the compacted context, and about a quarter of summaries used to be rejected
21
- * whole. Credential-shaped values are redacted first.
23
+ * whole. As every automatic write (src/storable.js), lines holding a secret
24
+ * are withheld and emails and local paths masked first.
22
25
  *
23
26
  * @param {string} text
24
27
  */
25
28
  export function summaryMessages(text) {
26
- const parts = splitContent(redactSecrets(text), SUMMARY_PART_CHARS).slice(0, MAX_MESSAGES - 1)
29
+ const parts = splitContent(storable(text, "summary").text, SUMMARY_PART_CHARS).slice(0, MAX_MESSAGES - 1)
27
30
  return parts.map((part, index) => ({
28
31
  role: "user",
29
32
  content:
@@ -54,8 +57,21 @@ export function summaryMessages(text) {
54
57
  *
55
58
  * @param {any} ctx
56
59
  * @param {ReturnType<import("./runtime.js").createRuntime>} runtime
60
+ * @param {{ shutdownMs?: number }} [options] `shutdownMs`: how long shutdown waits, in total, for captures still sending (default `CAPTURE_SHUTDOWN_MS`)
57
61
  */
58
- export function attachCapture(ctx, runtime) {
62
+ /**
63
+ * How long shutdown waits, in total, for turn captures that are still sending.
64
+ * A capture's send runs in the background (the turn is in the spool first), so
65
+ * a hung gateway never holds up the event queue; at shutdown what has not
66
+ * finished by this deadline is released to the spool and replayed on the next
67
+ * start, where the gateway's receipt dedupes it (docs/capture-delivery.md).
68
+ */
69
+ export const CAPTURE_SHUTDOWN_MS = 5_000
70
+ /** Why a send was cut short at shutdown; classed `unreachable`, so the entry is kept for replay. */
71
+ const SHUTDOWN_ERROR = () => Object.assign(new Error("the plugin shut down before this capture finished"), { status: 408 })
72
+
73
+ export function attachCapture(ctx, runtime, options = {}) {
74
+ const shutdownMs = Number(options.shutdownMs) > 0 ? Number(options.shutdownMs) : CAPTURE_SHUTDOWN_MS
59
75
  const capturing = Boolean(runtime.config.capture)
60
76
  if (!ctx?.event || typeof ctx.event.subscribe !== "function") {
61
77
  if (capturing) runtime.log.warn("host has no event.subscribe; capture disabled")
@@ -71,6 +87,10 @@ export function attachCapture(ctx, runtime) {
71
87
  let unavailableLogged = false
72
88
  /** Sessions already logged as having an unlinked speaker; bounded like the other maps. */
73
89
  const unlinkedLogged = new Set()
90
+ /** Capture sends running in the background. Each turn is staged in the spool before its send starts. */
91
+ const sending = new Set()
92
+ /** Set when shutdown's deadline passed: a turn staged after that is released, not sent. */
93
+ let shutdownExpired = false
74
94
 
75
95
  /**
76
96
  * Memory is unavailable (a gateway or Jelly install with no credential), or
@@ -111,6 +131,13 @@ export function attachCapture(ctx, runtime) {
111
131
  return { agent: turn.agent ?? runtime.agentFor?.(sessionID) ?? info.agent, parentID: info.parentID }
112
132
  }
113
133
 
134
+ /** The person paused capture for this session (/mem-pause): nothing is kept, nothing is spooled. */
135
+ function skipPaused(sessionID, what, turn) {
136
+ if (!turn?.paused && !runtime.capturePausedSince?.(sessionID)) return false
137
+ runtime.log.debug(`${what} not captured: capture is paused for this session`, { sessionID })
138
+ return true
139
+ }
140
+
114
141
  function skipSubagent(sessionID, what) {
115
142
  if (!runtime.skipsSubagent?.(sessionID)) return false
116
143
  runtime.log.debug(`subagent ${what} not captured`, { sessionID, parent: runtime.parentOf?.(sessionID) })
@@ -123,6 +150,9 @@ export function attachCapture(ctx, runtime) {
123
150
  * flushed (captured as it stands) rather than dropped.
124
151
  */
125
152
  async function openTurn(sessionID, turn) {
153
+ // A turn that starts while capture is paused stays out of memory even if
154
+ // the person resumes before it ends.
155
+ if (runtime.capturePausedSince?.(sessionID)) turn.paused = true
126
156
  turns.set(sessionID, turn)
127
157
  while (turns.size > OPEN_TURNS_MAX) {
128
158
  const oldest = turns.keys().next().value
@@ -144,6 +174,7 @@ export function attachCapture(ctx, runtime) {
144
174
  const turn = turns.get(sessionID)
145
175
  if (!turn) return
146
176
  turns.delete(sessionID)
177
+ if (skipPaused(sessionID, "turn", turn)) return
147
178
  if (await skipUnavailable("turn", sessionID)) return
148
179
  if (skipSubagent(sessionID, "turn")) return
149
180
  if (turn.model === "unknown" && models.has(sessionID)) turn.model = models.get(sessionID)
@@ -160,8 +191,14 @@ export function attachCapture(ctx, runtime) {
160
191
  runtime.log.warn("session context read failed", error)
161
192
  }
162
193
 
163
- const payload = buildSessionPayload({ turn, messages })
164
- if (!payload) return
194
+ const built = buildSessionPayload({ turn, messages })
195
+ if (!built) return
196
+ // The turn that asks to forget something usually restates it, and its
197
+ // reply may too: each run of forgotten words is masked before the turn is
198
+ // spooled or sent (src/forget-purge.js).
199
+ const masked = runtime.maskForgotten?.(built) ?? { value: built, hits: [] }
200
+ if (masked.hits.length > 0) runtime.log.info("turn capture: forgotten text masked", { sessionID, purge: masked.hits })
201
+ const payload = masked.value
165
202
 
166
203
  if (turn.messageID && !runtime.claimCapture(`${sessionID}:${turn.messageID}`)) {
167
204
  runtime.log.debug("duplicate capture suppressed", { sessionID })
@@ -174,7 +211,13 @@ export function attachCapture(ctx, runtime) {
174
211
  const task = runtime.taskFor?.(sessionID)
175
212
  // What a replay needs to send the turn again. In control mode that is all
176
213
  // the gateway needs to place it; no container is recorded.
177
- const meta = { sessionID, agent: turn.agent, visibility, ...(task ? { task } : {}) }
214
+ // The turn's idempotency key travels with the entry, so the live send and
215
+ // every replay of it name the same turn and the gateway stores it once.
216
+ const key = turnKey(sessionID, turn.messageID, payload)
217
+ // The space it will likely land in: a pending forget holds back that
218
+ // space's turns only (src/forget-purge.js). Not used to place it.
219
+ const space = runtime.spaceHint?.(sessionID, visibility) ?? ""
220
+ const meta = { sessionID, agent: turn.agent, visibility, key, ...(task ? { task } : {}), ...(space ? { space } : {}) }
178
221
  // Control mode: the gateway's capture route places the turn. Local direct
179
222
  // mode: the fixed local scope.
180
223
  const scoped = runtime.gatewayCapture ? null : runtime.scope({ sessionID, agent: turn.agent, visibility, task })
@@ -183,27 +226,45 @@ export function attachCapture(ctx, runtime) {
183
226
  // The entry is written already claimed, so a concurrent flush in another
184
227
  // process cannot send it a second time.
185
228
  const staged = (runtime.stageSpool ?? runtime.writeSpool)("session", payload, scoped, meta)
186
- try {
187
- const placed = runtime.gatewayCapture
188
- ? await runtime.captureTurn({ ...meta, payload })
189
- : await runtime.backend.addSessionTurn({ payload, scope: scoped })
190
- runtime.removeSpool(staged)
191
- runtime.log.info("turn captured", {
192
- sessionID,
193
- reason,
194
- messages: payload.messages.length,
195
- visibility: placed?.space_kind ?? visibility,
196
- space: placed?.space_id ?? null,
197
- })
198
- runtime.drainSpool?.()
199
- } catch (error) {
200
- // A gateway or control failure leaves the turn in the spool with its
201
- // visibility; the replay asks the gateway to place it again. A speaker
202
- // Jelly says has no linked account (learnt during this send) is not
203
- // kept: the spool deletes the entry.
204
- const outcome = runtime.releaseSpool?.(staged, error)
205
- runtime.log.warn("turn capture failed", { outcome: outcome ?? "kept", error: errorMessage(error) })
229
+ if (staged && shutdownExpired) {
230
+ runtime.releaseSpool?.(staged, SHUTDOWN_ERROR())
231
+ return
232
+ }
233
+ /** This send's own record: shutdown marks it abandoned when it releases the claim. */
234
+ const ticket = { staged, abandoned: false, done: null }
235
+ const send = async () => {
236
+ try {
237
+ const placed = runtime.gatewayCapture
238
+ ? await runtime.captureTurn({ ...meta, payload })
239
+ : await runtime.backend.addSessionTurn({ payload, scope: scoped })
240
+ runtime.removeSpool(staged)
241
+ runtime.log.info("turn captured", {
242
+ sessionID,
243
+ reason,
244
+ messages: payload.messages.length,
245
+ visibility: placed?.space_kind ?? visibility,
246
+ space: placed?.space_id ?? null,
247
+ })
248
+ if (!controller.signal.aborted) runtime.drainSpool?.()
249
+ } catch (error) {
250
+ // Shutdown already released this claim; the spool replays the turn.
251
+ if (ticket.abandoned) return
252
+ // A gateway or control failure leaves the turn in the spool with its
253
+ // visibility; the replay asks the gateway to place it again. A speaker
254
+ // Jelly says has no linked account (learnt during this send) is not
255
+ // kept: the spool deletes the entry.
256
+ const outcome = runtime.releaseSpool?.(staged, error)
257
+ runtime.log.warn("turn capture failed", { outcome: outcome ?? "kept", error: errorMessage(error) })
258
+ }
206
259
  }
260
+ // Nothing durable to fall back on (spooling is off or the write failed):
261
+ // the caller waits for the send, as it always did.
262
+ if (!staged) return send()
263
+ // The turn is durable, so the send runs in the background: the event
264
+ // handler returns now, and a slow or hung gateway cannot hold up this
265
+ // process's other events. Shutdown waits for it, up to a deadline.
266
+ ticket.done = send().finally(() => sending.delete(ticket))
267
+ sending.add(ticket)
207
268
  }
208
269
 
209
270
  function captureCompaction(sessionID) {
@@ -213,13 +274,22 @@ export function attachCapture(ctx, runtime) {
213
274
  async function captureCompactionSummary(sessionID) {
214
275
  if (!runtime.config.captureScenarios) return
215
276
  try {
277
+ // The summary covers the whole transcript: a session paused at any point gets none.
278
+ if (runtime.captureEverPaused?.(sessionID)) {
279
+ runtime.log.debug("compaction summary not captured: capture was paused in this session", { sessionID })
280
+ return
281
+ }
216
282
  if (await skipUnavailable("compaction summary", sessionID)) return
217
283
  if (skipSubagent(sessionID, "compaction")) return
218
284
  const facts = await sessionFacts(sessionID, {})
219
285
  if (skipSubagent(sessionID, "compaction")) return
220
286
  const messages = await ctx.session.context({ sessionID })
221
- const summary = extractCompactionSummary(messages)
222
- if (!summary) return
287
+ const extracted = extractCompactionSummary(messages)
288
+ if (!extracted) return
289
+ // As the checkpoint: forgotten text in the host's summary is masked.
290
+ const maskedSummary = runtime.maskForgotten?.(extracted.text) ?? { value: extracted.text, hits: [] }
291
+ if (maskedSummary.hits.length > 0) runtime.log.info("compaction summary: forgotten text masked", { sessionID, purge: maskedSummary.hits })
292
+ const summary = maskedSummary.hits.length > 0 ? { ...extracted, text: maskedSummary.value } : extracted
223
293
  // The host may emit both `session.compacted` and
224
294
  // `session.compaction.ended` for one compaction. The summary identity is
225
295
  // stable, so the second event is not a second capture.
@@ -311,8 +381,21 @@ export function attachCapture(ctx, runtime) {
311
381
  }
312
382
  // A new user message: the person speaking may have changed, so the
313
383
  // session's Jelly token is asked for again; also with capture off.
314
- if (event.type === "session.inbox.enqueued" && data.item?.type === "user") runtime.userTurn?.(data.sessionID, data.inboxID)
315
- if (event.type === "session.next.prompt.admitted") runtime.userTurn?.(data.sessionID, data.messageID)
384
+ if (event.type === "session.inbox.enqueued" && data.item?.type === "user") {
385
+ runtime.userTurn?.(data.sessionID, data.inboxID)
386
+ runtime.notePrompt?.(data.sessionID, data.item?.payload?.text)
387
+ }
388
+ if (event.type === "session.next.prompt.admitted") {
389
+ runtime.userTurn?.(data.sessionID, data.messageID)
390
+ runtime.notePrompt?.(data.sessionID, data.prompt?.text)
391
+ }
392
+ // A child's parent, also with capture off: a subagent's brief is checked
393
+ // against its root session's messages, not as the person's (src/stated.js).
394
+ if (event.type === "session.created") {
395
+ const sessionID = data.sessionID ?? data.info?.id ?? data.session?.id
396
+ const parentID = parentIDOf(data)
397
+ if (sessionID && parentID) runtime.noteSession?.(sessionID, { parentID })
398
+ }
316
399
  if (!capturing) return
317
400
  if (!seenTypes.has(event.type)) {
318
401
  seenTypes.add(event.type)
@@ -375,9 +458,8 @@ export function attachCapture(ctx, runtime) {
375
458
  const sessionID = data.sessionID ?? data.info?.id ?? data.session?.id
376
459
  // Belt and braces: a child created by the task tool is known before
377
460
  // its brief is admitted, so no turn is ever opened for it. Only a
378
- // reported parent is recorded; an absent field is not proof of a root.
379
- const parentID = parentIDOf(data)
380
- if (sessionID && parentID) runtime.noteSession?.(sessionID, { parentID })
461
+ // reported parent is recorded (above, before the capture check); an
462
+ // absent field is not proof of a root.
381
463
  if (sessionID && data.model?.providerID && data.model?.id) {
382
464
  rememberBounded(models, sessionID, `${data.model.providerID}/${data.model.id}`, MODELS_MAX)
383
465
  }
@@ -425,11 +507,37 @@ export function attachCapture(ctx, runtime) {
425
507
  }
426
508
  })()
427
509
 
510
+ /**
511
+ * Stop capturing. Open turns are flushed and sends still running are
512
+ * awaited, together for at most `shutdownMs`. A send that has not finished by
513
+ * then is left in the spool (its claim is released so a later flush replays
514
+ * it) and is no longer waited for.
515
+ */
428
516
  async function dispose() {
429
517
  controller.abort()
430
- for (const sessionID of Array.from(turns.keys())) {
431
- await flush(sessionID, "shutdown").catch(() => {})
518
+ let timer
519
+ const expired = new Promise((resolve) => {
520
+ // Not unref'd: shutdown is waiting on it, and it is cleared when the work finishes.
521
+ timer = setTimeout(() => resolve(false), shutdownMs)
522
+ })
523
+ const work = (async () => {
524
+ for (const sessionID of Array.from(turns.keys())) {
525
+ await flush(sessionID, "shutdown").catch(() => {})
526
+ }
527
+ await Promise.all(Array.from(sending, (ticket) => ticket.done))
528
+ return true
529
+ })()
530
+ const finished = await Promise.race([work, expired])
531
+ clearTimeout(timer)
532
+ if (finished) return
533
+ shutdownExpired = true
534
+ let left = 0
535
+ for (const ticket of sending) {
536
+ ticket.abandoned = true
537
+ left += 1
538
+ runtime.releaseSpool?.(ticket.staged, SHUTDOWN_ERROR())
432
539
  }
540
+ runtime.log.warn("capture shutdown deadline reached; unsent turns stay in the spool", { left, afterMs: shutdownMs })
433
541
  }
434
542
  /** Sizes of the per-process maps, for tests and diagnostics. */
435
543
  dispose.stats = () => ({ turns: turns.size, models: models.size, seenEvents: seenEvents.size, compacted: compacted.size })
@@ -0,0 +1,377 @@
1
+ /**
2
+ * Forget, end to end on the client (0.9.8 slice 6b; FABLE runtime #2;
3
+ * docs/design/forget-erase.md "The purge contract").
4
+ *
5
+ * A forget deletes on the server. Four client paths could send the text back:
6
+ * a turn waiting in the spool, a quarantined `.failed` entry, the turn that
7
+ * asks for the forget, and the next compaction checkpoint. This module closes
8
+ * them with one matcher:
9
+ *
10
+ * - **Entries.** The caller's own purge entries from `forgotten.match` (salt,
11
+ * word counts and keyed hashes of each erased sentence; never text), plus a
12
+ * local entry of the same shape the plugin makes from the text it holds when
13
+ * it makes a forget itself. Both live until their `expires` (21 days, the
14
+ * longest the spool keeps a turn) in `forget-purge.json` in the state
15
+ * directory, private to the user, with no text.
16
+ * - **Purge.** Every spooled, in-flight and quarantined entry, and its retry
17
+ * state, that holds a matching run of words is deleted through the spool's
18
+ * `purgeMatching`. The log names the entry's file and the purge ids, never
19
+ * the text.
20
+ * - **When.** Right after a forget succeeds; at startup and before every
21
+ * replay of the spool (so a forget made in the console or on another
22
+ * machine, and a crash between a forget and its purge, are covered); and
23
+ * each entry is checked again just before it is replayed.
24
+ * - **Pending.** Once a forget succeeds, its space is marked `pending`
25
+ * (durably) until the service's entries for it are read. While it is, the
26
+ * spool's entries of that space wait; other spaces replay. A 401 or 403
27
+ * (removed from the space, or a page grant, which `forgotten.match`
28
+ * refuses) clears the mark at once, and a mark ends after
29
+ * `PENDING_MAX_MS` (logged once). Either way the local entry, made from the
30
+ * text this plugin just forgot, has already purged what it matches. A
31
+ * denied forget, or one that deleted nothing, sets no mark. A crash after
32
+ * the forget is repaired at the next start, which reads the entries before
33
+ * the spool is replayed.
34
+ * - **Mask.** The asking turn's capture payload, the compaction checkpoint and
35
+ * the host's compaction summary go through `mask` before they are sent or
36
+ * spooled: each matching run of words becomes `[forgotten]`.
37
+ */
38
+ import fs from "node:fs"
39
+ import path from "node:path"
40
+ import { randomBytes } from "node:crypto"
41
+ import { localPurgeEntry, purgeMatcher } from "./forgotten.js"
42
+ import { errorMessage } from "./util.js"
43
+
44
+ /** Days an entry is kept (records PURGE_WINDOW_DAYS): 7 pending plus 14 quarantined. */
45
+ export const PURGE_WINDOW_MS = 21 * 86_400_000
46
+ /** A pending mark ends after this long: the local entries have purged what they match. */
47
+ export const PENDING_MAX_MS = 24 * 3_600_000
48
+ /** How often a flush asks the service for new entries, at most. */
49
+ export const PURGE_REFRESH_MS = 5 * 60_000
50
+ /** The time one `forgotten.match` call may take. */
51
+ const MATCH_TIMEOUT_MS = 5_000
52
+ /** Spaces whose cursor is kept: the default space and those a forget ran in. */
53
+ const MAX_SPACES = 50
54
+ /** Entries kept at most; the oldest go first. */
55
+ const MAX_ENTRIES = 2_000
56
+ const FILE = "forget-purge.json"
57
+ /** The key of the default space: a call that names none (the gateway places it). */
58
+ const DEFAULT = ""
59
+
60
+ /** Every string in a value, depth first. */
61
+ function stringsOf(value, out = [], depth = 0) {
62
+ if (depth > 12 || out.length > 5_000) return out
63
+ if (typeof value === "string") out.push(value)
64
+ else if (Array.isArray(value)) for (const item of value) stringsOf(item, out, depth + 1)
65
+ else if (value && typeof value === "object") for (const item of Object.values(value)) stringsOf(item, out, depth + 1)
66
+ return out
67
+ }
68
+
69
+ /** A copy of a value with every string masked; `hits` collects the purge ids. */
70
+ function maskValue(value, matcher, hits, depth = 0) {
71
+ if (depth > 12) return value
72
+ if (typeof value === "string") {
73
+ const masked = matcher.mask(value)
74
+ for (const id of masked.hits) hits.add(id)
75
+ return masked.text
76
+ }
77
+ if (Array.isArray(value)) return value.map((item) => maskValue(item, matcher, hits, depth + 1))
78
+ if (value && typeof value === "object") {
79
+ const out = {}
80
+ for (const [key, item] of Object.entries(value)) out[key] = maskValue(item, matcher, hits, depth + 1)
81
+ return out
82
+ }
83
+ return value
84
+ }
85
+
86
+ /** A refusal that will not pass by itself: not signed in, removed from the space, a page grant only. */
87
+ function denied(error) {
88
+ const status = Number(error?.status)
89
+ return status === 401 || status === 403
90
+ }
91
+
92
+ /** A service that does not know `forgotten.match` (older than 0.9.8). */
93
+ function unsupported(error) {
94
+ const status = Number(error?.status)
95
+ const code = String(error?.code ?? error?.body?.code ?? "")
96
+ return status === 404 || code === "unknown_method" || code === "invalid_method" || /unknown (records )?method/i.test(errorMessage(error))
97
+ }
98
+
99
+ /**
100
+ * @param {{ config: any, log: any, records: { call: Function } | null, spool: { purgeMatching: Function }, now?: () => number, scope?: () => any, hasEntries?: () => boolean }} input
101
+ */
102
+ export function createForgetPurge(input) {
103
+ const { config, log, spool } = input
104
+ const now = typeof input.now === "function" ? input.now : Date.now
105
+ const records = input.records && typeof input.records.call === "function" ? input.records : null
106
+ const defaultScope = typeof input.scope === "function" ? input.scope : () => undefined
107
+ /** Whether the spool holds anything (pending, in flight or quarantined). */
108
+ const hasEntries = typeof input.hasEntries === "function" ? input.hasEntries : () => true
109
+ const file = config?.stateDir ? path.join(config.stateDir, FILE) : null
110
+ let state = load()
111
+ let cached = null
112
+ let lastRefresh = 0
113
+
114
+ function empty() {
115
+ return { version: 1, spaces: {}, entries: [], pending: {} }
116
+ }
117
+
118
+ function load() {
119
+ if (!file) return empty()
120
+ try {
121
+ const data = JSON.parse(fs.readFileSync(file, "utf8"))
122
+ if (!data || typeof data !== "object") return empty()
123
+ return {
124
+ version: 1,
125
+ spaces: data.spaces && typeof data.spaces === "object" ? data.spaces : {},
126
+ entries: Array.isArray(data.entries) ? data.entries : [],
127
+ pending: data.pending && typeof data.pending === "object" ? data.pending : {},
128
+ }
129
+ } catch {
130
+ return empty()
131
+ }
132
+ }
133
+
134
+ /** Temp file, fsync, rename: a crash leaves the old state or the new. */
135
+ function save() {
136
+ if (!file) return
137
+ try {
138
+ fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 })
139
+ const temp = `${file}.${process.pid}-${randomBytes(4).toString("hex")}.tmp`
140
+ const fd = fs.openSync(temp, "wx", 0o600)
141
+ try {
142
+ fs.writeSync(fd, JSON.stringify(state))
143
+ fs.fsyncSync(fd)
144
+ } finally {
145
+ fs.closeSync(fd)
146
+ }
147
+ fs.renameSync(temp, file)
148
+ } catch (error) {
149
+ log.warn("forget purge state not written", errorMessage(error))
150
+ }
151
+ }
152
+
153
+ /** Drop expired entries and cap the rest. */
154
+ function prune() {
155
+ const at = now()
156
+ const before = state.entries.length
157
+ state.entries = state.entries.filter((entry) => Date.parse(entry.expires ?? "") > at).slice(-MAX_ENTRIES)
158
+ if (state.entries.length !== before) cached = null
159
+ const spaces = Object.entries(state.spaces).sort((a, b) => String(b[1]?.seen ?? "").localeCompare(String(a[1]?.seen ?? "")))
160
+ if (spaces.length > MAX_SPACES) state.spaces = Object.fromEntries(spaces.slice(0, MAX_SPACES))
161
+ }
162
+
163
+ function matcher() {
164
+ if (!cached) {
165
+ prune()
166
+ cached = purgeMatcher(state.entries)
167
+ }
168
+ return cached
169
+ }
170
+
171
+ function addEntries(space, entries) {
172
+ const known = new Set(state.entries.map((entry) => entry.id))
173
+ let added = 0
174
+ for (const entry of entries) {
175
+ if (!entry || typeof entry.id !== "string" || known.has(entry.id)) continue
176
+ if (typeof entry.salt !== "string" || !Array.isArray(entry.hashes) || !Array.isArray(entry.lengths)) continue
177
+ state.entries.push({ id: entry.id, space, created: String(entry.created ?? ""), expires: String(entry.expires ?? ""), salt: entry.salt, lengths: entry.lengths, hashes: entry.hashes })
178
+ known.add(entry.id)
179
+ added += 1
180
+ }
181
+ if (added > 0) cached = null
182
+ return added
183
+ }
184
+
185
+ function noteSpace(space) {
186
+ const key = String(space ?? DEFAULT)
187
+ state.spaces[key] = { ...(state.spaces[key] ?? {}), seen: new Date(now()).toISOString() }
188
+ return key
189
+ }
190
+
191
+ /**
192
+ * Ask the service for the caller's entries newer than each space's cursor.
193
+ * The cursor moves only past entries it stored.
194
+ *
195
+ * @param {{ spaces?: string[], scope?: any }} [options]
196
+ * @returns {Promise<{ ok: boolean, added: number, failed: string[], denied: string[], unsupported: string[] }>}
197
+ */
198
+ async function refresh(options = {}) {
199
+ const result = { ok: true, added: 0, failed: [], denied: [], unsupported: [] }
200
+ if (!records) return { ...result, ok: false, failed: ["no records client"] }
201
+ const spaces = options.spaces ?? [...new Set([DEFAULT, ...Object.keys(state.spaces)])]
202
+ for (const space of spaces) {
203
+ const key = noteSpace(space)
204
+ const since = state.spaces[key]?.since
205
+ try {
206
+ const answer = await records.call("forgotten.match", since ? { since } : {}, {
207
+ scope: options.scope ?? defaultScope(),
208
+ ...(key ? { space: key } : {}),
209
+ timeoutMs: MATCH_TIMEOUT_MS,
210
+ })
211
+ const entries = Array.isArray(answer?.entries) ? answer.entries : []
212
+ result.added += addEntries(key, entries)
213
+ const newest = entries.map((entry) => String(entry?.created ?? "")).filter((value) => !Number.isNaN(Date.parse(value))).sort().pop()
214
+ if (newest && (!since || newest > since)) state.spaces[key].since = newest
215
+ } catch (error) {
216
+ if (unsupported(error)) {
217
+ result.unsupported.push(key)
218
+ continue
219
+ }
220
+ if (denied(error)) {
221
+ result.denied.push(key)
222
+ continue
223
+ }
224
+ result.ok = false
225
+ result.failed.push(key)
226
+ log.warn("forgotten entries could not be read; the spool keeps what it holds until they can", { space: key || "default", error: errorMessage(error) })
227
+ }
228
+ }
229
+ lastRefresh = now()
230
+ prune()
231
+ save()
232
+ return result
233
+ }
234
+
235
+ /** Whether an entry of the spool holds forgotten text: the purge ids it matches. */
236
+ function entryMatches(entry, raw) {
237
+ const m = matcher()
238
+ if (m.size === 0) return []
239
+ const texts = entry ? stringsOf(entry.data ?? entry) : [String(raw ?? "")]
240
+ const ids = new Set()
241
+ for (const text of texts) for (const id of m.matches(text)) ids.add(id)
242
+ return [...ids]
243
+ }
244
+
245
+ /**
246
+ * Delete every spooled, in-flight and quarantined entry, and its state,
247
+ * that holds forgotten text. Logs file names and purge ids only.
248
+ */
249
+ function purge() {
250
+ if (matcher().size === 0) return { purged: 0, failed: 0 }
251
+ const found = []
252
+ const result = spool.purgeMatching((entry, info) => {
253
+ const ids = entryMatches(entry, info.raw)
254
+ if (ids.length > 0) found.push({ file: info.file, state: info.state, purge: ids })
255
+ return ids.length > 0
256
+ })
257
+ if (found.length > 0) log.info("spooled entries holding forgotten text deleted", { purged: result.purged, entries: found.map((item) => ({ file: item.file, state: item.state, purge: item.purge })) })
258
+ return result
259
+ }
260
+
261
+ /** When a space's mark was set, in ms; a mark from 0.9.8's first cut is an ISO string. */
262
+ function markedAt(key) {
263
+ const mark = state.pending[key]
264
+ return Date.parse(typeof mark === "string" ? mark : mark?.at ?? "")
265
+ }
266
+
267
+ /** The spaces whose entries wait: marked, and within PENDING_MAX_MS. */
268
+ function livePending() {
269
+ const at = now()
270
+ return Object.keys(state.pending).filter((key) => !(at - markedAt(key) >= PENDING_MAX_MS) && Number.isFinite(markedAt(key)))
271
+ }
272
+
273
+ /** End the marks past PENDING_MAX_MS, with one log line each. */
274
+ function expirePending() {
275
+ const at = now()
276
+ for (const key of Object.keys(state.pending)) {
277
+ const since = markedAt(key)
278
+ if (Number.isFinite(since) && at - since < PENDING_MAX_MS) continue
279
+ delete state.pending[key]
280
+ log.warn("pending forget ended after 24 hours without reading its purge entries; the spool was purged with this machine's own entries, and the space's turns replay again", { space: key || "default" })
281
+ }
282
+ }
283
+
284
+ /** Clear the marks a refresh settled: read, refused for good (401 or 403), or a service without the contract. */
285
+ function settle(keys, fetched) {
286
+ for (const key of keys) {
287
+ if (fetched.failed.includes(key)) continue
288
+ if (fetched.denied.includes(key)) log.warn("forgotten entries refused (401 or 403); the spool was purged with this machine's own entries, and the pending mark is cleared", { space: key || "default" })
289
+ delete state.pending[key]
290
+ }
291
+ }
292
+
293
+ /**
294
+ * After a forget succeeded: keep a local entry for the text this plugin
295
+ * held, mark the space pending, fetch the service's entries for it, purge,
296
+ * and clear the mark unless the fetch failed for a passing reason.
297
+ *
298
+ * @param {{ space?: string, texts?: string[], scope?: any }} input
299
+ */
300
+ async function afterForget({ space, texts = [], scope } = {}) {
301
+ const key = noteSpace(space)
302
+ const at = now()
303
+ const local = localPurgeEntry(texts.filter(Boolean).map(String), {
304
+ id: `local_${at.toString(36)}${randomBytes(6).toString("hex")}`,
305
+ salt: randomBytes(32).toString("base64url"),
306
+ created: new Date(at).toISOString(),
307
+ expires: new Date(at + PURGE_WINDOW_MS).toISOString(),
308
+ })
309
+ if (local) addEntries(key, [local])
310
+ state.pending[key] = { at: new Date(at).toISOString() }
311
+ save()
312
+ const fetched = await refresh({ spaces: [key], scope })
313
+ const purged = purge()
314
+ settle([key], fetched)
315
+ save()
316
+ return { purged: purged.purged, failed: purged.failed, fetched: fetched.ok && fetched.denied.length === 0, pending: key in state.pending, entries: matcher().size }
317
+ }
318
+
319
+ /**
320
+ * Before the spool is replayed (startup, a drain, /mem-sync): end old
321
+ * marks, fetch new entries when due (always while a space is pending), and
322
+ * purge. A pending space's entries wait in the replay itself (`sendable`);
323
+ * nothing else is held back.
324
+ */
325
+ async function beforeFlush({ force = false } = {}) {
326
+ expirePending()
327
+ const pending = Object.keys(state.pending)
328
+ // Nothing queued and no forget pending: nothing to purge or hold back.
329
+ if (pending.length === 0 && !force && !hasEntries()) {
330
+ save()
331
+ return { ok: true }
332
+ }
333
+ if (force || pending.length > 0 || now() - lastRefresh >= PURGE_REFRESH_MS) settle(pending, await refresh())
334
+ purge()
335
+ save()
336
+ return { ok: true }
337
+ }
338
+
339
+ /** The space a spool entry was captured in, as far as this machine knew it ("" when not). */
340
+ function spaceOf(entry) {
341
+ return String(entry?.space ?? entry?.scope?.space_id ?? "")
342
+ }
343
+
344
+ /**
345
+ * Whether a spool entry may be sent now (checked just before each replay):
346
+ * false when it holds forgotten text (it is deleted), "hold" when its space
347
+ * has a pending forget (it waits), true otherwise.
348
+ *
349
+ * @returns {boolean | "hold"}
350
+ */
351
+ function sendable(entry) {
352
+ if (entryMatches(entry, "").length > 0) return false
353
+ return livePending().includes(spaceOf(entry)) ? "hold" : true
354
+ }
355
+
356
+ /**
357
+ * A copy of a payload with each run of forgotten words masked, and the
358
+ * purge ids it hit (empty when nothing changed: `value` is then the input).
359
+ *
360
+ * @template T
361
+ * @param {T} value
362
+ * @returns {{ value: T, hits: string[] }}
363
+ */
364
+ function mask(value) {
365
+ const m = matcher()
366
+ if (m.size === 0) return { value, hits: [] }
367
+ const hits = new Set()
368
+ const masked = maskValue(value, m, hits)
369
+ return hits.size > 0 ? { value: masked, hits: [...hits] } : { value, hits: [] }
370
+ }
371
+
372
+ function status() {
373
+ return { entries: matcher().size, pending: livePending().length, spaces: Object.keys(state.spaces).length }
374
+ }
375
+
376
+ return { refresh, purge, afterForget, beforeFlush, sendable, mask, status, matches: (text) => matcher().matches(text) }
377
+ }