@joe-broadhead/opencode-mem 0.9.5 → 0.9.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1281 -0
- package/README.md +105 -31
- package/docs/agent-protocol.md +28 -27
- package/docs/permissions.md +148 -0
- package/package.json +24 -12
- package/skills/mem.pages/SKILL.md +35 -16
- package/skills/mem.sharing/SKILL.md +39 -6
- package/skills/mem.sources/SKILL.md +41 -10
- package/skills/mem.usage/SKILL.md +37 -16
- package/src/backend/records.js +8 -0
- package/src/backend/v3.js +15 -10
- package/src/call-shapes.js +18 -25
- package/src/capture-pause.js +136 -0
- package/src/capture.js +7 -5
- package/src/collector-service.js +212 -35
- package/src/collector.js +186 -47
- package/src/commands.js +37 -0
- package/src/compaction.js +4 -2
- package/src/config.js +151 -1
- package/src/curation.js +6 -10
- package/src/events.js +144 -36
- package/src/forget-purge.js +377 -0
- package/src/forgotten.js +152 -0
- package/src/hooks.js +9 -4
- package/src/index.js +5 -3
- package/src/permissions.js +116 -38
- package/src/protocol.js +21 -11
- package/src/records.js +4 -1
- package/src/runtime.js +202 -15
- package/src/shapes.js +63 -0
- package/src/sources.js +183 -12
- package/src/spaces.js +58 -1
- package/src/spool.js +442 -64
- package/src/stated.js +334 -0
- package/src/storable.js +477 -0
- package/src/tools/aliases.js +36 -147
- package/src/tools/commitments.js +69 -12
- package/src/tools/index.js +25 -43
- package/src/tools/ingest.js +33 -128
- package/src/tools/memory.js +503 -904
- package/src/tools/page-sections.js +147 -0
- package/src/tools/page.js +314 -82
- package/src/tools/recall-browse.js +173 -0
- package/src/tools/recall.js +36 -8
- package/src/tools/share.js +494 -131
- package/src/tools/shared.js +97 -19
- package/src/tools/spaces.js +194 -101
- package/src/tools/status.js +31 -3
- package/src/tools/wiki.js +5 -13
- package/src/util.js +10 -59
- package/src/views/profile.js +2 -3
- package/src/views/records-reads.js +6 -1
- package/src/erase.js +0 -36
package/src/events.js
CHANGED
|
@@ -2,7 +2,9 @@ import { buildSessionPayload, extractCompactionSummary, newTurn } from "./captur
|
|
|
2
2
|
import { UNLINKED_SENTENCE, asSession } from "./credentials.js"
|
|
3
3
|
import { CAPTURE_MESSAGE_CHARS, MAX_MESSAGES, splitContent } from "./limits.js"
|
|
4
4
|
import { parentIDOf, readSession } from "./sessions.js"
|
|
5
|
-
import {
|
|
5
|
+
import { turnKey } from "./spool.js"
|
|
6
|
+
import { storable } from "./storable.js"
|
|
7
|
+
import { errorMessage, rememberBounded, shortHash } from "./util.js"
|
|
6
8
|
|
|
7
9
|
/** Bounds on the per-process maps; old sessions fall out in insertion order. */
|
|
8
10
|
const COMPACTED_MAX = 200
|
|
@@ -18,12 +20,13 @@ const SUMMARY_PART_CHARS = CAPTURE_MESSAGE_CHARS - 80
|
|
|
18
20
|
* A host compaction summary as L0 user messages. A summary longer than the
|
|
19
21
|
* upstream per-message limit is split, not truncated: it is the one record of
|
|
20
22
|
* the compacted context, and about a quarter of summaries used to be rejected
|
|
21
|
-
* whole.
|
|
23
|
+
* whole. As every automatic write (src/storable.js), lines holding a secret
|
|
24
|
+
* are withheld and emails and local paths masked first.
|
|
22
25
|
*
|
|
23
26
|
* @param {string} text
|
|
24
27
|
*/
|
|
25
28
|
export function summaryMessages(text) {
|
|
26
|
-
const parts = splitContent(
|
|
29
|
+
const parts = splitContent(storable(text, "summary").text, SUMMARY_PART_CHARS).slice(0, MAX_MESSAGES - 1)
|
|
27
30
|
return parts.map((part, index) => ({
|
|
28
31
|
role: "user",
|
|
29
32
|
content:
|
|
@@ -54,8 +57,21 @@ export function summaryMessages(text) {
|
|
|
54
57
|
*
|
|
55
58
|
* @param {any} ctx
|
|
56
59
|
* @param {ReturnType<import("./runtime.js").createRuntime>} runtime
|
|
60
|
+
* @param {{ shutdownMs?: number }} [options] `shutdownMs`: how long shutdown waits, in total, for captures still sending (default `CAPTURE_SHUTDOWN_MS`)
|
|
57
61
|
*/
|
|
58
|
-
|
|
62
|
+
/**
|
|
63
|
+
* How long shutdown waits, in total, for turn captures that are still sending.
|
|
64
|
+
* A capture's send runs in the background (the turn is in the spool first), so
|
|
65
|
+
* a hung gateway never holds up the event queue; at shutdown what has not
|
|
66
|
+
* finished by this deadline is released to the spool and replayed on the next
|
|
67
|
+
* start, where the gateway's receipt dedupes it (docs/capture-delivery.md).
|
|
68
|
+
*/
|
|
69
|
+
export const CAPTURE_SHUTDOWN_MS = 5_000
|
|
70
|
+
/** Why a send was cut short at shutdown; classed `unreachable`, so the entry is kept for replay. */
|
|
71
|
+
const SHUTDOWN_ERROR = () => Object.assign(new Error("the plugin shut down before this capture finished"), { status: 408 })
|
|
72
|
+
|
|
73
|
+
export function attachCapture(ctx, runtime, options = {}) {
|
|
74
|
+
const shutdownMs = Number(options.shutdownMs) > 0 ? Number(options.shutdownMs) : CAPTURE_SHUTDOWN_MS
|
|
59
75
|
const capturing = Boolean(runtime.config.capture)
|
|
60
76
|
if (!ctx?.event || typeof ctx.event.subscribe !== "function") {
|
|
61
77
|
if (capturing) runtime.log.warn("host has no event.subscribe; capture disabled")
|
|
@@ -71,6 +87,10 @@ export function attachCapture(ctx, runtime) {
|
|
|
71
87
|
let unavailableLogged = false
|
|
72
88
|
/** Sessions already logged as having an unlinked speaker; bounded like the other maps. */
|
|
73
89
|
const unlinkedLogged = new Set()
|
|
90
|
+
/** Capture sends running in the background. Each turn is staged in the spool before its send starts. */
|
|
91
|
+
const sending = new Set()
|
|
92
|
+
/** Set when shutdown's deadline passed: a turn staged after that is released, not sent. */
|
|
93
|
+
let shutdownExpired = false
|
|
74
94
|
|
|
75
95
|
/**
|
|
76
96
|
* Memory is unavailable (a gateway or Jelly install with no credential), or
|
|
@@ -111,6 +131,13 @@ export function attachCapture(ctx, runtime) {
|
|
|
111
131
|
return { agent: turn.agent ?? runtime.agentFor?.(sessionID) ?? info.agent, parentID: info.parentID }
|
|
112
132
|
}
|
|
113
133
|
|
|
134
|
+
/** The person paused capture for this session (/mem-pause): nothing is kept, nothing is spooled. */
|
|
135
|
+
function skipPaused(sessionID, what, turn) {
|
|
136
|
+
if (!turn?.paused && !runtime.capturePausedSince?.(sessionID)) return false
|
|
137
|
+
runtime.log.debug(`${what} not captured: capture is paused for this session`, { sessionID })
|
|
138
|
+
return true
|
|
139
|
+
}
|
|
140
|
+
|
|
114
141
|
function skipSubagent(sessionID, what) {
|
|
115
142
|
if (!runtime.skipsSubagent?.(sessionID)) return false
|
|
116
143
|
runtime.log.debug(`subagent ${what} not captured`, { sessionID, parent: runtime.parentOf?.(sessionID) })
|
|
@@ -123,6 +150,9 @@ export function attachCapture(ctx, runtime) {
|
|
|
123
150
|
* flushed (captured as it stands) rather than dropped.
|
|
124
151
|
*/
|
|
125
152
|
async function openTurn(sessionID, turn) {
|
|
153
|
+
// A turn that starts while capture is paused stays out of memory even if
|
|
154
|
+
// the person resumes before it ends.
|
|
155
|
+
if (runtime.capturePausedSince?.(sessionID)) turn.paused = true
|
|
126
156
|
turns.set(sessionID, turn)
|
|
127
157
|
while (turns.size > OPEN_TURNS_MAX) {
|
|
128
158
|
const oldest = turns.keys().next().value
|
|
@@ -144,6 +174,7 @@ export function attachCapture(ctx, runtime) {
|
|
|
144
174
|
const turn = turns.get(sessionID)
|
|
145
175
|
if (!turn) return
|
|
146
176
|
turns.delete(sessionID)
|
|
177
|
+
if (skipPaused(sessionID, "turn", turn)) return
|
|
147
178
|
if (await skipUnavailable("turn", sessionID)) return
|
|
148
179
|
if (skipSubagent(sessionID, "turn")) return
|
|
149
180
|
if (turn.model === "unknown" && models.has(sessionID)) turn.model = models.get(sessionID)
|
|
@@ -160,8 +191,14 @@ export function attachCapture(ctx, runtime) {
|
|
|
160
191
|
runtime.log.warn("session context read failed", error)
|
|
161
192
|
}
|
|
162
193
|
|
|
163
|
-
const
|
|
164
|
-
if (!
|
|
194
|
+
const built = buildSessionPayload({ turn, messages })
|
|
195
|
+
if (!built) return
|
|
196
|
+
// The turn that asks to forget something usually restates it, and its
|
|
197
|
+
// reply may too: each run of forgotten words is masked before the turn is
|
|
198
|
+
// spooled or sent (src/forget-purge.js).
|
|
199
|
+
const masked = runtime.maskForgotten?.(built) ?? { value: built, hits: [] }
|
|
200
|
+
if (masked.hits.length > 0) runtime.log.info("turn capture: forgotten text masked", { sessionID, purge: masked.hits })
|
|
201
|
+
const payload = masked.value
|
|
165
202
|
|
|
166
203
|
if (turn.messageID && !runtime.claimCapture(`${sessionID}:${turn.messageID}`)) {
|
|
167
204
|
runtime.log.debug("duplicate capture suppressed", { sessionID })
|
|
@@ -174,7 +211,13 @@ export function attachCapture(ctx, runtime) {
|
|
|
174
211
|
const task = runtime.taskFor?.(sessionID)
|
|
175
212
|
// What a replay needs to send the turn again. In control mode that is all
|
|
176
213
|
// the gateway needs to place it; no container is recorded.
|
|
177
|
-
|
|
214
|
+
// The turn's idempotency key travels with the entry, so the live send and
|
|
215
|
+
// every replay of it name the same turn and the gateway stores it once.
|
|
216
|
+
const key = turnKey(sessionID, turn.messageID, payload)
|
|
217
|
+
// The space it will likely land in: a pending forget holds back that
|
|
218
|
+
// space's turns only (src/forget-purge.js). Not used to place it.
|
|
219
|
+
const space = runtime.spaceHint?.(sessionID, visibility) ?? ""
|
|
220
|
+
const meta = { sessionID, agent: turn.agent, visibility, key, ...(task ? { task } : {}), ...(space ? { space } : {}) }
|
|
178
221
|
// Control mode: the gateway's capture route places the turn. Local direct
|
|
179
222
|
// mode: the fixed local scope.
|
|
180
223
|
const scoped = runtime.gatewayCapture ? null : runtime.scope({ sessionID, agent: turn.agent, visibility, task })
|
|
@@ -183,27 +226,45 @@ export function attachCapture(ctx, runtime) {
|
|
|
183
226
|
// The entry is written already claimed, so a concurrent flush in another
|
|
184
227
|
// process cannot send it a second time.
|
|
185
228
|
const staged = (runtime.stageSpool ?? runtime.writeSpool)("session", payload, scoped, meta)
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
229
|
+
if (staged && shutdownExpired) {
|
|
230
|
+
runtime.releaseSpool?.(staged, SHUTDOWN_ERROR())
|
|
231
|
+
return
|
|
232
|
+
}
|
|
233
|
+
/** This send's own record: shutdown marks it abandoned when it releases the claim. */
|
|
234
|
+
const ticket = { staged, abandoned: false, done: null }
|
|
235
|
+
const send = async () => {
|
|
236
|
+
try {
|
|
237
|
+
const placed = runtime.gatewayCapture
|
|
238
|
+
? await runtime.captureTurn({ ...meta, payload })
|
|
239
|
+
: await runtime.backend.addSessionTurn({ payload, scope: scoped })
|
|
240
|
+
runtime.removeSpool(staged)
|
|
241
|
+
runtime.log.info("turn captured", {
|
|
242
|
+
sessionID,
|
|
243
|
+
reason,
|
|
244
|
+
messages: payload.messages.length,
|
|
245
|
+
visibility: placed?.space_kind ?? visibility,
|
|
246
|
+
space: placed?.space_id ?? null,
|
|
247
|
+
})
|
|
248
|
+
if (!controller.signal.aborted) runtime.drainSpool?.()
|
|
249
|
+
} catch (error) {
|
|
250
|
+
// Shutdown already released this claim; the spool replays the turn.
|
|
251
|
+
if (ticket.abandoned) return
|
|
252
|
+
// A gateway or control failure leaves the turn in the spool with its
|
|
253
|
+
// visibility; the replay asks the gateway to place it again. A speaker
|
|
254
|
+
// Jelly says has no linked account (learnt during this send) is not
|
|
255
|
+
// kept: the spool deletes the entry.
|
|
256
|
+
const outcome = runtime.releaseSpool?.(staged, error)
|
|
257
|
+
runtime.log.warn("turn capture failed", { outcome: outcome ?? "kept", error: errorMessage(error) })
|
|
258
|
+
}
|
|
206
259
|
}
|
|
260
|
+
// Nothing durable to fall back on (spooling is off or the write failed):
|
|
261
|
+
// the caller waits for the send, as it always did.
|
|
262
|
+
if (!staged) return send()
|
|
263
|
+
// The turn is durable, so the send runs in the background: the event
|
|
264
|
+
// handler returns now, and a slow or hung gateway cannot hold up this
|
|
265
|
+
// process's other events. Shutdown waits for it, up to a deadline.
|
|
266
|
+
ticket.done = send().finally(() => sending.delete(ticket))
|
|
267
|
+
sending.add(ticket)
|
|
207
268
|
}
|
|
208
269
|
|
|
209
270
|
function captureCompaction(sessionID) {
|
|
@@ -213,13 +274,22 @@ export function attachCapture(ctx, runtime) {
|
|
|
213
274
|
async function captureCompactionSummary(sessionID) {
|
|
214
275
|
if (!runtime.config.captureScenarios) return
|
|
215
276
|
try {
|
|
277
|
+
// The summary covers the whole transcript: a session paused at any point gets none.
|
|
278
|
+
if (runtime.captureEverPaused?.(sessionID)) {
|
|
279
|
+
runtime.log.debug("compaction summary not captured: capture was paused in this session", { sessionID })
|
|
280
|
+
return
|
|
281
|
+
}
|
|
216
282
|
if (await skipUnavailable("compaction summary", sessionID)) return
|
|
217
283
|
if (skipSubagent(sessionID, "compaction")) return
|
|
218
284
|
const facts = await sessionFacts(sessionID, {})
|
|
219
285
|
if (skipSubagent(sessionID, "compaction")) return
|
|
220
286
|
const messages = await ctx.session.context({ sessionID })
|
|
221
|
-
const
|
|
222
|
-
if (!
|
|
287
|
+
const extracted = extractCompactionSummary(messages)
|
|
288
|
+
if (!extracted) return
|
|
289
|
+
// As the checkpoint: forgotten text in the host's summary is masked.
|
|
290
|
+
const maskedSummary = runtime.maskForgotten?.(extracted.text) ?? { value: extracted.text, hits: [] }
|
|
291
|
+
if (maskedSummary.hits.length > 0) runtime.log.info("compaction summary: forgotten text masked", { sessionID, purge: maskedSummary.hits })
|
|
292
|
+
const summary = maskedSummary.hits.length > 0 ? { ...extracted, text: maskedSummary.value } : extracted
|
|
223
293
|
// The host may emit both `session.compacted` and
|
|
224
294
|
// `session.compaction.ended` for one compaction. The summary identity is
|
|
225
295
|
// stable, so the second event is not a second capture.
|
|
@@ -311,8 +381,21 @@ export function attachCapture(ctx, runtime) {
|
|
|
311
381
|
}
|
|
312
382
|
// A new user message: the person speaking may have changed, so the
|
|
313
383
|
// session's Jelly token is asked for again; also with capture off.
|
|
314
|
-
if (event.type === "session.inbox.enqueued" && data.item?.type === "user")
|
|
315
|
-
|
|
384
|
+
if (event.type === "session.inbox.enqueued" && data.item?.type === "user") {
|
|
385
|
+
runtime.userTurn?.(data.sessionID, data.inboxID)
|
|
386
|
+
runtime.notePrompt?.(data.sessionID, data.item?.payload?.text)
|
|
387
|
+
}
|
|
388
|
+
if (event.type === "session.next.prompt.admitted") {
|
|
389
|
+
runtime.userTurn?.(data.sessionID, data.messageID)
|
|
390
|
+
runtime.notePrompt?.(data.sessionID, data.prompt?.text)
|
|
391
|
+
}
|
|
392
|
+
// A child's parent, also with capture off: a subagent's brief is checked
|
|
393
|
+
// against its root session's messages, not as the person's (src/stated.js).
|
|
394
|
+
if (event.type === "session.created") {
|
|
395
|
+
const sessionID = data.sessionID ?? data.info?.id ?? data.session?.id
|
|
396
|
+
const parentID = parentIDOf(data)
|
|
397
|
+
if (sessionID && parentID) runtime.noteSession?.(sessionID, { parentID })
|
|
398
|
+
}
|
|
316
399
|
if (!capturing) return
|
|
317
400
|
if (!seenTypes.has(event.type)) {
|
|
318
401
|
seenTypes.add(event.type)
|
|
@@ -375,9 +458,8 @@ export function attachCapture(ctx, runtime) {
|
|
|
375
458
|
const sessionID = data.sessionID ?? data.info?.id ?? data.session?.id
|
|
376
459
|
// Belt and braces: a child created by the task tool is known before
|
|
377
460
|
// its brief is admitted, so no turn is ever opened for it. Only a
|
|
378
|
-
// reported parent is recorded
|
|
379
|
-
|
|
380
|
-
if (sessionID && parentID) runtime.noteSession?.(sessionID, { parentID })
|
|
461
|
+
// reported parent is recorded (above, before the capture check); an
|
|
462
|
+
// absent field is not proof of a root.
|
|
381
463
|
if (sessionID && data.model?.providerID && data.model?.id) {
|
|
382
464
|
rememberBounded(models, sessionID, `${data.model.providerID}/${data.model.id}`, MODELS_MAX)
|
|
383
465
|
}
|
|
@@ -425,11 +507,37 @@ export function attachCapture(ctx, runtime) {
|
|
|
425
507
|
}
|
|
426
508
|
})()
|
|
427
509
|
|
|
510
|
+
/**
|
|
511
|
+
* Stop capturing. Open turns are flushed and sends still running are
|
|
512
|
+
* awaited, together for at most `shutdownMs`. A send that has not finished by
|
|
513
|
+
* then is left in the spool (its claim is released so a later flush replays
|
|
514
|
+
* it) and is no longer waited for.
|
|
515
|
+
*/
|
|
428
516
|
async function dispose() {
|
|
429
517
|
controller.abort()
|
|
430
|
-
|
|
431
|
-
|
|
518
|
+
let timer
|
|
519
|
+
const expired = new Promise((resolve) => {
|
|
520
|
+
// Not unref'd: shutdown is waiting on it, and it is cleared when the work finishes.
|
|
521
|
+
timer = setTimeout(() => resolve(false), shutdownMs)
|
|
522
|
+
})
|
|
523
|
+
const work = (async () => {
|
|
524
|
+
for (const sessionID of Array.from(turns.keys())) {
|
|
525
|
+
await flush(sessionID, "shutdown").catch(() => {})
|
|
526
|
+
}
|
|
527
|
+
await Promise.all(Array.from(sending, (ticket) => ticket.done))
|
|
528
|
+
return true
|
|
529
|
+
})()
|
|
530
|
+
const finished = await Promise.race([work, expired])
|
|
531
|
+
clearTimeout(timer)
|
|
532
|
+
if (finished) return
|
|
533
|
+
shutdownExpired = true
|
|
534
|
+
let left = 0
|
|
535
|
+
for (const ticket of sending) {
|
|
536
|
+
ticket.abandoned = true
|
|
537
|
+
left += 1
|
|
538
|
+
runtime.releaseSpool?.(ticket.staged, SHUTDOWN_ERROR())
|
|
432
539
|
}
|
|
540
|
+
runtime.log.warn("capture shutdown deadline reached; unsent turns stay in the spool", { left, afterMs: shutdownMs })
|
|
433
541
|
}
|
|
434
542
|
/** Sizes of the per-process maps, for tests and diagnostics. */
|
|
435
543
|
dispose.stats = () => ({ turns: turns.size, models: models.size, seenEvents: seenEvents.size, compacted: compacted.size })
|
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Forget, end to end on the client (0.9.8 slice 6b; FABLE runtime #2;
|
|
3
|
+
* docs/design/forget-erase.md "The purge contract").
|
|
4
|
+
*
|
|
5
|
+
* A forget deletes on the server. Four client paths could send the text back:
|
|
6
|
+
* a turn waiting in the spool, a quarantined `.failed` entry, the turn that
|
|
7
|
+
* asks for the forget, and the next compaction checkpoint. This module closes
|
|
8
|
+
* them with one matcher:
|
|
9
|
+
*
|
|
10
|
+
* - **Entries.** The caller's own purge entries from `forgotten.match` (salt,
|
|
11
|
+
* word counts and keyed hashes of each erased sentence; never text), plus a
|
|
12
|
+
* local entry of the same shape the plugin makes from the text it holds when
|
|
13
|
+
* it makes a forget itself. Both live until their `expires` (21 days, the
|
|
14
|
+
* longest the spool keeps a turn) in `forget-purge.json` in the state
|
|
15
|
+
* directory, private to the user, with no text.
|
|
16
|
+
* - **Purge.** Every spooled, in-flight and quarantined entry, and its retry
|
|
17
|
+
* state, that holds a matching run of words is deleted through the spool's
|
|
18
|
+
* `purgeMatching`. The log names the entry's file and the purge ids, never
|
|
19
|
+
* the text.
|
|
20
|
+
* - **When.** Right after a forget succeeds; at startup and before every
|
|
21
|
+
* replay of the spool (so a forget made in the console or on another
|
|
22
|
+
* machine, and a crash between a forget and its purge, are covered); and
|
|
23
|
+
* each entry is checked again just before it is replayed.
|
|
24
|
+
* - **Pending.** Once a forget succeeds, its space is marked `pending`
|
|
25
|
+
* (durably) until the service's entries for it are read. While it is, the
|
|
26
|
+
* spool's entries of that space wait; other spaces replay. A 401 or 403
|
|
27
|
+
* (removed from the space, or a page grant, which `forgotten.match`
|
|
28
|
+
* refuses) clears the mark at once, and a mark ends after
|
|
29
|
+
* `PENDING_MAX_MS` (logged once). Either way the local entry, made from the
|
|
30
|
+
* text this plugin just forgot, has already purged what it matches. A
|
|
31
|
+
* denied forget, or one that deleted nothing, sets no mark. A crash after
|
|
32
|
+
* the forget is repaired at the next start, which reads the entries before
|
|
33
|
+
* the spool is replayed.
|
|
34
|
+
* - **Mask.** The asking turn's capture payload, the compaction checkpoint and
|
|
35
|
+
* the host's compaction summary go through `mask` before they are sent or
|
|
36
|
+
* spooled: each matching run of words becomes `[forgotten]`.
|
|
37
|
+
*/
|
|
38
|
+
import fs from "node:fs"
|
|
39
|
+
import path from "node:path"
|
|
40
|
+
import { randomBytes } from "node:crypto"
|
|
41
|
+
import { localPurgeEntry, purgeMatcher } from "./forgotten.js"
|
|
42
|
+
import { errorMessage } from "./util.js"
|
|
43
|
+
|
|
44
|
+
/** Days an entry is kept (records PURGE_WINDOW_DAYS): 7 pending plus 14 quarantined. */
|
|
45
|
+
export const PURGE_WINDOW_MS = 21 * 86_400_000
|
|
46
|
+
/** A pending mark ends after this long: the local entries have purged what they match. */
|
|
47
|
+
export const PENDING_MAX_MS = 24 * 3_600_000
|
|
48
|
+
/** How often a flush asks the service for new entries, at most. */
|
|
49
|
+
export const PURGE_REFRESH_MS = 5 * 60_000
|
|
50
|
+
/** The time one `forgotten.match` call may take. */
|
|
51
|
+
const MATCH_TIMEOUT_MS = 5_000
|
|
52
|
+
/** Spaces whose cursor is kept: the default space and those a forget ran in. */
|
|
53
|
+
const MAX_SPACES = 50
|
|
54
|
+
/** Entries kept at most; the oldest go first. */
|
|
55
|
+
const MAX_ENTRIES = 2_000
|
|
56
|
+
const FILE = "forget-purge.json"
|
|
57
|
+
/** The key of the default space: a call that names none (the gateway places it). */
|
|
58
|
+
const DEFAULT = ""
|
|
59
|
+
|
|
60
|
+
/** Every string in a value, depth first. */
|
|
61
|
+
function stringsOf(value, out = [], depth = 0) {
|
|
62
|
+
if (depth > 12 || out.length > 5_000) return out
|
|
63
|
+
if (typeof value === "string") out.push(value)
|
|
64
|
+
else if (Array.isArray(value)) for (const item of value) stringsOf(item, out, depth + 1)
|
|
65
|
+
else if (value && typeof value === "object") for (const item of Object.values(value)) stringsOf(item, out, depth + 1)
|
|
66
|
+
return out
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** A copy of a value with every string masked; `hits` collects the purge ids. */
|
|
70
|
+
function maskValue(value, matcher, hits, depth = 0) {
|
|
71
|
+
if (depth > 12) return value
|
|
72
|
+
if (typeof value === "string") {
|
|
73
|
+
const masked = matcher.mask(value)
|
|
74
|
+
for (const id of masked.hits) hits.add(id)
|
|
75
|
+
return masked.text
|
|
76
|
+
}
|
|
77
|
+
if (Array.isArray(value)) return value.map((item) => maskValue(item, matcher, hits, depth + 1))
|
|
78
|
+
if (value && typeof value === "object") {
|
|
79
|
+
const out = {}
|
|
80
|
+
for (const [key, item] of Object.entries(value)) out[key] = maskValue(item, matcher, hits, depth + 1)
|
|
81
|
+
return out
|
|
82
|
+
}
|
|
83
|
+
return value
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** A refusal that will not pass by itself: not signed in, removed from the space, a page grant only. */
|
|
87
|
+
function denied(error) {
|
|
88
|
+
const status = Number(error?.status)
|
|
89
|
+
return status === 401 || status === 403
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** A service that does not know `forgotten.match` (older than 0.9.8). */
|
|
93
|
+
function unsupported(error) {
|
|
94
|
+
const status = Number(error?.status)
|
|
95
|
+
const code = String(error?.code ?? error?.body?.code ?? "")
|
|
96
|
+
return status === 404 || code === "unknown_method" || code === "invalid_method" || /unknown (records )?method/i.test(errorMessage(error))
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* @param {{ config: any, log: any, records: { call: Function } | null, spool: { purgeMatching: Function }, now?: () => number, scope?: () => any, hasEntries?: () => boolean }} input
|
|
101
|
+
*/
|
|
102
|
+
export function createForgetPurge(input) {
|
|
103
|
+
const { config, log, spool } = input
|
|
104
|
+
const now = typeof input.now === "function" ? input.now : Date.now
|
|
105
|
+
const records = input.records && typeof input.records.call === "function" ? input.records : null
|
|
106
|
+
const defaultScope = typeof input.scope === "function" ? input.scope : () => undefined
|
|
107
|
+
/** Whether the spool holds anything (pending, in flight or quarantined). */
|
|
108
|
+
const hasEntries = typeof input.hasEntries === "function" ? input.hasEntries : () => true
|
|
109
|
+
const file = config?.stateDir ? path.join(config.stateDir, FILE) : null
|
|
110
|
+
let state = load()
|
|
111
|
+
let cached = null
|
|
112
|
+
let lastRefresh = 0
|
|
113
|
+
|
|
114
|
+
function empty() {
|
|
115
|
+
return { version: 1, spaces: {}, entries: [], pending: {} }
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function load() {
|
|
119
|
+
if (!file) return empty()
|
|
120
|
+
try {
|
|
121
|
+
const data = JSON.parse(fs.readFileSync(file, "utf8"))
|
|
122
|
+
if (!data || typeof data !== "object") return empty()
|
|
123
|
+
return {
|
|
124
|
+
version: 1,
|
|
125
|
+
spaces: data.spaces && typeof data.spaces === "object" ? data.spaces : {},
|
|
126
|
+
entries: Array.isArray(data.entries) ? data.entries : [],
|
|
127
|
+
pending: data.pending && typeof data.pending === "object" ? data.pending : {},
|
|
128
|
+
}
|
|
129
|
+
} catch {
|
|
130
|
+
return empty()
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Temp file, fsync, rename: a crash leaves the old state or the new. */
|
|
135
|
+
function save() {
|
|
136
|
+
if (!file) return
|
|
137
|
+
try {
|
|
138
|
+
fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 })
|
|
139
|
+
const temp = `${file}.${process.pid}-${randomBytes(4).toString("hex")}.tmp`
|
|
140
|
+
const fd = fs.openSync(temp, "wx", 0o600)
|
|
141
|
+
try {
|
|
142
|
+
fs.writeSync(fd, JSON.stringify(state))
|
|
143
|
+
fs.fsyncSync(fd)
|
|
144
|
+
} finally {
|
|
145
|
+
fs.closeSync(fd)
|
|
146
|
+
}
|
|
147
|
+
fs.renameSync(temp, file)
|
|
148
|
+
} catch (error) {
|
|
149
|
+
log.warn("forget purge state not written", errorMessage(error))
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Drop expired entries and cap the rest. */
|
|
154
|
+
function prune() {
|
|
155
|
+
const at = now()
|
|
156
|
+
const before = state.entries.length
|
|
157
|
+
state.entries = state.entries.filter((entry) => Date.parse(entry.expires ?? "") > at).slice(-MAX_ENTRIES)
|
|
158
|
+
if (state.entries.length !== before) cached = null
|
|
159
|
+
const spaces = Object.entries(state.spaces).sort((a, b) => String(b[1]?.seen ?? "").localeCompare(String(a[1]?.seen ?? "")))
|
|
160
|
+
if (spaces.length > MAX_SPACES) state.spaces = Object.fromEntries(spaces.slice(0, MAX_SPACES))
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function matcher() {
|
|
164
|
+
if (!cached) {
|
|
165
|
+
prune()
|
|
166
|
+
cached = purgeMatcher(state.entries)
|
|
167
|
+
}
|
|
168
|
+
return cached
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function addEntries(space, entries) {
|
|
172
|
+
const known = new Set(state.entries.map((entry) => entry.id))
|
|
173
|
+
let added = 0
|
|
174
|
+
for (const entry of entries) {
|
|
175
|
+
if (!entry || typeof entry.id !== "string" || known.has(entry.id)) continue
|
|
176
|
+
if (typeof entry.salt !== "string" || !Array.isArray(entry.hashes) || !Array.isArray(entry.lengths)) continue
|
|
177
|
+
state.entries.push({ id: entry.id, space, created: String(entry.created ?? ""), expires: String(entry.expires ?? ""), salt: entry.salt, lengths: entry.lengths, hashes: entry.hashes })
|
|
178
|
+
known.add(entry.id)
|
|
179
|
+
added += 1
|
|
180
|
+
}
|
|
181
|
+
if (added > 0) cached = null
|
|
182
|
+
return added
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function noteSpace(space) {
|
|
186
|
+
const key = String(space ?? DEFAULT)
|
|
187
|
+
state.spaces[key] = { ...(state.spaces[key] ?? {}), seen: new Date(now()).toISOString() }
|
|
188
|
+
return key
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Ask the service for the caller's entries newer than each space's cursor.
|
|
193
|
+
* The cursor moves only past entries it stored.
|
|
194
|
+
*
|
|
195
|
+
* @param {{ spaces?: string[], scope?: any }} [options]
|
|
196
|
+
* @returns {Promise<{ ok: boolean, added: number, failed: string[], denied: string[], unsupported: string[] }>}
|
|
197
|
+
*/
|
|
198
|
+
async function refresh(options = {}) {
|
|
199
|
+
const result = { ok: true, added: 0, failed: [], denied: [], unsupported: [] }
|
|
200
|
+
if (!records) return { ...result, ok: false, failed: ["no records client"] }
|
|
201
|
+
const spaces = options.spaces ?? [...new Set([DEFAULT, ...Object.keys(state.spaces)])]
|
|
202
|
+
for (const space of spaces) {
|
|
203
|
+
const key = noteSpace(space)
|
|
204
|
+
const since = state.spaces[key]?.since
|
|
205
|
+
try {
|
|
206
|
+
const answer = await records.call("forgotten.match", since ? { since } : {}, {
|
|
207
|
+
scope: options.scope ?? defaultScope(),
|
|
208
|
+
...(key ? { space: key } : {}),
|
|
209
|
+
timeoutMs: MATCH_TIMEOUT_MS,
|
|
210
|
+
})
|
|
211
|
+
const entries = Array.isArray(answer?.entries) ? answer.entries : []
|
|
212
|
+
result.added += addEntries(key, entries)
|
|
213
|
+
const newest = entries.map((entry) => String(entry?.created ?? "")).filter((value) => !Number.isNaN(Date.parse(value))).sort().pop()
|
|
214
|
+
if (newest && (!since || newest > since)) state.spaces[key].since = newest
|
|
215
|
+
} catch (error) {
|
|
216
|
+
if (unsupported(error)) {
|
|
217
|
+
result.unsupported.push(key)
|
|
218
|
+
continue
|
|
219
|
+
}
|
|
220
|
+
if (denied(error)) {
|
|
221
|
+
result.denied.push(key)
|
|
222
|
+
continue
|
|
223
|
+
}
|
|
224
|
+
result.ok = false
|
|
225
|
+
result.failed.push(key)
|
|
226
|
+
log.warn("forgotten entries could not be read; the spool keeps what it holds until they can", { space: key || "default", error: errorMessage(error) })
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
lastRefresh = now()
|
|
230
|
+
prune()
|
|
231
|
+
save()
|
|
232
|
+
return result
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** Whether an entry of the spool holds forgotten text: the purge ids it matches. */
|
|
236
|
+
function entryMatches(entry, raw) {
|
|
237
|
+
const m = matcher()
|
|
238
|
+
if (m.size === 0) return []
|
|
239
|
+
const texts = entry ? stringsOf(entry.data ?? entry) : [String(raw ?? "")]
|
|
240
|
+
const ids = new Set()
|
|
241
|
+
for (const text of texts) for (const id of m.matches(text)) ids.add(id)
|
|
242
|
+
return [...ids]
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Delete every spooled, in-flight and quarantined entry, and its state,
|
|
247
|
+
* that holds forgotten text. Logs file names and purge ids only.
|
|
248
|
+
*/
|
|
249
|
+
function purge() {
|
|
250
|
+
if (matcher().size === 0) return { purged: 0, failed: 0 }
|
|
251
|
+
const found = []
|
|
252
|
+
const result = spool.purgeMatching((entry, info) => {
|
|
253
|
+
const ids = entryMatches(entry, info.raw)
|
|
254
|
+
if (ids.length > 0) found.push({ file: info.file, state: info.state, purge: ids })
|
|
255
|
+
return ids.length > 0
|
|
256
|
+
})
|
|
257
|
+
if (found.length > 0) log.info("spooled entries holding forgotten text deleted", { purged: result.purged, entries: found.map((item) => ({ file: item.file, state: item.state, purge: item.purge })) })
|
|
258
|
+
return result
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** When a space's mark was set, in ms; a mark from 0.9.8's first cut is an ISO string. */
|
|
262
|
+
function markedAt(key) {
|
|
263
|
+
const mark = state.pending[key]
|
|
264
|
+
return Date.parse(typeof mark === "string" ? mark : mark?.at ?? "")
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** The spaces whose entries wait: marked, and within PENDING_MAX_MS. */
|
|
268
|
+
function livePending() {
|
|
269
|
+
const at = now()
|
|
270
|
+
return Object.keys(state.pending).filter((key) => !(at - markedAt(key) >= PENDING_MAX_MS) && Number.isFinite(markedAt(key)))
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** End the marks past PENDING_MAX_MS, with one log line each. */
|
|
274
|
+
function expirePending() {
|
|
275
|
+
const at = now()
|
|
276
|
+
for (const key of Object.keys(state.pending)) {
|
|
277
|
+
const since = markedAt(key)
|
|
278
|
+
if (Number.isFinite(since) && at - since < PENDING_MAX_MS) continue
|
|
279
|
+
delete state.pending[key]
|
|
280
|
+
log.warn("pending forget ended after 24 hours without reading its purge entries; the spool was purged with this machine's own entries, and the space's turns replay again", { space: key || "default" })
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/** Clear the marks a refresh settled: read, refused for good (401 or 403), or a service without the contract. */
|
|
285
|
+
function settle(keys, fetched) {
|
|
286
|
+
for (const key of keys) {
|
|
287
|
+
if (fetched.failed.includes(key)) continue
|
|
288
|
+
if (fetched.denied.includes(key)) log.warn("forgotten entries refused (401 or 403); the spool was purged with this machine's own entries, and the pending mark is cleared", { space: key || "default" })
|
|
289
|
+
delete state.pending[key]
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* After a forget succeeded: keep a local entry for the text this plugin
|
|
295
|
+
* held, mark the space pending, fetch the service's entries for it, purge,
|
|
296
|
+
* and clear the mark unless the fetch failed for a passing reason.
|
|
297
|
+
*
|
|
298
|
+
* @param {{ space?: string, texts?: string[], scope?: any }} input
|
|
299
|
+
*/
|
|
300
|
+
async function afterForget({ space, texts = [], scope } = {}) {
|
|
301
|
+
const key = noteSpace(space)
|
|
302
|
+
const at = now()
|
|
303
|
+
const local = localPurgeEntry(texts.filter(Boolean).map(String), {
|
|
304
|
+
id: `local_${at.toString(36)}${randomBytes(6).toString("hex")}`,
|
|
305
|
+
salt: randomBytes(32).toString("base64url"),
|
|
306
|
+
created: new Date(at).toISOString(),
|
|
307
|
+
expires: new Date(at + PURGE_WINDOW_MS).toISOString(),
|
|
308
|
+
})
|
|
309
|
+
if (local) addEntries(key, [local])
|
|
310
|
+
state.pending[key] = { at: new Date(at).toISOString() }
|
|
311
|
+
save()
|
|
312
|
+
const fetched = await refresh({ spaces: [key], scope })
|
|
313
|
+
const purged = purge()
|
|
314
|
+
settle([key], fetched)
|
|
315
|
+
save()
|
|
316
|
+
return { purged: purged.purged, failed: purged.failed, fetched: fetched.ok && fetched.denied.length === 0, pending: key in state.pending, entries: matcher().size }
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/**
|
|
320
|
+
* Before the spool is replayed (startup, a drain, /mem-sync): end old
|
|
321
|
+
* marks, fetch new entries when due (always while a space is pending), and
|
|
322
|
+
* purge. A pending space's entries wait in the replay itself (`sendable`);
|
|
323
|
+
* nothing else is held back.
|
|
324
|
+
*/
|
|
325
|
+
async function beforeFlush({ force = false } = {}) {
|
|
326
|
+
expirePending()
|
|
327
|
+
const pending = Object.keys(state.pending)
|
|
328
|
+
// Nothing queued and no forget pending: nothing to purge or hold back.
|
|
329
|
+
if (pending.length === 0 && !force && !hasEntries()) {
|
|
330
|
+
save()
|
|
331
|
+
return { ok: true }
|
|
332
|
+
}
|
|
333
|
+
if (force || pending.length > 0 || now() - lastRefresh >= PURGE_REFRESH_MS) settle(pending, await refresh())
|
|
334
|
+
purge()
|
|
335
|
+
save()
|
|
336
|
+
return { ok: true }
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/** The space a spool entry was captured in, as far as this machine knew it ("" when not). */
|
|
340
|
+
function spaceOf(entry) {
|
|
341
|
+
return String(entry?.space ?? entry?.scope?.space_id ?? "")
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* Whether a spool entry may be sent now (checked just before each replay):
|
|
346
|
+
* false when it holds forgotten text (it is deleted), "hold" when its space
|
|
347
|
+
* has a pending forget (it waits), true otherwise.
|
|
348
|
+
*
|
|
349
|
+
* @returns {boolean | "hold"}
|
|
350
|
+
*/
|
|
351
|
+
function sendable(entry) {
|
|
352
|
+
if (entryMatches(entry, "").length > 0) return false
|
|
353
|
+
return livePending().includes(spaceOf(entry)) ? "hold" : true
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/**
|
|
357
|
+
* A copy of a payload with each run of forgotten words masked, and the
|
|
358
|
+
* purge ids it hit (empty when nothing changed: `value` is then the input).
|
|
359
|
+
*
|
|
360
|
+
* @template T
|
|
361
|
+
* @param {T} value
|
|
362
|
+
* @returns {{ value: T, hits: string[] }}
|
|
363
|
+
*/
|
|
364
|
+
function mask(value) {
|
|
365
|
+
const m = matcher()
|
|
366
|
+
if (m.size === 0) return { value, hits: [] }
|
|
367
|
+
const hits = new Set()
|
|
368
|
+
const masked = maskValue(value, m, hits)
|
|
369
|
+
return hits.size > 0 ? { value: masked, hits: [...hits] } : { value, hits: [] }
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
function status() {
|
|
373
|
+
return { entries: matcher().size, pending: livePending().length, spaces: Object.keys(state.spaces).length }
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
return { refresh, purge, afterForget, beforeFlush, sendable, mask, status, matches: (text) => matcher().matches(text) }
|
|
377
|
+
}
|