@joe-broadhead/opencode-mem 0.9.2 → 0.9.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/CHANGELOG.md +1605 -0
  2. package/README.md +135 -43
  3. package/docs/agent-protocol.md +29 -28
  4. package/docs/permissions.md +148 -0
  5. package/package.json +277 -16
  6. package/skills/mem.pages/SKILL.md +35 -16
  7. package/skills/mem.sharing/SKILL.md +39 -6
  8. package/skills/mem.sources/SKILL.md +41 -10
  9. package/skills/mem.usage/SKILL.md +47 -16
  10. package/src/backend/records.js +413 -50
  11. package/src/backend/v3.js +15 -10
  12. package/src/call-shapes.js +18 -25
  13. package/src/capture-pause.js +136 -0
  14. package/src/capture.js +7 -4
  15. package/src/collector-service.js +212 -35
  16. package/src/collector.js +186 -47
  17. package/src/commands.js +37 -0
  18. package/src/compaction.js +4 -2
  19. package/src/config.js +162 -1
  20. package/src/connect.js +353 -0
  21. package/src/copies.js +28 -3
  22. package/src/credentials.js +35 -7
  23. package/src/curation.js +33 -14
  24. package/src/events.js +144 -36
  25. package/src/forget-purge.js +377 -0
  26. package/src/forgotten.js +356 -0
  27. package/src/hooks.js +9 -4
  28. package/src/index.js +25 -6
  29. package/src/permissions.js +116 -38
  30. package/src/protocol.js +21 -11
  31. package/src/records.js +4 -1
  32. package/src/runtime.js +207 -16
  33. package/src/shapes.js +63 -0
  34. package/src/sources.js +183 -12
  35. package/src/spaces.js +58 -1
  36. package/src/spool.js +442 -64
  37. package/src/stated.js +334 -0
  38. package/src/storable.js +477 -0
  39. package/src/tools/aliases.js +36 -147
  40. package/src/tools/commitments.js +69 -12
  41. package/src/tools/held-write.js +9 -1
  42. package/src/tools/index.js +29 -44
  43. package/src/tools/ingest.js +33 -128
  44. package/src/tools/memory.js +594 -901
  45. package/src/tools/page-sections.js +147 -0
  46. package/src/tools/page.js +314 -82
  47. package/src/tools/recall-browse.js +173 -0
  48. package/src/tools/recall.js +79 -19
  49. package/src/tools/share.js +494 -131
  50. package/src/tools/shared.js +97 -19
  51. package/src/tools/spaces.js +215 -104
  52. package/src/tools/status.js +31 -3
  53. package/src/tools/wiki.js +5 -13
  54. package/src/util.js +10 -20
  55. package/src/views/contracts.js +137 -6
  56. package/src/views/overdue.js +125 -0
  57. package/src/views/profile.js +154 -0
  58. package/src/views/records-reads.js +132 -33
  59. package/src/views/rpc.js +63 -12
  60. package/src/views/shape.js +49 -0
  61. package/views/commitments.html +31 -4
  62. package/views/home.html +22 -3
  63. package/views/memory.html +126 -24
  64. package/src/erase.js +0 -36
@@ -1,3 +1,5 @@
1
+ import { AsyncLocalStorage } from "node:async_hooks"
2
+ import { createHash, randomBytes } from "node:crypto"
1
3
  import { errorMessage, sleep } from "../util.js"
2
4
  import { codedMessage, codedParts, failureFor } from "../errors.js"
3
5
  import { scopeHeaders } from "../scope.js"
@@ -23,16 +25,32 @@ import { scopeHeaders } from "../scope.js"
23
25
  * (`src/errors.js`), and it keeps the service's `code` (`version_conflict`,
24
26
  * `refused`, `not_found`, …), the HTTP status, and the error body.
25
27
  *
26
- * Reads are retried on a timeout, a 429 or a 5xx, `config.retries` times;
27
- * writes never are: a retried write can duplicate a record.
28
+ * Reads are retried on a timeout, a 429 or a 5xx, `config.retries` times.
29
+ *
30
+ * Writes (docs/design/records-idempotency.md): a write whose response is lost
31
+ * (a timeout, a reset, a 408, a 5xx, an unreadable 2xx) has an **unknown**
32
+ * outcome. To a records service that advertises replay for the method
33
+ * (`space.stats` `idempotent_replay`, asked before the first write to a
34
+ * space and kept), the client sends an `idempotency_key` and, after an
35
+ * unknown outcome, resends the same body with the same key,
36
+ * `config.recordsWriteRetries` times, on the cached capability (no fresh ask:
37
+ * a slow service would fail it). The service applies the key once, answers
38
+ * the original result, and marks every keyed answer `keyed: true`; a retry
39
+ * answered without it is reported as possibly applied twice. To an older
40
+ * service it sends no key and never retries a write: an unknown outcome is
41
+ * reported as one, to check before retrying.
28
42
  *
29
43
  * @param {Record<string, any>} config the resolved configuration
30
- * @param {{ log?: any, fetchImpl?: typeof fetch, credentials?: { bearer(): Promise<string>, refresh(rejected: string): Promise<string> }, baseURL?: string | (() => string) }} [deps]
44
+ * @param {{ log?: any, fetchImpl?: typeof fetch, credentials?: { bearer(): Promise<string>, refresh(rejected: string): Promise<string> }, baseURL?: string | (() => string), now?: () => number, sleep?: (ms: number) => Promise<void> }} [deps]
31
45
  */
32
46
  export function createRecordsClient(config, deps = {}) {
33
47
  const log = deps.log
34
48
  const fetchImpl = deps.fetchImpl ?? globalThis.fetch
35
49
  const credentials = deps.credentials ?? null
50
+ const clock = deps.now ?? Date.now
51
+ const pause = deps.sleep ?? sleep
52
+ /** Per space (or session, when no space is named): the methods the service replays, until when. */
53
+ const capabilities = new Map()
36
54
 
37
55
  /** Read on every call: inside Jelly the gateway origin can appear after setup. */
38
56
  function base() {
@@ -78,7 +96,7 @@ export function createRecordsClient(config, deps = {}) {
78
96
  /**
79
97
  * @param {string} method a §2.4 method, for example `records.put`
80
98
  * @param {unknown} body
81
- * @param {{ sessionID?: string, space?: string, scope?: Record<string, any>, timeoutMs?: number }} [options]
99
+ * @param {{ sessionID?: string, space?: string, scope?: Record<string, any>, timeoutMs?: number, retries?: number }} [options]
82
100
  */
83
101
  async function call(method, body = {}, options = {}) {
84
102
  const name = String(method ?? "")
@@ -92,84 +110,413 @@ export function createRecordsClient(config, deps = {}) {
92
110
  next: "set the Memory address in Memory's settings, or sign in to Jelly",
93
111
  })
94
112
  }
95
- const headers = { ...requestHeaders(options), "x-mem-tz": timeZone() }
96
113
  const read = READ_METHODS.has(name)
97
- let request
98
- try {
99
- request = JSON.stringify(body ?? {})
100
- } catch (error) {
101
- throw new RecordsError(path, 400, { message: `request could not be encoded: ${errorMessage(error)}` })
114
+ const encode = (value) => {
115
+ try {
116
+ return JSON.stringify(value ?? {})
117
+ } catch (error) {
118
+ throw new RecordsError(path, 400, { message: `request could not be encoded: ${errorMessage(error)}` })
119
+ }
102
120
  }
121
+ // Encoded before anything is sent: a body that cannot be encoded is a definite failure.
122
+ const request = encode(body)
123
+ const plan = read ? null : await writePlan(name, body, options)
124
+ const exchange = await connection(name, path, options, read)
125
+ if (read) return readLoop(name, path, request, exchange, options)
126
+ return writeLoop(name, path, plan, request, encode, exchange, options)
127
+ }
128
+
129
+ /**
130
+ * One HTTP exchange at a time on one bearer: a 401 asks the credentials for
131
+ * another token once and sends again (the gateway refused the request, so
132
+ * nothing was applied). Transport failures throw as they are.
133
+ */
134
+ async function connection(name, path, options, read) {
135
+ const headers = { ...requestHeaders(options), "x-mem-tz": timeZone() }
103
136
  const timeoutMs = Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : read ? config.timeoutMs : (config.recordsWriteTimeoutMs ?? 10000)
104
- const retries = read ? Math.max(0, Number(config.retries) || 0) : 0
105
137
  let token = await bearer(path)
106
138
  let renewed = false
107
- for (let attempt = 0; ; attempt += 1) {
108
- const controller = new AbortController()
109
- const timer = setTimeout(() => controller.abort(), timeoutMs)
110
- let response
111
- try {
112
- response = await fetchImpl(`${root}/v1/${name}`, {
113
- method: "POST",
114
- headers: token ? { ...headers, authorization: `Bearer ${token}` } : headers,
115
- body: request,
116
- signal: controller.signal,
117
- })
118
- if (response.status === 401 && token && !renewed) {
119
- const next = await refreshed(token)
120
- if (next) {
121
- try {
122
- await response.body?.cancel()
123
- } catch {
124
- /* the refused body is not read */
139
+ const exchange = async (request) => {
140
+ for (;;) {
141
+ const controller = new AbortController()
142
+ const timer = setTimeout(() => controller.abort(), timeoutMs)
143
+ try {
144
+ const response = await fetchImpl(`${base()}/v1/${name}`, {
145
+ method: "POST",
146
+ headers: token ? { ...headers, authorization: `Bearer ${token}` } : headers,
147
+ body: request,
148
+ signal: controller.signal,
149
+ })
150
+ if (response.status === 401 && token && !renewed) {
151
+ const next = await refreshed(token)
152
+ if (next) {
153
+ try {
154
+ await response.body?.cancel()
155
+ } catch {
156
+ /* the refused body is not read */
157
+ }
158
+ token = next
159
+ renewed = true
160
+ continue
125
161
  }
126
- token = next
127
- renewed = true
128
- attempt -= 1
129
- continue
130
162
  }
163
+ return response
164
+ } finally {
165
+ clearTimeout(timer)
131
166
  }
132
- if (!read && (response.status === 408 || response.status >= 500)) throw unknownWrite(path, response.status, `server returned ${response.status}`)
167
+ }
168
+ }
169
+ exchange.timeoutMs = timeoutMs
170
+ return exchange
171
+ }
172
+
173
+ async function readLoop(name, path, request, exchange, options = {}) {
174
+ const retries = Number.isInteger(options.retries) ? Math.max(0, options.retries) : Math.max(0, Number(config.retries) || 0)
175
+ for (let attempt = 0; ; attempt += 1) {
176
+ let response
177
+ try {
178
+ response = await exchange(request)
133
179
  const answer = await readAnswer(response, path)
134
- if (response.ok) {
135
- if (!read && !validWriteAnswer(name, answer)) throw unknownWrite(path, response.status, "the service returned an invalid success response")
136
- return answer
137
- }
180
+ if (response.ok) return answer
138
181
  if (attempt < retries && (response.status >= 500 || response.status === 429)) {
139
182
  await sleep(150 * 2 ** attempt)
140
183
  continue
141
184
  }
142
185
  throw RecordsError.fromAnswer(path, response.status, answer)
143
186
  } catch (error) {
144
- if (error instanceof RecordsError) {
145
- if (!read && response?.ok && !error.outcomeUnknown) throw unknownWrite(path, response.status, "the service returned an unreadable success response")
146
- throw error
147
- }
187
+ if (error instanceof RecordsError) throw error
148
188
  const timedOut = error?.name === "AbortError"
149
189
  if (attempt < retries) {
150
190
  await sleep(150 * 2 ** attempt)
151
191
  continue
152
192
  }
153
193
  log?.debug?.("records request failed", { method: name, error: errorMessage(error) })
154
- if (!read) throw unknownWrite(path, timedOut ? 408 : 0, timedOut ? `timeout after ${timeoutMs}ms` : errorMessage(error))
155
194
  throw timedOut
156
- ? new RecordsError(path, 408, { message: `timeout after ${timeoutMs}ms` })
195
+ ? new RecordsError(path, 408, { message: `timeout after ${exchange.timeoutMs}ms` })
157
196
  : new RecordsError(path, 0, { message: errorMessage(error) })
158
- } finally {
159
- clearTimeout(timer)
160
197
  }
161
198
  }
162
199
  }
163
200
 
201
+ /**
202
+ * How a write is sent: with a key the client derives (`keyed`), with the
203
+ * key the caller put in the body (a held write's own ledger), or with none.
204
+ * `retry` is whether an unknown outcome may be retried at all; each retry
205
+ * still asks the service first.
206
+ */
207
+ async function writePlan(name, body, options) {
208
+ if (!KEYED_METHODS.has(name) || !body || typeof body !== "object" || Array.isArray(body) || body.migrate_legacy === true) return { keyed: false, retry: false }
209
+ const own = typeof body.idempotency_key === "string" || (body.held_batch && typeof body.held_batch === "object")
210
+ if (!(await replays(name, options))) return { keyed: false, retry: false }
211
+ if (own) return { keyed: false, retry: true }
212
+ const space = String(options.space ?? options.scope?.space_id ?? "")
213
+ return { keyed: true, retry: true, key: writeKey(name, space, body) }
214
+ }
215
+
216
+ async function writeLoop(name, path, plan, request, encode, exchange, options) {
217
+ const first = clock()
218
+ const limit = plan.retry ? Math.min(Math.max(0, Number(config.recordsWriteRetries ?? 2) || 0), 5) : 0
219
+ let unknown = null
220
+ let attempts = 0
221
+ for (let attempt = 0; ; attempt += 1) {
222
+ attempts += 1
223
+ const sent = plan.keyed ? encode({ ...decoded(request), idempotency_key: plan.key, idempotency_age_ms: attempt === 0 ? 0 : Math.max(0, clock() - first) }) : request
224
+ const outcome = await attemptWrite(name, path, sent, exchange, attempt)
225
+ if (outcome.answer) {
226
+ if (outcome.answer.replayed === true) log?.debug?.("records write replayed", { method: name, attempt })
227
+ // A keyed write answered without `keyed: true` reached a service that
228
+ // ignored the key (it was rolled back): forget the capability. On a
229
+ // retry, an earlier attempt may have been applied too.
230
+ if (plan.keyed && outcome.answer.keyed !== true) {
231
+ forgetCapability(options)
232
+ if (attempt > 0) throw possiblyTwice(path, outcome.answer, attempts)
233
+ }
234
+ if (outcome.answer.replayed === true && outcome.answer.unavailable === true) throw alreadyApplied(path, outcome.answer)
235
+ return outcome.answer
236
+ }
237
+ if (outcome.definite) throw outcome.definite
238
+ unknown = outcome.unknown
239
+ if (attempt >= limit) break
240
+ await pause(Math.min(250 * 2 ** attempt, 2000))
241
+ // A retry trusts the capability the first attempt was keyed on, with no
242
+ // fresh ask: when the write has just timed out, an ask would time out
243
+ // too, and no retry would be sent. The cache is dropped when any keyed
244
+ // answer lacks `keyed: true`, and then no retry is sent; a retry that
245
+ // reaches a service ignoring its key is reported as possibly applied
246
+ // twice (above).
247
+ if (!replaysCached(name, options)) break
248
+ log?.debug?.("records write outcome unknown; retrying with the same key", { method: name, attempt: attempt + 1 })
249
+ }
250
+ throw unknownWrite(path, unknown.status, unknown.what, attempts > 1)
251
+ }
252
+
253
+ /**
254
+ * One attempt of a write: `{answer}`, `{definite}` (an error: no attempt was
255
+ * applied), or `{unknown}` (whether this or an earlier attempt was applied
256
+ * is not known). On a retry, only the records service's own refusal is
257
+ * definite: it looks the key up before any other check, and a write commits
258
+ * with its receipt, so no receipt means no attempt was applied.
259
+ */
260
+ async function attemptWrite(name, path, request, exchange, attempt) {
261
+ let response
262
+ try {
263
+ response = await exchange(request)
264
+ } catch (error) {
265
+ // Anything thrown once the request may have left is an unknown outcome.
266
+ const timedOut = error?.name === "AbortError"
267
+ log?.debug?.("records write failed", { method: name, attempt, error: errorMessage(error) })
268
+ return { unknown: { status: timedOut ? 408 : 0, what: timedOut ? `timeout after ${exchange.timeoutMs}ms` : errorMessage(error) } }
269
+ }
270
+ if (response.status === 408 || response.status >= 500) {
271
+ try {
272
+ await response.body?.cancel()
273
+ } catch {
274
+ /* not read */
275
+ }
276
+ return { unknown: { status: response.status, what: `server returned ${response.status}` } }
277
+ }
278
+ let answer
279
+ try {
280
+ answer = await readAnswer(response, path)
281
+ } catch {
282
+ return response.ok
283
+ ? { unknown: { status: response.status, what: "the service returned an unreadable success response" } }
284
+ : { unknown: { status: response.status, what: `the service returned an unreadable ${response.status}` } }
285
+ }
286
+ if (response.ok) {
287
+ if (!validWriteAnswer(name, answer)) return { unknown: { status: response.status, what: "the service returned an invalid success response" } }
288
+ return { answer }
289
+ }
290
+ const error = RecordsError.fromAnswer(path, response.status, answer)
291
+ if (attempt === 0) return { definite: error }
292
+ const fromService = answer?.error && typeof answer.error === "object" && typeof answer.error.code === "string"
293
+ if (fromService && response.status !== 401 && !UNCHECKABLE.has(answer.error.code)) return { definite: error }
294
+ return { unknown: { status: response.status, what: error.what.replace(/^[^:]+: /, "") } }
295
+ }
296
+
297
+ /**
298
+ * Whether the service replays `method` for the space these options name,
299
+ * from `space.stats`. A service that replays is remembered for the life of
300
+ * the process, until a keyed answer comes back without `keyed: true` (a
301
+ * mismatch: the service was rolled back); one that does not, or refused the
302
+ * ask, is asked again after 10 minutes, and an ask that did not reach it
303
+ * after 30 seconds. The ask has a short timeout and no retry, so it never
304
+ * holds the first write up for long; retries never ask (`replaysCached`).
305
+ * The ask names the same space and session as the write, so it reaches the
306
+ * same cell and changes nothing the write would not.
307
+ */
308
+ async function replays(method, options) {
309
+ const where = capabilityKey(options)
310
+ const now = clock()
311
+ const cached = capabilities.get(where)
312
+ if (cached && now < cached.until) return cached.methods.has(method)
313
+ let methods = new Set()
314
+ let until = now + CAPABILITY_RETRY_MS
315
+ try {
316
+ const stats = await call("space.stats", {}, { scope: options.scope, space: options.space, sessionID: options.sessionID, timeoutMs: CAPABILITY_TIMEOUT_MS, retries: 0 })
317
+ const replay = stats?.idempotent_replay
318
+ if (replay && replay.version === 1 && Array.isArray(replay.methods)) methods = new Set(replay.methods.filter((value) => typeof value === "string"))
319
+ if (methods.size > 0) until = Number.POSITIVE_INFINITY
320
+ } catch (error) {
321
+ // A refusal (a grant-only caller may not read space.stats) is an answer;
322
+ // a failure to reach the service is asked again sooner.
323
+ const status = Number(error?.status) || 0
324
+ if (!(status >= 400 && status < 500 && status !== 408 && status !== 429)) until = now + CAPABILITY_FAILURE_TTL_MS
325
+ log?.debug?.("records capability check failed; writes go without a key", { error: errorMessage(error) })
326
+ }
327
+ capabilities.delete(where)
328
+ capabilities.set(where, { methods, until })
329
+ if (capabilities.size > CAPABILITY_MAX) capabilities.delete(capabilities.keys().next().value)
330
+ return methods.has(method)
331
+ }
332
+
333
+ /** Whether the cached capability (no ask) still lists `method`: what a retry relies on. */
334
+ function replaysCached(method, options) {
335
+ const cached = capabilities.get(capabilityKey(options))
336
+ return Boolean(cached && clock() < cached.until && cached.methods.has(method))
337
+ }
338
+
339
+ function capabilityKey(options) {
340
+ return options.space ? `space:${options.space}` : `session:${options.sessionID ?? ""}`
341
+ }
342
+
343
+ function forgetCapability(options) {
344
+ capabilities.delete(capabilityKey(options))
345
+ log?.debug?.("records service ignored an idempotency key; asking again before the next write")
346
+ }
347
+
164
348
  return { call, base }
165
349
  }
166
350
 
167
- function unknownWrite(path, status, what) {
168
- const error = new RecordsError(path, status, { message: `${what}; the write outcome is unknown`, next: "check before retrying: list or read the current records" })
351
+ /** The body a request was encoded from: every attempt sends the same body, encoded once. */
352
+ function decoded(request) {
353
+ return JSON.parse(request)
354
+ }
355
+
356
+ /** Codes from the service that do not tell whether an earlier attempt was applied. */
357
+ const UNCHECKABLE = new Set(["idempotency_expired", "idempotency_conflict"])
358
+ /** How soon a service that does not replay (or refused the ask) is asked again. */
359
+ const CAPABILITY_RETRY_MS = 10 * 60 * 1000
360
+ const CAPABILITY_FAILURE_TTL_MS = 30 * 1000
361
+ const CAPABILITY_TIMEOUT_MS = 1000
362
+ const CAPABILITY_MAX = 500
363
+
364
+ /** The methods a records service may replay (docs/design/records-idempotency.md). */
365
+ export const KEYED_METHODS = Object.freeze(new Set(["records.put", "records.revise", "records.append", "records.move", "records.delete"]))
366
+
367
+ function unknownWrite(path, status, what, retried = false) {
368
+ const error = new RecordsError(path, status, {
369
+ message: `${what}; the write outcome is unknown${retried ? " (it was retried with the same idempotency key)" : ""}`,
370
+ next: "check before retrying: list or read the current records",
371
+ })
169
372
  error.outcomeUnknown = true
170
373
  return error
171
374
  }
172
375
 
376
+ /**
377
+ * A retry answered without `keyed: true`: the service ignored the key (it was
378
+ * rolled back to one without replay), so it applied this attempt as a new
379
+ * write, and an earlier attempt whose answer was lost may have been applied
380
+ * too. Neither a success nor an unknown outcome to retry: check first.
381
+ */
382
+ function possiblyTwice(path, answer, attempts) {
383
+ const id = typeof answer?.record?.id === "string" ? ` (${answer.record.id})` : ""
384
+ const error = new RecordsError(path, 200, {
385
+ code: "possibly_applied_twice",
386
+ errorCode: "MEMORY_POSSIBLY_DUPLICATED",
387
+ message: `possibly applied twice: the write was sent ${attempts} times and the records service ignored its idempotency key${id}, so an earlier attempt may have been applied as well`,
388
+ next: "check before writing again: list or read the current records, and remove a duplicate if there is one",
389
+ })
390
+ error.outcomeUnknown = true
391
+ error.possiblyDuplicated = true
392
+ return error
393
+ }
394
+
395
+ /** A replay whose record has since been deleted or hidden: the write was applied, once. */
396
+ function alreadyApplied(path, answer) {
397
+ const id = typeof answer.record_id === "string" ? answer.record_id : "its record"
398
+ const error = new RecordsError(path, 200, {
399
+ code: "already_applied",
400
+ errorCode: "MEMORY_ALREADY_WRITTEN",
401
+ message: `the write was already applied (${id}); that record has since been deleted or is no longer readable, so it was not applied again`,
402
+ next: "read the current records before writing again",
403
+ })
404
+ error.committed = true
405
+ return error
406
+ }
407
+
408
+ // ---------------------------------------------------------------------------
409
+ // Keys (docs/design/records-idempotency.md#keys)
410
+
411
+ const writeCalls = new AsyncLocalStorage()
412
+
413
+ /**
414
+ * How many times each (call id, tool, arguments) has run in each message of
415
+ * this process, so the n-th identical call in a message gets the n-th root.
416
+ * Counted per message: unrelated calls, in this message or in others, never
417
+ * push a message's own counts out. Bounded by messages, the least recently
418
+ * used first; a message keeps its counts until it is pushed out, however many
419
+ * calls it makes.
420
+ */
421
+ const occurrences = new Map()
422
+ const OCCURRENCE_MESSAGES_MAX = 10000
423
+
424
+ /**
425
+ * Run one tool call as one logical call, and derive the root its writes'
426
+ * keys come from.
427
+ *
428
+ * OpenCode 2.2.6 passes `sessionID`, `messageID` and `id` in a tool's
429
+ * context, but under Code Mode `id` is the outer `execute` call's id, shared
430
+ * by every tool call in that script. So the root is the session, the message,
431
+ * that id when present, the tool, a canonical digest of the arguments, and the
432
+ * occurrence index of that same (tool, arguments) within the message, counted
433
+ * in memory. A host that sends a call again after a restart starts counting at
434
+ * zero and derives the original keys; two intentional identical calls in one
435
+ * message get indexes 0 and 1 and differ. (Within one process a re-delivery of
436
+ * the same call is indistinguishable from an intentional repeat, and is
437
+ * treated as one.) Without a session and a message the root is random.
438
+ *
439
+ * @template T
440
+ * @param {{ sessionID?: string, messageID?: string, id?: string, callID?: string } | undefined} context
441
+ * @param {string} tool
442
+ * @param {unknown} input the tool's arguments
443
+ * @param {() => T} run
444
+ * @returns {T}
445
+ */
446
+ export function withWriteCall(context, tool, input, run) {
447
+ return writeCalls.run({ root: callRoot(context, tool, input), counts: new Map() }, run)
448
+ }
449
+
450
+ /** The root of the current tool call's keys, or "" outside one (held writes key on it too). */
451
+ export function writeCallRoot() {
452
+ return writeCalls.getStore()?.root ?? ""
453
+ }
454
+
455
+ function callRoot(context, tool, input) {
456
+ const session = context?.sessionID
457
+ const message = context?.messageID
458
+ if (typeof session !== "string" || !session || typeof message !== "string" || !message) return randomBytes(32).toString("hex")
459
+ const callId = context?.id ?? context?.callID
460
+ const call = [typeof callId === "string" ? callId : "", String(tool ?? ""), sha256(canonicalJson(input ?? {}))]
461
+ const where = JSON.stringify([session, message])
462
+ const counts = occurrences.get(where) ?? new Map()
463
+ occurrences.delete(where)
464
+ occurrences.set(where, counts)
465
+ if (occurrences.size > OCCURRENCE_MESSAGES_MAX) occurrences.delete(occurrences.keys().next().value)
466
+ const occurrence = counts.get(JSON.stringify(call)) ?? 0
467
+ counts.set(JSON.stringify(call), occurrence + 1)
468
+ // The root 0.9.3 derived: the same calls give the same keys across the upgrade.
469
+ const group = JSON.stringify([session, message, ...call])
470
+ return sha256(JSON.stringify(["opencode-mem/write/v2", group, occurrence]))
471
+ }
472
+
473
+ /** Forget the occurrence counts, as a restart of the plugin does (tests). */
474
+ export function forgetWriteCalls() {
475
+ occurrences.clear()
476
+ }
477
+
478
+ /**
479
+ * A write's key: `wk1_` and 43 base64url characters of
480
+ * sha256([root, space, method, intent, ordinal]). The ordinal counts earlier
481
+ * writes of the same intent in the call, so two identical writes differ.
482
+ */
483
+ export function writeKey(method, space, payload) {
484
+ const call = writeCalls.getStore()
485
+ const intent = writeIntent(payload)
486
+ if (!call) return `wk1_${digest64(JSON.stringify([randomBytes(32).toString("hex"), space, method, intent]))}`
487
+ const group = JSON.stringify([space, method, intent])
488
+ const ordinal = call.counts.get(group) ?? 0
489
+ call.counts.set(group, ordinal + 1)
490
+ return `wk1_${digest64(JSON.stringify([call.root, space, method, intent, ordinal]))}`
491
+ }
492
+
493
+ /** A write's intent: its body without a key, an age or base_version, keys sorted (as the service's digest). */
494
+ export function writeIntent(payload) {
495
+ const { idempotency_key: _key, idempotency_age_ms: _age, base_version: _version, ...rest } = payload ?? {}
496
+ if (rest.record && typeof rest.record === "object" && !Array.isArray(rest.record)) {
497
+ const { base_version: _recordVersion, ...record } = rest.record
498
+ rest.record = record
499
+ }
500
+ return sha256(canonicalJson(rest))
501
+ }
502
+
503
+ function canonicalJson(value) {
504
+ if (Array.isArray(value)) return `[${value.map((item) => canonicalJson(item === undefined ? null : item)).join(",")}]`
505
+ if (value && typeof value === "object") {
506
+ const keys = Object.keys(value).filter((key) => value[key] !== undefined).sort()
507
+ return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`
508
+ }
509
+ return JSON.stringify(value) ?? "null"
510
+ }
511
+
512
+ function sha256(text) {
513
+ return createHash("sha256").update(text).digest("hex")
514
+ }
515
+
516
+ function digest64(text) {
517
+ return createHash("sha256").update(text).digest("base64url")
518
+ }
519
+
173
520
  /** A successful mutation must contain the service's confirmation, not an empty or unrelated body. */
174
521
  function validWriteAnswer(method, answer) {
175
522
  if (!answer || typeof answer !== "object" || Array.isArray(answer) || answer.binary || Object.keys(answer).length === 0) return false
@@ -179,14 +526,24 @@ function validWriteAnswer(method, answer) {
179
526
  typeof value.body === "string" && typeof value.status === "string" && value.fields &&
180
527
  typeof value.fields === "object" && !Array.isArray(value.fields) && Number.isInteger(value.version) && value.version > 0,
181
528
  )
182
- if (method === "records.put") return Boolean(record(answer.record) || (typeof answer.receipt === "string" && answer.receipt.length > 0) || (answer.legacy_migration === true && answer.deleted === true))
183
- if (["records.revise", "records.append", "records.move"].includes(method)) return record(answer.record)
529
+ // A replay whose record has since gone or become unreadable: applied once, nothing to show.
530
+ const unavailable = answer.replayed === true && answer.unavailable === true && answer.record === null
531
+ if (method === "records.put") return Boolean(record(answer.record) || unavailable || (typeof answer.receipt === "string" && answer.receipt.length > 0) || (answer.legacy_migration === true && answer.deleted === true))
532
+ // A batch revise answers each id's outcome, not a record.
533
+ if (method === "records.revise" && Array.isArray(answer.results)) return answer.results.every((item) => item && typeof item.id === "string" && typeof item.ok === "boolean")
534
+ // A folder move (records.move with path_prefix) answers every moved page.
535
+ if (method === "records.move" && Array.isArray(answer.records) && Array.isArray(answer.moved)) return answer.records.every(record)
536
+ if (["records.revise", "records.append", "records.move"].includes(method)) return record(answer.record) || unavailable
184
537
  if (method === "records.delete") return Array.isArray(answer.deleted) && answer.deleted.every((id) => typeof id === "string")
185
538
  if (method === "review.decide") return record(answer.proposal)
186
539
  const count = (value) => Number.isInteger(value) && value >= 0
187
540
  if (method === "events.ingest") return count(answer.accepted) && count(answer.duplicates) && count(answer.refused) && typeof answer.cursor_saved === "boolean"
188
541
  if (method === "sources.bind") return Boolean(answer.source && typeof answer.source === "object" && typeof answer.source.source === "string")
189
542
  if (method === "sources.unbind") return count(answer.deleted_events)
543
+ if (method === "sources.request" || method === "sources.report") return Boolean(answer.request && typeof answer.request === "object" && typeof answer.request.source === "string")
544
+ if (method === "sources.cancel") return typeof answer.cancelled === "boolean"
545
+ if (method === "forgotten.put") return typeof answer.fingerprint?.id === "string" && /^fp_/.test(answer.fingerprint.id)
546
+ if (method === "forgotten.lift") return Array.isArray(answer.lifted) && answer.lifted.every((id) => typeof id === "string")
190
547
  return true
191
548
  }
192
549
 
@@ -209,6 +566,9 @@ export const READ_METHODS = Object.freeze(
209
566
  "review.list",
210
567
  "routines.status",
211
568
  "space.stats",
569
+ "forgotten.list",
570
+ "forgotten.match",
571
+ "forgotten.check",
212
572
  ]),
213
573
  )
214
574
 
@@ -260,6 +620,9 @@ const CODES = Object.freeze({
260
620
  forbidden: "MEMORY_FORBIDDEN",
261
621
  records_unavailable: "MEMORY_RECORDS_UNAVAILABLE",
262
622
  source_unbound: "MEMORY_SOURCE_UNBOUND",
623
+ // The records service refused a keyed write before applying it: its ledger
624
+ // of recent writes is full. A definite "not saved", never an unknown outcome.
625
+ idempotency_capacity: "MEMORY_NOT_SAVED",
263
626
  })
264
627
 
265
628
  /** What a cell without the records service (503 `records_unavailable`) means, in words a person can act on. */
package/src/backend/v3.js CHANGED
@@ -115,15 +115,16 @@ export function createBackend(config, deps = {}) {
115
115
  * @param {string} path
116
116
  * @param {unknown} body
117
117
  * @param {Record<string, string>} requestHeaders
118
- * @param {{ attempt?: number, auth?: typeof sharedAuth, token?: string, renewed?: boolean }} [options]
118
+ * @param {{ attempt?: number, auth?: typeof sharedAuth, token?: string, renewed?: boolean, timeoutMs?: number }} [options]
119
119
  * @returns {Promise<any>}
120
120
  */
121
121
  async function post(baseURL, path, body, requestHeaders, options = {}) {
122
122
  const { attempt = 0, auth = sharedAuth, renewed = false } = options
123
+ const timeoutMs = Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : config.timeoutMs
123
124
  const token = options.token ?? (await bearerFor(auth, path))
124
- const again = (extra) => post(baseURL, path, body, requestHeaders, { attempt, auth, token, renewed, ...extra })
125
+ const again = (extra) => post(baseURL, path, body, requestHeaders, { attempt, auth, token, renewed, timeoutMs, ...extra })
125
126
  const controller = new AbortController()
126
- const timer = setTimeout(() => controller.abort(), config.timeoutMs)
127
+ const timer = setTimeout(() => controller.abort(), timeoutMs)
127
128
  try {
128
129
  const response = await fetchImpl(`${baseURL}${path}`, {
129
130
  method: "POST",
@@ -173,7 +174,7 @@ export function createBackend(config, deps = {}) {
173
174
  await sleep(150 * 2 ** attempt)
174
175
  return again({ attempt: attempt + 1 })
175
176
  }
176
- throw new BackendError(path, 408, `timeout after ${config.timeoutMs}ms`, 408)
177
+ throw new BackendError(path, 408, `timeout after ${timeoutMs}ms`, 408)
177
178
  }
178
179
  if (RETRYABLE_READ_PATHS.has(path) && attempt < config.retries) {
179
180
  await sleep(150 * 2 ** attempt)
@@ -222,7 +223,7 @@ export function createBackend(config, deps = {}) {
222
223
  return h
223
224
  }
224
225
 
225
- const gateway = (path, body, task) => {
226
+ const gateway = (path, body, task, options = {}) => {
226
227
  if (!config.controlURL) {
227
228
  return Promise.reject(
228
229
  new BackendError(path, 0, "the capture route needs controlURL (the public gateway origin)", 0, {
@@ -231,7 +232,7 @@ export function createBackend(config, deps = {}) {
231
232
  }),
232
233
  )
233
234
  }
234
- return post(config.controlURL, path, body, gatewayHeaders(task))
235
+ return post(config.controlURL, path, body, gatewayHeaders(task), options)
235
236
  }
236
237
 
237
238
  /**
@@ -423,21 +424,25 @@ export function createBackend(config, deps = {}) {
423
424
  * the caller's personal space; team: the shared space bound to the
424
425
  * session, else `409 capture_not_bound`) and writes the turn to the
425
426
  * cell's `conversation/add`. Nothing sent here can choose a container. The
426
- * task travels as the `x-tdai-task-id` header. Not retried.
427
+ * task travels as the `x-tdai-task-id` header. Not retried here: the spool
428
+ * retries, with the same `key`, which the gateway dedupes
429
+ * (docs/capture-delivery.md). The write has its own timeout,
430
+ * `captureTimeoutMs`, longer than a read's.
427
431
  *
428
- * @param {{ sessionID: string, agent?: string, visibility: "private"|"team", task?: string, payload: object }} input
432
+ * @param {{ sessionID: string, agent?: string, visibility: "private"|"team", task?: string, payload: object, key?: string }} input
429
433
  * @returns {Promise<{ space_id: string, space_kind: string, visibility: string, result: unknown }>}
430
434
  */
431
- captureTurn({ sessionID, agent, visibility, task, payload: raw }) {
435
+ captureTurn({ sessionID, agent, visibility, task, payload: raw, key }) {
432
436
  const body = {
433
437
  session_id: String(sessionID ?? ""),
434
438
  ...(agent ? { agent: String(agent) } : {}),
435
439
  visibility: visibility === "team" ? "team" : "private",
436
440
  turn: fitSessionPayload(raw),
441
+ ...(key ? { idempotency_key: String(key) } : {}),
437
442
  }
438
443
  // A literal path: scripts/verify-edge.mjs derives its probe list from
439
444
  // the `core(`, `knowledge(`, and `gateway(` calls in this file.
440
- return gateway("/v1/capture", body, task)
445
+ return gateway("/v1/capture", body, task, { timeoutMs: config.captureTimeoutMs })
441
446
  },
442
447
 
443
448
  readCore({ scope }) {