@joe-broadhead/opencode-mem 0.9.1 → 0.9.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +337 -0
- package/README.md +33 -9
- package/docs/agent-protocol.md +1 -1
- package/package.json +254 -5
- package/skills/mem.usage/SKILL.md +10 -0
- package/src/backend/records.js +427 -31
- package/src/call-shapes.js +1 -1
- package/src/capture.js +6 -5
- package/src/config.js +14 -1
- package/src/connect.js +353 -0
- package/src/copies.js +28 -3
- package/src/credentials.js +35 -7
- package/src/curation.js +27 -4
- package/src/forgotten.js +204 -0
- package/src/index.js +20 -3
- package/src/runtime.js +5 -1
- package/src/tools/commitments.js +1 -2
- package/src/tools/held-write.js +9 -1
- package/src/tools/index.js +4 -1
- package/src/tools/ingest.js +1 -1
- package/src/tools/memory.js +106 -8
- package/src/tools/recall.js +44 -12
- package/src/tools/records-kit.js +1 -1
- package/src/tools/share.js +14 -9
- package/src/tools/spaces.js +25 -7
- package/src/tools/wiki.js +1 -1
- package/src/util.js +39 -0
- package/src/views/contracts.js +137 -6
- package/src/views/overdue.js +125 -0
- package/src/views/profile.js +155 -0
- package/src/views/records-reads.js +126 -32
- package/src/views/rpc.js +63 -12
- package/src/views/shape.js +49 -0
- package/views/commitments.html +31 -4
- package/views/home.html +22 -3
- package/views/memory.html +126 -24
package/src/backend/records.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks"
|
|
2
|
+
import { createHash, randomBytes } from "node:crypto"
|
|
1
3
|
import { errorMessage, sleep } from "../util.js"
|
|
2
4
|
import { codedMessage, codedParts, failureFor } from "../errors.js"
|
|
3
5
|
import { scopeHeaders } from "../scope.js"
|
|
@@ -23,16 +25,32 @@ import { scopeHeaders } from "../scope.js"
|
|
|
23
25
|
* (`src/errors.js`), and it keeps the service's `code` (`version_conflict`,
|
|
24
26
|
* `refused`, `not_found`, …), the HTTP status, and the error body.
|
|
25
27
|
*
|
|
26
|
-
* Reads are retried on a timeout, a 429 or a 5xx, `config.retries` times
|
|
27
|
-
*
|
|
28
|
+
* Reads are retried on a timeout, a 429 or a 5xx, `config.retries` times.
|
|
29
|
+
*
|
|
30
|
+
* Writes (docs/design/records-idempotency.md): a write whose response is lost
|
|
31
|
+
* (a timeout, a reset, a 408, a 5xx, an unreadable 2xx) has an **unknown**
|
|
32
|
+
* outcome. To a records service that advertises replay for the method
|
|
33
|
+
* (`space.stats` `idempotent_replay`, asked before the first write to a
|
|
34
|
+
* space and kept), the client sends an `idempotency_key` and, after an
|
|
35
|
+
* unknown outcome, resends the same body with the same key,
|
|
36
|
+
* `config.recordsWriteRetries` times, on the cached capability (no fresh ask:
|
|
37
|
+
* a slow service would fail it). The service applies the key once, answers
|
|
38
|
+
* the original result, and marks every keyed answer `keyed: true`; a retry
|
|
39
|
+
* answered without it is reported as possibly applied twice. To an older
|
|
40
|
+
* service it sends no key and never retries a write: an unknown outcome is
|
|
41
|
+
* reported as one, to check before retrying.
|
|
28
42
|
*
|
|
29
43
|
* @param {Record<string, any>} config the resolved configuration
|
|
30
|
-
* @param {{ log?: any, fetchImpl?: typeof fetch, credentials?: { bearer(): Promise<string>, refresh(rejected: string): Promise<string> }, baseURL?: string | (() => string) }} [deps]
|
|
44
|
+
* @param {{ log?: any, fetchImpl?: typeof fetch, credentials?: { bearer(): Promise<string>, refresh(rejected: string): Promise<string> }, baseURL?: string | (() => string), now?: () => number, sleep?: (ms: number) => Promise<void> }} [deps]
|
|
31
45
|
*/
|
|
32
46
|
export function createRecordsClient(config, deps = {}) {
|
|
33
47
|
const log = deps.log
|
|
34
48
|
const fetchImpl = deps.fetchImpl ?? globalThis.fetch
|
|
35
49
|
const credentials = deps.credentials ?? null
|
|
50
|
+
const clock = deps.now ?? Date.now
|
|
51
|
+
const pause = deps.sleep ?? sleep
|
|
52
|
+
/** Per space (or session, when no space is named): the methods the service replays, until when. */
|
|
53
|
+
const capabilities = new Map()
|
|
36
54
|
|
|
37
55
|
/** Read on every call: inside Jelly the gateway origin can appear after setup. */
|
|
38
56
|
function base() {
|
|
@@ -78,7 +96,7 @@ export function createRecordsClient(config, deps = {}) {
|
|
|
78
96
|
/**
|
|
79
97
|
* @param {string} method a §2.4 method, for example `records.put`
|
|
80
98
|
* @param {unknown} body
|
|
81
|
-
* @param {{ sessionID?: string, space?: string, scope?: Record<string, any>, timeoutMs?: number }} [options]
|
|
99
|
+
* @param {{ sessionID?: string, space?: string, scope?: Record<string, any>, timeoutMs?: number, retries?: number }} [options]
|
|
82
100
|
*/
|
|
83
101
|
async function call(method, body = {}, options = {}) {
|
|
84
102
|
const name = String(method ?? "")
|
|
@@ -92,36 +110,72 @@ export function createRecordsClient(config, deps = {}) {
|
|
|
92
110
|
next: "set the Memory address in Memory's settings, or sign in to Jelly",
|
|
93
111
|
})
|
|
94
112
|
}
|
|
113
|
+
const read = READ_METHODS.has(name)
|
|
114
|
+
const encode = (value) => {
|
|
115
|
+
try {
|
|
116
|
+
return JSON.stringify(value ?? {})
|
|
117
|
+
} catch (error) {
|
|
118
|
+
throw new RecordsError(path, 400, { message: `request could not be encoded: ${errorMessage(error)}` })
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
// Encoded before anything is sent: a body that cannot be encoded is a definite failure.
|
|
122
|
+
const request = encode(body)
|
|
123
|
+
const plan = read ? null : await writePlan(name, body, options)
|
|
124
|
+
const exchange = await connection(name, path, options, read)
|
|
125
|
+
if (read) return readLoop(name, path, request, exchange, options)
|
|
126
|
+
return writeLoop(name, path, plan, request, encode, exchange, options)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* One HTTP exchange at a time on one bearer: a 401 asks the credentials for
|
|
131
|
+
* another token once and sends again (the gateway refused the request, so
|
|
132
|
+
* nothing was applied). Transport failures throw as they are.
|
|
133
|
+
*/
|
|
134
|
+
async function connection(name, path, options, read) {
|
|
95
135
|
const headers = { ...requestHeaders(options), "x-mem-tz": timeZone() }
|
|
96
|
-
const timeoutMs = Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : config.timeoutMs
|
|
97
|
-
const retries = READ_METHODS.has(name) ? Math.max(0, Number(config.retries) || 0) : 0
|
|
136
|
+
const timeoutMs = Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : read ? config.timeoutMs : (config.recordsWriteTimeoutMs ?? 10000)
|
|
98
137
|
let token = await bearer(path)
|
|
99
138
|
let renewed = false
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
139
|
+
const exchange = async (request) => {
|
|
140
|
+
for (;;) {
|
|
141
|
+
const controller = new AbortController()
|
|
142
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs)
|
|
143
|
+
try {
|
|
144
|
+
const response = await fetchImpl(`${base()}/v1/${name}`, {
|
|
145
|
+
method: "POST",
|
|
146
|
+
headers: token ? { ...headers, authorization: `Bearer ${token}` } : headers,
|
|
147
|
+
body: request,
|
|
148
|
+
signal: controller.signal,
|
|
149
|
+
})
|
|
150
|
+
if (response.status === 401 && token && !renewed) {
|
|
151
|
+
const next = await refreshed(token)
|
|
152
|
+
if (next) {
|
|
153
|
+
try {
|
|
154
|
+
await response.body?.cancel()
|
|
155
|
+
} catch {
|
|
156
|
+
/* the refused body is not read */
|
|
157
|
+
}
|
|
158
|
+
token = next
|
|
159
|
+
renewed = true
|
|
160
|
+
continue
|
|
118
161
|
}
|
|
119
|
-
token = next
|
|
120
|
-
renewed = true
|
|
121
|
-
attempt -= 1
|
|
122
|
-
continue
|
|
123
162
|
}
|
|
163
|
+
return response
|
|
164
|
+
} finally {
|
|
165
|
+
clearTimeout(timer)
|
|
124
166
|
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
exchange.timeoutMs = timeoutMs
|
|
170
|
+
return exchange
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
async function readLoop(name, path, request, exchange, options = {}) {
|
|
174
|
+
const retries = Number.isInteger(options.retries) ? Math.max(0, options.retries) : Math.max(0, Number(config.retries) || 0)
|
|
175
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
176
|
+
let response
|
|
177
|
+
try {
|
|
178
|
+
response = await exchange(request)
|
|
125
179
|
const answer = await readAnswer(response, path)
|
|
126
180
|
if (response.ok) return answer
|
|
127
181
|
if (attempt < retries && (response.status >= 500 || response.status === 429)) {
|
|
@@ -138,17 +192,355 @@ export function createRecordsClient(config, deps = {}) {
|
|
|
138
192
|
}
|
|
139
193
|
log?.debug?.("records request failed", { method: name, error: errorMessage(error) })
|
|
140
194
|
throw timedOut
|
|
141
|
-
? new RecordsError(path, 408, { message: `timeout after ${timeoutMs}ms` })
|
|
195
|
+
? new RecordsError(path, 408, { message: `timeout after ${exchange.timeoutMs}ms` })
|
|
142
196
|
: new RecordsError(path, 0, { message: errorMessage(error) })
|
|
143
|
-
} finally {
|
|
144
|
-
clearTimeout(timer)
|
|
145
197
|
}
|
|
146
198
|
}
|
|
147
199
|
}
|
|
148
200
|
|
|
201
|
+
/**
|
|
202
|
+
* How a write is sent: with a key the client derives (`keyed`), with the
|
|
203
|
+
* key the caller put in the body (a held write's own ledger), or with none.
|
|
204
|
+
* `retry` is whether an unknown outcome may be retried at all; each retry
|
|
205
|
+
* still asks the service first.
|
|
206
|
+
*/
|
|
207
|
+
async function writePlan(name, body, options) {
|
|
208
|
+
if (!KEYED_METHODS.has(name) || !body || typeof body !== "object" || Array.isArray(body) || body.migrate_legacy === true) return { keyed: false, retry: false }
|
|
209
|
+
const own = typeof body.idempotency_key === "string" || (body.held_batch && typeof body.held_batch === "object")
|
|
210
|
+
if (!(await replays(name, options))) return { keyed: false, retry: false }
|
|
211
|
+
if (own) return { keyed: false, retry: true }
|
|
212
|
+
const space = String(options.space ?? options.scope?.space_id ?? "")
|
|
213
|
+
return { keyed: true, retry: true, key: writeKey(name, space, body) }
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
async function writeLoop(name, path, plan, request, encode, exchange, options) {
|
|
217
|
+
const first = clock()
|
|
218
|
+
const limit = plan.retry ? Math.min(Math.max(0, Number(config.recordsWriteRetries ?? 2) || 0), 5) : 0
|
|
219
|
+
let unknown = null
|
|
220
|
+
let attempts = 0
|
|
221
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
222
|
+
attempts += 1
|
|
223
|
+
const sent = plan.keyed ? encode({ ...decoded(request), idempotency_key: plan.key, idempotency_age_ms: attempt === 0 ? 0 : Math.max(0, clock() - first) }) : request
|
|
224
|
+
const outcome = await attemptWrite(name, path, sent, exchange, attempt)
|
|
225
|
+
if (outcome.answer) {
|
|
226
|
+
if (outcome.answer.replayed === true) log?.debug?.("records write replayed", { method: name, attempt })
|
|
227
|
+
// A keyed write answered without `keyed: true` reached a service that
|
|
228
|
+
// ignored the key (it was rolled back): forget the capability. On a
|
|
229
|
+
// retry, an earlier attempt may have been applied too.
|
|
230
|
+
if (plan.keyed && outcome.answer.keyed !== true) {
|
|
231
|
+
forgetCapability(options)
|
|
232
|
+
if (attempt > 0) throw possiblyTwice(path, outcome.answer, attempts)
|
|
233
|
+
}
|
|
234
|
+
if (outcome.answer.replayed === true && outcome.answer.unavailable === true) throw alreadyApplied(path, outcome.answer)
|
|
235
|
+
return outcome.answer
|
|
236
|
+
}
|
|
237
|
+
if (outcome.definite) throw outcome.definite
|
|
238
|
+
unknown = outcome.unknown
|
|
239
|
+
if (attempt >= limit) break
|
|
240
|
+
await pause(Math.min(250 * 2 ** attempt, 2000))
|
|
241
|
+
// A retry trusts the capability the first attempt was keyed on, with no
|
|
242
|
+
// fresh ask: when the write has just timed out, an ask would time out
|
|
243
|
+
// too, and no retry would be sent. The cache is dropped when any keyed
|
|
244
|
+
// answer lacks `keyed: true`, and then no retry is sent; a retry that
|
|
245
|
+
// reaches a service ignoring its key is reported as possibly applied
|
|
246
|
+
// twice (above).
|
|
247
|
+
if (!replaysCached(name, options)) break
|
|
248
|
+
log?.debug?.("records write outcome unknown; retrying with the same key", { method: name, attempt: attempt + 1 })
|
|
249
|
+
}
|
|
250
|
+
throw unknownWrite(path, unknown.status, unknown.what, attempts > 1)
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* One attempt of a write: `{answer}`, `{definite}` (an error: no attempt was
|
|
255
|
+
* applied), or `{unknown}` (whether this or an earlier attempt was applied
|
|
256
|
+
* is not known). On a retry, only the records service's own refusal is
|
|
257
|
+
* definite: it looks the key up before any other check, and a write commits
|
|
258
|
+
* with its receipt, so no receipt means no attempt was applied.
|
|
259
|
+
*/
|
|
260
|
+
async function attemptWrite(name, path, request, exchange, attempt) {
|
|
261
|
+
let response
|
|
262
|
+
try {
|
|
263
|
+
response = await exchange(request)
|
|
264
|
+
} catch (error) {
|
|
265
|
+
// Anything thrown once the request may have left is an unknown outcome.
|
|
266
|
+
const timedOut = error?.name === "AbortError"
|
|
267
|
+
log?.debug?.("records write failed", { method: name, attempt, error: errorMessage(error) })
|
|
268
|
+
return { unknown: { status: timedOut ? 408 : 0, what: timedOut ? `timeout after ${exchange.timeoutMs}ms` : errorMessage(error) } }
|
|
269
|
+
}
|
|
270
|
+
if (response.status === 408 || response.status >= 500) {
|
|
271
|
+
try {
|
|
272
|
+
await response.body?.cancel()
|
|
273
|
+
} catch {
|
|
274
|
+
/* not read */
|
|
275
|
+
}
|
|
276
|
+
return { unknown: { status: response.status, what: `server returned ${response.status}` } }
|
|
277
|
+
}
|
|
278
|
+
let answer
|
|
279
|
+
try {
|
|
280
|
+
answer = await readAnswer(response, path)
|
|
281
|
+
} catch {
|
|
282
|
+
return response.ok
|
|
283
|
+
? { unknown: { status: response.status, what: "the service returned an unreadable success response" } }
|
|
284
|
+
: { unknown: { status: response.status, what: `the service returned an unreadable ${response.status}` } }
|
|
285
|
+
}
|
|
286
|
+
if (response.ok) {
|
|
287
|
+
if (!validWriteAnswer(name, answer)) return { unknown: { status: response.status, what: "the service returned an invalid success response" } }
|
|
288
|
+
return { answer }
|
|
289
|
+
}
|
|
290
|
+
const error = RecordsError.fromAnswer(path, response.status, answer)
|
|
291
|
+
if (attempt === 0) return { definite: error }
|
|
292
|
+
const fromService = answer?.error && typeof answer.error === "object" && typeof answer.error.code === "string"
|
|
293
|
+
if (fromService && response.status !== 401 && !UNCHECKABLE.has(answer.error.code)) return { definite: error }
|
|
294
|
+
return { unknown: { status: response.status, what: error.what.replace(/^[^:]+: /, "") } }
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* Whether the service replays `method` for the space these options name,
|
|
299
|
+
* from `space.stats`. A service that replays is remembered for the life of
|
|
300
|
+
* the process, until a keyed answer comes back without `keyed: true` (a
|
|
301
|
+
* mismatch: the service was rolled back); one that does not, or refused the
|
|
302
|
+
* ask, is asked again after 10 minutes, and an ask that did not reach it
|
|
303
|
+
* after 30 seconds. The ask has a short timeout and no retry, so it never
|
|
304
|
+
* holds the first write up for long; retries never ask (`replaysCached`).
|
|
305
|
+
* The ask names the same space and session as the write, so it reaches the
|
|
306
|
+
* same cell and changes nothing the write would not.
|
|
307
|
+
*/
|
|
308
|
+
async function replays(method, options) {
|
|
309
|
+
const where = capabilityKey(options)
|
|
310
|
+
const now = clock()
|
|
311
|
+
const cached = capabilities.get(where)
|
|
312
|
+
if (cached && now < cached.until) return cached.methods.has(method)
|
|
313
|
+
let methods = new Set()
|
|
314
|
+
let until = now + CAPABILITY_RETRY_MS
|
|
315
|
+
try {
|
|
316
|
+
const stats = await call("space.stats", {}, { scope: options.scope, space: options.space, sessionID: options.sessionID, timeoutMs: CAPABILITY_TIMEOUT_MS, retries: 0 })
|
|
317
|
+
const replay = stats?.idempotent_replay
|
|
318
|
+
if (replay && replay.version === 1 && Array.isArray(replay.methods)) methods = new Set(replay.methods.filter((value) => typeof value === "string"))
|
|
319
|
+
if (methods.size > 0) until = Number.POSITIVE_INFINITY
|
|
320
|
+
} catch (error) {
|
|
321
|
+
// A refusal (a grant-only caller may not read space.stats) is an answer;
|
|
322
|
+
// a failure to reach the service is asked again sooner.
|
|
323
|
+
const status = Number(error?.status) || 0
|
|
324
|
+
if (!(status >= 400 && status < 500 && status !== 408 && status !== 429)) until = now + CAPABILITY_FAILURE_TTL_MS
|
|
325
|
+
log?.debug?.("records capability check failed; writes go without a key", { error: errorMessage(error) })
|
|
326
|
+
}
|
|
327
|
+
capabilities.delete(where)
|
|
328
|
+
capabilities.set(where, { methods, until })
|
|
329
|
+
if (capabilities.size > CAPABILITY_MAX) capabilities.delete(capabilities.keys().next().value)
|
|
330
|
+
return methods.has(method)
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/** Whether the cached capability (no ask) still lists `method`: what a retry relies on. */
|
|
334
|
+
function replaysCached(method, options) {
|
|
335
|
+
const cached = capabilities.get(capabilityKey(options))
|
|
336
|
+
return Boolean(cached && clock() < cached.until && cached.methods.has(method))
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
function capabilityKey(options) {
|
|
340
|
+
return options.space ? `space:${options.space}` : `session:${options.sessionID ?? ""}`
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
function forgetCapability(options) {
|
|
344
|
+
capabilities.delete(capabilityKey(options))
|
|
345
|
+
log?.debug?.("records service ignored an idempotency key; asking again before the next write")
|
|
346
|
+
}
|
|
347
|
+
|
|
149
348
|
return { call, base }
|
|
150
349
|
}
|
|
151
350
|
|
|
351
|
+
/** The body a request was encoded from: every attempt sends the same body, encoded once. */
|
|
352
|
+
function decoded(request) {
|
|
353
|
+
return JSON.parse(request)
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/** Codes from the service that do not tell whether an earlier attempt was applied. */
|
|
357
|
+
const UNCHECKABLE = new Set(["idempotency_expired", "idempotency_conflict"])
|
|
358
|
+
/** How soon a service that does not replay (or refused the ask) is asked again. */
|
|
359
|
+
const CAPABILITY_RETRY_MS = 10 * 60 * 1000
|
|
360
|
+
const CAPABILITY_FAILURE_TTL_MS = 30 * 1000
|
|
361
|
+
const CAPABILITY_TIMEOUT_MS = 1000
|
|
362
|
+
const CAPABILITY_MAX = 500
|
|
363
|
+
|
|
364
|
+
/** The methods a records service may replay (docs/design/records-idempotency.md). */
|
|
365
|
+
export const KEYED_METHODS = Object.freeze(new Set(["records.put", "records.revise", "records.append", "records.move", "records.delete"]))
|
|
366
|
+
|
|
367
|
+
function unknownWrite(path, status, what, retried = false) {
|
|
368
|
+
const error = new RecordsError(path, status, {
|
|
369
|
+
message: `${what}; the write outcome is unknown${retried ? " (it was retried with the same idempotency key)" : ""}`,
|
|
370
|
+
next: "check before retrying: list or read the current records",
|
|
371
|
+
})
|
|
372
|
+
error.outcomeUnknown = true
|
|
373
|
+
return error
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* A retry answered without `keyed: true`: the service ignored the key (it was
|
|
378
|
+
* rolled back to one without replay), so it applied this attempt as a new
|
|
379
|
+
* write, and an earlier attempt whose answer was lost may have been applied
|
|
380
|
+
* too. Neither a success nor an unknown outcome to retry: check first.
|
|
381
|
+
*/
|
|
382
|
+
function possiblyTwice(path, answer, attempts) {
|
|
383
|
+
const id = typeof answer?.record?.id === "string" ? ` (${answer.record.id})` : ""
|
|
384
|
+
const error = new RecordsError(path, 200, {
|
|
385
|
+
code: "possibly_applied_twice",
|
|
386
|
+
errorCode: "MEMORY_POSSIBLY_DUPLICATED",
|
|
387
|
+
message: `possibly applied twice: the write was sent ${attempts} times and the records service ignored its idempotency key${id}, so an earlier attempt may have been applied as well`,
|
|
388
|
+
next: "check before writing again: list or read the current records, and remove a duplicate if there is one",
|
|
389
|
+
})
|
|
390
|
+
error.outcomeUnknown = true
|
|
391
|
+
error.possiblyDuplicated = true
|
|
392
|
+
return error
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/** A replay whose record has since been deleted or hidden: the write was applied, once. */
|
|
396
|
+
function alreadyApplied(path, answer) {
|
|
397
|
+
const id = typeof answer.record_id === "string" ? answer.record_id : "its record"
|
|
398
|
+
const error = new RecordsError(path, 200, {
|
|
399
|
+
code: "already_applied",
|
|
400
|
+
errorCode: "MEMORY_ALREADY_WRITTEN",
|
|
401
|
+
message: `the write was already applied (${id}); that record has since been deleted or is no longer readable, so it was not applied again`,
|
|
402
|
+
next: "read the current records before writing again",
|
|
403
|
+
})
|
|
404
|
+
error.committed = true
|
|
405
|
+
return error
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// ---------------------------------------------------------------------------
|
|
409
|
+
// Keys (docs/design/records-idempotency.md#keys)
|
|
410
|
+
|
|
411
|
+
const writeCalls = new AsyncLocalStorage()
|
|
412
|
+
|
|
413
|
+
/**
|
|
414
|
+
* How many times each (call id, tool, arguments) has run in each message of
|
|
415
|
+
* this process, so the n-th identical call in a message gets the n-th root.
|
|
416
|
+
* Counted per message: unrelated calls, in this message or in others, never
|
|
417
|
+
* push a message's own counts out. Bounded by messages, the least recently
|
|
418
|
+
* used first; a message keeps its counts until it is pushed out, however many
|
|
419
|
+
* calls it makes.
|
|
420
|
+
*/
|
|
421
|
+
const occurrences = new Map()
|
|
422
|
+
const OCCURRENCE_MESSAGES_MAX = 10000
|
|
423
|
+
|
|
424
|
+
/**
|
|
425
|
+
* Run one tool call as one logical call, and derive the root its writes'
|
|
426
|
+
* keys come from.
|
|
427
|
+
*
|
|
428
|
+
* OpenCode 2.2.6 passes `sessionID`, `messageID` and `id` in a tool's
|
|
429
|
+
* context, but under Code Mode `id` is the outer `execute` call's id, shared
|
|
430
|
+
* by every tool call in that script. So the root is the session, the message,
|
|
431
|
+
* that id when present, the tool, a canonical digest of the arguments, and the
|
|
432
|
+
* occurrence index of that same (tool, arguments) within the message, counted
|
|
433
|
+
* in memory. A host that sends a call again after a restart starts counting at
|
|
434
|
+
* zero and derives the original keys; two intentional identical calls in one
|
|
435
|
+
* message get indexes 0 and 1 and differ. (Within one process a re-delivery of
|
|
436
|
+
* the same call is indistinguishable from an intentional repeat, and is
|
|
437
|
+
* treated as one.) Without a session and a message the root is random.
|
|
438
|
+
*
|
|
439
|
+
* @template T
|
|
440
|
+
* @param {{ sessionID?: string, messageID?: string, id?: string, callID?: string } | undefined} context
|
|
441
|
+
* @param {string} tool
|
|
442
|
+
* @param {unknown} input the tool's arguments
|
|
443
|
+
* @param {() => T} run
|
|
444
|
+
* @returns {T}
|
|
445
|
+
*/
|
|
446
|
+
export function withWriteCall(context, tool, input, run) {
|
|
447
|
+
return writeCalls.run({ root: callRoot(context, tool, input), counts: new Map() }, run)
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/** The root of the current tool call's keys, or "" outside one (held writes key on it too). */
|
|
451
|
+
export function writeCallRoot() {
|
|
452
|
+
return writeCalls.getStore()?.root ?? ""
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
function callRoot(context, tool, input) {
|
|
456
|
+
const session = context?.sessionID
|
|
457
|
+
const message = context?.messageID
|
|
458
|
+
if (typeof session !== "string" || !session || typeof message !== "string" || !message) return randomBytes(32).toString("hex")
|
|
459
|
+
const callId = context?.id ?? context?.callID
|
|
460
|
+
const call = [typeof callId === "string" ? callId : "", String(tool ?? ""), sha256(canonicalJson(input ?? {}))]
|
|
461
|
+
const where = JSON.stringify([session, message])
|
|
462
|
+
const counts = occurrences.get(where) ?? new Map()
|
|
463
|
+
occurrences.delete(where)
|
|
464
|
+
occurrences.set(where, counts)
|
|
465
|
+
if (occurrences.size > OCCURRENCE_MESSAGES_MAX) occurrences.delete(occurrences.keys().next().value)
|
|
466
|
+
const occurrence = counts.get(JSON.stringify(call)) ?? 0
|
|
467
|
+
counts.set(JSON.stringify(call), occurrence + 1)
|
|
468
|
+
// The root 0.9.3 derived: the same calls give the same keys across the upgrade.
|
|
469
|
+
const group = JSON.stringify([session, message, ...call])
|
|
470
|
+
return sha256(JSON.stringify(["opencode-mem/write/v2", group, occurrence]))
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
/** Forget the occurrence counts, as a restart of the plugin does (tests). */
|
|
474
|
+
export function forgetWriteCalls() {
|
|
475
|
+
occurrences.clear()
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
/**
|
|
479
|
+
* A write's key: `wk1_` and 43 base64url characters of
|
|
480
|
+
* sha256([root, space, method, intent, ordinal]). The ordinal counts earlier
|
|
481
|
+
* writes of the same intent in the call, so two identical writes differ.
|
|
482
|
+
*/
|
|
483
|
+
export function writeKey(method, space, payload) {
|
|
484
|
+
const call = writeCalls.getStore()
|
|
485
|
+
const intent = writeIntent(payload)
|
|
486
|
+
if (!call) return `wk1_${digest64(JSON.stringify([randomBytes(32).toString("hex"), space, method, intent]))}`
|
|
487
|
+
const group = JSON.stringify([space, method, intent])
|
|
488
|
+
const ordinal = call.counts.get(group) ?? 0
|
|
489
|
+
call.counts.set(group, ordinal + 1)
|
|
490
|
+
return `wk1_${digest64(JSON.stringify([call.root, space, method, intent, ordinal]))}`
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/** A write's intent: its body without a key, an age or base_version, keys sorted (as the service's digest). */
|
|
494
|
+
export function writeIntent(payload) {
|
|
495
|
+
const { idempotency_key: _key, idempotency_age_ms: _age, base_version: _version, ...rest } = payload ?? {}
|
|
496
|
+
if (rest.record && typeof rest.record === "object" && !Array.isArray(rest.record)) {
|
|
497
|
+
const { base_version: _recordVersion, ...record } = rest.record
|
|
498
|
+
rest.record = record
|
|
499
|
+
}
|
|
500
|
+
return sha256(canonicalJson(rest))
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
function canonicalJson(value) {
|
|
504
|
+
if (Array.isArray(value)) return `[${value.map((item) => canonicalJson(item === undefined ? null : item)).join(",")}]`
|
|
505
|
+
if (value && typeof value === "object") {
|
|
506
|
+
const keys = Object.keys(value).filter((key) => value[key] !== undefined).sort()
|
|
507
|
+
return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`
|
|
508
|
+
}
|
|
509
|
+
return JSON.stringify(value) ?? "null"
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
function sha256(text) {
|
|
513
|
+
return createHash("sha256").update(text).digest("hex")
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
function digest64(text) {
|
|
517
|
+
return createHash("sha256").update(text).digest("base64url")
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
/** A successful mutation must contain the service's confirmation, not an empty or unrelated body. */
|
|
521
|
+
function validWriteAnswer(method, answer) {
|
|
522
|
+
if (!answer || typeof answer !== "object" || Array.isArray(answer) || answer.binary || Object.keys(answer).length === 0) return false
|
|
523
|
+
const record = (value) => Boolean(
|
|
524
|
+
value && typeof value === "object" && /^rec_[0-9A-HJKMNP-TV-Z]{26}$/.test(value.id) &&
|
|
525
|
+
typeof value.kind === "string" && typeof value.type === "string" && typeof value.title === "string" &&
|
|
526
|
+
typeof value.body === "string" && typeof value.status === "string" && value.fields &&
|
|
527
|
+
typeof value.fields === "object" && !Array.isArray(value.fields) && Number.isInteger(value.version) && value.version > 0,
|
|
528
|
+
)
|
|
529
|
+
// A replay whose record has since gone or become unreadable: applied once, nothing to show.
|
|
530
|
+
const unavailable = answer.replayed === true && answer.unavailable === true && answer.record === null
|
|
531
|
+
if (method === "records.put") return Boolean(record(answer.record) || unavailable || (typeof answer.receipt === "string" && answer.receipt.length > 0) || (answer.legacy_migration === true && answer.deleted === true))
|
|
532
|
+
if (["records.revise", "records.append", "records.move"].includes(method)) return record(answer.record) || unavailable
|
|
533
|
+
if (method === "records.delete") return Array.isArray(answer.deleted) && answer.deleted.every((id) => typeof id === "string")
|
|
534
|
+
if (method === "review.decide") return record(answer.proposal)
|
|
535
|
+
const count = (value) => Number.isInteger(value) && value >= 0
|
|
536
|
+
if (method === "events.ingest") return count(answer.accepted) && count(answer.duplicates) && count(answer.refused) && typeof answer.cursor_saved === "boolean"
|
|
537
|
+
if (method === "sources.bind") return Boolean(answer.source && typeof answer.source === "object" && typeof answer.source.source === "string")
|
|
538
|
+
if (method === "sources.unbind") return count(answer.deleted_events)
|
|
539
|
+
if (method === "forgotten.put") return typeof answer.fingerprint?.id === "string" && /^fp_/.test(answer.fingerprint.id)
|
|
540
|
+
if (method === "forgotten.lift") return Array.isArray(answer.lifted) && answer.lifted.every((id) => typeof id === "string")
|
|
541
|
+
return true
|
|
542
|
+
}
|
|
543
|
+
|
|
152
544
|
/** `records.put`, `events.ingest`, …: a service and a verb, both lower camel case. */
|
|
153
545
|
const METHOD = /^[a-z]+\.[a-z][A-Za-z]*$/
|
|
154
546
|
|
|
@@ -168,6 +560,7 @@ export const READ_METHODS = Object.freeze(
|
|
|
168
560
|
"review.list",
|
|
169
561
|
"routines.status",
|
|
170
562
|
"space.stats",
|
|
563
|
+
"forgotten.list",
|
|
171
564
|
]),
|
|
172
565
|
)
|
|
173
566
|
|
|
@@ -219,6 +612,9 @@ const CODES = Object.freeze({
|
|
|
219
612
|
forbidden: "MEMORY_FORBIDDEN",
|
|
220
613
|
records_unavailable: "MEMORY_RECORDS_UNAVAILABLE",
|
|
221
614
|
source_unbound: "MEMORY_SOURCE_UNBOUND",
|
|
615
|
+
// The records service refused a keyed write before applying it: its ledger
|
|
616
|
+
// of recent writes is full. A definite "not saved", never an unknown outcome.
|
|
617
|
+
idempotency_capacity: "MEMORY_NOT_SAVED",
|
|
222
618
|
})
|
|
223
619
|
|
|
224
620
|
/** What a cell without the records service (503 `records_unavailable`) means, in words a person can act on. */
|
package/src/call-shapes.js
CHANGED
|
@@ -53,7 +53,7 @@ export const MEM_NAMESPACE_DESCRIPTION = [
|
|
|
53
53
|
'page({action: "list"|"read"|"write"|"append"|"move"|"delete"|"links"|"history"|"export", path?, prefix?, type?, title?, content?, section?, to?, depth?, confirm?, space?}) — journal/YYYY-MM-DD, people/<name>, projects/<slug>; read known paths directly, list only unknown paths; append once per change; write needs path and content, append needs path, section and content; delete previews unless confirm is true;',
|
|
54
54
|
'commitments({action: "list"|"add"|"update"|"delete", what?, owner?, due?, id?, state?}) — add needs what (due is YYYY-MM-DD), update and delete need id;',
|
|
55
55
|
'share({action: "list"|"grant"|"revoke"|"transfer"|"copy", path?, ids?, to?, role?, expires?, space?, rationale?, grant_id?}) — only when the person asks; grant needs path and to, copy needs ids and rationale, revoke needs grant_id, ids, or path alone; every change asks;',
|
|
56
|
-
'spaces({action: "current"|"list"|"members"|"access"|"health"|"sources"|"review", space?, bind?, unbind?, confirm?, decide?, limit?}) — health when recall finds nothing; sources binds an account only when the person asks; review lists and decides proposals;',
|
|
56
|
+
'spaces({action: "current"|"list"|"members"|"access"|"health"|"profile"|"sources"|"review", space?, bind?, unbind?, confirm?, decide?, limit?}) — health when recall finds nothing; sources binds an account only when the person asks; review lists and decides proposals;',
|
|
57
57
|
'collect({source?, wait?}) — only for routines and the person\'s explicit "collect now"; it never asks and answers at once (started, running or skipped) unless wait is true;',
|
|
58
58
|
'wiki({action: "list"|"search"|"read"|"status"|"create"|"write"|"append"|"delete", query?, wiki_id?, ref?, filename?, name?, content?, ingest?, confirm?}) — write and append need filename, a flat name such as "notes.md"; documents only, not authored pages; delete previews unless confirm is true.',
|
|
59
59
|
"Deprecated until 0.10: forget (use revise), promote (use share copy), status (use spaces health), grants and members (use spaces). ingest and cursor belong to the collector: never call them.",
|
package/src/capture.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { CAPTURE_MESSAGE_CHARS, capMessageCount, clampContent } from "./limits.js"
|
|
2
|
-
import { nowIso, redactSecrets } from "./util.js"
|
|
2
|
+
import { nowIso, redactLocalPaths, redactSecrets } from "./util.js"
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* @param {{ sessionID: string, messageID?: string, promptText: string, agent?: string, model?: string }} input
|
|
@@ -24,7 +24,7 @@ export function newTurn(input) {
|
|
|
24
24
|
export function flattenMessage(message) {
|
|
25
25
|
const info = message?.info ?? message ?? {}
|
|
26
26
|
const parts = message?.parts ?? info?.parts ?? message?.content ?? []
|
|
27
|
-
const role = info.role ?? message?.role ?? "unknown"
|
|
27
|
+
const role = info.role ?? message?.role ?? info.type ?? message?.type ?? "unknown"
|
|
28
28
|
const texts = []
|
|
29
29
|
const tools = []
|
|
30
30
|
if (Array.isArray(parts)) {
|
|
@@ -36,6 +36,7 @@ export function flattenMessage(message) {
|
|
|
36
36
|
} else if (typeof parts === "string") {
|
|
37
37
|
texts.push(parts)
|
|
38
38
|
}
|
|
39
|
+
if (texts.length === 0 && info.type === "user" && typeof info.text === "string") texts.push(info.text)
|
|
39
40
|
return {
|
|
40
41
|
id: info.id ?? message?.id,
|
|
41
42
|
role,
|
|
@@ -112,12 +113,13 @@ export function buildSessionPayload({ turn, messages, now = new Date() }) {
|
|
|
112
113
|
|
|
113
114
|
/**
|
|
114
115
|
* Every captured text is stored in L0 and read by extraction. Redact
|
|
115
|
-
* credential-shaped values and
|
|
116
|
+
* credential-shaped values and absolute local paths (a path keeps its file
|
|
117
|
+
* name: `<path>/app.ts`), and cap the length under the upstream per-message
|
|
116
118
|
* limit, with a visible marker, before the text leaves this process.
|
|
117
119
|
*/
|
|
118
120
|
function storable(text) {
|
|
119
121
|
if (!text) return ""
|
|
120
|
-
return clampContent(redactSecrets(text), CAPTURE_MESSAGE_CHARS)
|
|
122
|
+
return clampContent(redactLocalPaths(redactSecrets(text)), CAPTURE_MESSAGE_CHARS)
|
|
121
123
|
}
|
|
122
124
|
|
|
123
125
|
/**
|
|
@@ -140,4 +142,3 @@ export function extractCompactionSummary(messages) {
|
|
|
140
142
|
}
|
|
141
143
|
return null
|
|
142
144
|
}
|
|
143
|
-
|
package/src/config.js
CHANGED
|
@@ -61,6 +61,13 @@ const DEFAULTS = Object.freeze({
|
|
|
61
61
|
recallLimit: 8,
|
|
62
62
|
recallChars: 6000,
|
|
63
63
|
timeoutMs: 2000,
|
|
64
|
+
recordsWriteTimeoutMs: 10000,
|
|
65
|
+
/**
|
|
66
|
+
* Retries of one keyed records write after an unknown outcome, 0 to 5
|
|
67
|
+
* (docs/design/records-idempotency.md). Sent only to a records service that
|
|
68
|
+
* advertises replay; 0 sends the key but never retries.
|
|
69
|
+
*/
|
|
70
|
+
recordsWriteRetries: 2,
|
|
64
71
|
retries: 1,
|
|
65
72
|
|
|
66
73
|
/**
|
|
@@ -93,6 +100,8 @@ const ENV_MAP = {
|
|
|
93
100
|
stateDir: "MEMORY_STATE_DIR",
|
|
94
101
|
timeZone: "MEMORY_TIME_ZONE",
|
|
95
102
|
collect: "MEMORY_COLLECT",
|
|
103
|
+
recordsWriteTimeoutMs: "MEMORY_RECORDS_WRITE_TIMEOUT_MS",
|
|
104
|
+
recordsWriteRetries: "MEMORY_RECORDS_WRITE_RETRIES",
|
|
96
105
|
}
|
|
97
106
|
|
|
98
107
|
const BOOLEAN_KEYS = [
|
|
@@ -110,7 +119,7 @@ const BOOLEAN_KEYS = [
|
|
|
110
119
|
"debug",
|
|
111
120
|
]
|
|
112
121
|
|
|
113
|
-
const NUMBER_KEYS = ["bootstrapChars", "recallLimit", "recallChars", "timeoutMs", "retries", "collectEveryMinutes"]
|
|
122
|
+
const NUMBER_KEYS = ["bootstrapChars", "recallLimit", "recallChars", "timeoutMs", "recordsWriteTimeoutMs", "retries", "collectEveryMinutes"]
|
|
114
123
|
|
|
115
124
|
/** An IANA zone this runtime knows, or "". */
|
|
116
125
|
function validTimeZone(value) {
|
|
@@ -177,6 +186,10 @@ export function resolveConfig(options = {}, env = process.env) {
|
|
|
177
186
|
merged[key] = toPositiveInt(merged[key], DEFAULTS[key])
|
|
178
187
|
}
|
|
179
188
|
merged.retries = Math.min(merged.retries, 5)
|
|
189
|
+
{
|
|
190
|
+
const value = Number(merged.recordsWriteRetries)
|
|
191
|
+
merged.recordsWriteRetries = Number.isInteger(value) && value >= 0 ? Math.min(value, 5) : DEFAULTS.recordsWriteRetries
|
|
192
|
+
}
|
|
180
193
|
merged.collectEveryMinutes = Math.min(Math.max(merged.collectEveryMinutes, 5), 1440)
|
|
181
194
|
|
|
182
195
|
for (const key of ["baseURL", "controlURL", "knowledgeURL"]) {
|