@joe-broadhead/opencode-mem 0.0.0-stage → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +323 -2
- package/SECURITY.md +131 -0
- package/docs/agent-protocol.md +190 -0
- package/index.js +4 -0
- package/package.json +174 -4
- package/skills/mem-usage/SKILL.md +427 -0
- package/src/backend/v3.js +562 -0
- package/src/bootstrap.js +85 -0
- package/src/capture.js +143 -0
- package/src/commands.js +151 -0
- package/src/compaction.js +402 -0
- package/src/config.js +193 -0
- package/src/copies.js +322 -0
- package/src/credentials.js +294 -0
- package/src/curation.js +240 -0
- package/src/erase.js +36 -0
- package/src/events.js +394 -0
- package/src/hooks.js +182 -0
- package/src/index.js +189 -0
- package/src/limits.js +90 -0
- package/src/log.js +161 -0
- package/src/protocol.js +48 -0
- package/src/recall.js +368 -0
- package/src/runtime.js +868 -0
- package/src/scope.js +127 -0
- package/src/sessions.js +62 -0
- package/src/skill-contexts.js +168 -0
- package/src/skills.js +252 -0
- package/src/spaces.js +478 -0
- package/src/spool.js +520 -0
- package/src/tools/index.js +51 -0
- package/src/tools/knowledge.js +90 -0
- package/src/tools/memory.js +1239 -0
- package/src/tools/recall.js +273 -0
- package/src/tools/shared.js +339 -0
- package/src/tools/spaces.js +179 -0
- package/src/tools/status.js +64 -0
- package/src/tools/wiki.js +412 -0
- package/src/util.js +176 -0
package/src/spool.js
ADDED
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
import fs from "node:fs"
|
|
2
|
+
import path from "node:path"
|
|
3
|
+
import { errorMessage, oneLine, redactSecrets, shortHash } from "./util.js"
|
|
4
|
+
|
|
5
|
+
/** At most this many sends per flush; entries that are not due do not count. */
|
|
6
|
+
const MAX_SENDS_PER_FLUSH = 25
|
|
7
|
+
/** A transient failure is retried with backoff until either cap is reached. */
|
|
8
|
+
const MAX_ATTEMPTS = 50
|
|
9
|
+
const MAX_AGE_MS = 7 * 86_400_000
|
|
10
|
+
const BACKOFF_BASE_MS = 30_000
|
|
11
|
+
const BACKOFF_MAX_MS = 60 * 60_000
|
|
12
|
+
/** A claim older than this belongs to a process that died mid-send. */
|
|
13
|
+
const CLAIM_STALE_MS = 5 * 60_000
|
|
14
|
+
/** Stop a flush after this many consecutive "backend unreachable" failures. */
|
|
15
|
+
const UNREACHABLE_STOP = 3
|
|
16
|
+
|
|
17
|
+
const PENDING = ".json"
|
|
18
|
+
const INFLIGHT = ".inflight"
|
|
19
|
+
const FAILED = ".failed"
|
|
20
|
+
const CLAIM_SUFFIX = /\.(\d+)-(\d+)\.inflight$/
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Classify a failed send.
|
|
24
|
+
*
|
|
25
|
+
* - `permanent`: the server rejected this entry (a 4xx such as 400, 403, 409,
|
|
26
|
+
* 413, or 422). Retrying the same bytes cannot succeed, so the entry is
|
|
27
|
+
* quarantined with the reason.
|
|
28
|
+
* - `unreachable`: no answer from the backend (transport error, timeout, 502,
|
|
29
|
+
* 503, 504). The rest of the queue would fail the same way.
|
|
30
|
+
* - `transient`: anything else, including 401 (a credential problem, fixed by
|
|
31
|
+
* the operator, not by the entry), 404 (a route the edge does not forward),
|
|
32
|
+
* 408, 425, 429, other 5xx, and every control-plane failure: placement is
|
|
33
|
+
* resolved again on the next attempt.
|
|
34
|
+
*
|
|
35
|
+
* @param {any} error
|
|
36
|
+
* @returns {"permanent" | "unreachable" | "transient"}
|
|
37
|
+
*/
|
|
38
|
+
export function classifyFailure(error) {
|
|
39
|
+
const status = Number(error?.status)
|
|
40
|
+
if (!Number.isFinite(status) || status === 0 || status === 408) return "unreachable"
|
|
41
|
+
if (status === 502 || status === 503 || status === 504) return "unreachable"
|
|
42
|
+
if (error?.name === "ControlError") return "transient"
|
|
43
|
+
if (status >= 400 && status < 500 && ![401, 404, 425, 429].includes(status)) return "permanent"
|
|
44
|
+
return "transient"
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function backoffMs(attempts) {
|
|
48
|
+
return Math.min(BACKOFF_MAX_MS, BACKOFF_BASE_MS * 2 ** Math.max(0, attempts - 1))
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Durable local queue for captures that could not reach the backend, plus the
|
|
53
|
+
* cross-process claims that keep duplicate event streams from capturing a turn
|
|
54
|
+
* twice.
|
|
55
|
+
*
|
|
56
|
+
* The spool holds memory text, so the directory is private to the user (0700)
|
|
57
|
+
* and each entry is readable only by them (0600).
|
|
58
|
+
*
|
|
59
|
+
* Entry life cycle:
|
|
60
|
+
*
|
|
61
|
+
* - `<name>.json` is pending.
|
|
62
|
+
* - `<name>.json.<ms>-<pid>.inflight` is claimed by one sender. The claim is an
|
|
63
|
+
* atomic rename, so two processes, or a flush racing a live capture, never
|
|
64
|
+
* send the same entry twice. A claim older than five minutes is returned to
|
|
65
|
+
* pending.
|
|
66
|
+
* - `<name>.json.failed` is quarantined. It holds the entry plus
|
|
67
|
+
* `quarantine: { reason, at }`. Rename it back to `.json` to retry it; the
|
|
68
|
+
* attempt count restarts.
|
|
69
|
+
*
|
|
70
|
+
* In control mode a turn entry records the session, agent, task, and the
|
|
71
|
+
* visibility it was captured with, never a container. Its replay goes through
|
|
72
|
+
* the gateway's capture route with that visibility (`captureTurn`), so the
|
|
73
|
+
* gateway places it again, and a private turn can never land in a shared space
|
|
74
|
+
* that was bound while it waited. A container stored by an older plugin
|
|
75
|
+
* version is ignored on replay. In local direct mode the entry carries the
|
|
76
|
+
* fixed local scope and is written to `conversation/add`.
|
|
77
|
+
*
|
|
78
|
+
* @param {{ config: any, log: any, backend: any, captureTurn?: Function | null, now?: () => number }} input
|
|
79
|
+
*/
|
|
80
|
+
export function createSpool(input) {
|
|
81
|
+
const { config, log, backend } = input
|
|
82
|
+
const captureTurn = typeof input.captureTurn === "function" ? input.captureTurn : null
|
|
83
|
+
const now = typeof input.now === "function" ? input.now : Date.now
|
|
84
|
+
const spoolDir = path.join(config.stateDir, "spool")
|
|
85
|
+
const claimsDir = path.join(config.stateDir, "claims")
|
|
86
|
+
let sequence = 0
|
|
87
|
+
|
|
88
|
+
function ensureDir() {
|
|
89
|
+
fs.mkdirSync(spoolDir, { recursive: true, mode: 0o700 })
|
|
90
|
+
try {
|
|
91
|
+
fs.chmodSync(spoolDir, 0o700)
|
|
92
|
+
} catch {
|
|
93
|
+
/* best effort on platforms without POSIX modes */
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function writePrivate(file, record, flag = "w") {
|
|
98
|
+
fs.writeFileSync(file, JSON.stringify(record, null, 2), { mode: 0o600, flag })
|
|
99
|
+
try {
|
|
100
|
+
fs.chmodSync(file, 0o600)
|
|
101
|
+
} catch {
|
|
102
|
+
/* best effort on platforms without POSIX modes */
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Names sort in enqueue order, which is what "oldest first" relies on: the
|
|
108
|
+
* timestamp is fixed-width for centuries and the per-process sequence is
|
|
109
|
+
* zero-padded, so `...-10-` cannot sort before `...-9-` within a millisecond.
|
|
110
|
+
*/
|
|
111
|
+
function newName(data) {
|
|
112
|
+
sequence += 1
|
|
113
|
+
const stamp = String(now()).padStart(13, "0")
|
|
114
|
+
return `${stamp}-${process.pid}-${String(sequence).padStart(9, "0")}-${shortHash(JSON.stringify(data), 8)}${PENDING}`
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function claimName(file) {
|
|
118
|
+
return `${file}.${now()}-${process.pid}${INFLIGHT}`
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** The pending path a claimed or quarantined file belongs to. */
|
|
122
|
+
function pendingOf(file) {
|
|
123
|
+
return file.replace(CLAIM_SUFFIX, "").replace(/\.failed$/, "")
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Persist one capture locally as pending. Returns the file path on success
|
|
128
|
+
* and null when spooling is disabled or the write fails; callers must not
|
|
129
|
+
* claim a capture is queued otherwise.
|
|
130
|
+
*/
|
|
131
|
+
function writeSpool(kind, data, scope, meta = {}) {
|
|
132
|
+
if (!config.spool) return null
|
|
133
|
+
try {
|
|
134
|
+
ensureDir()
|
|
135
|
+
const file = path.join(spoolDir, newName(data))
|
|
136
|
+
writePrivate(file, { kind, data, scope, ...meta, queuedAt: new Date(now()).toISOString() }, "wx")
|
|
137
|
+
return file
|
|
138
|
+
} catch (error) {
|
|
139
|
+
log.error("spool write failed", errorMessage(error))
|
|
140
|
+
return null
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Persist a capture that is about to be sent by this process. It is written
|
|
146
|
+
* already claimed, so a concurrent flush cannot send it too. On success the
|
|
147
|
+
* caller removes it; on failure `releaseSpool` returns it to the queue or
|
|
148
|
+
* quarantines it. If the process exits mid-send, the stale claim is
|
|
149
|
+
* recovered by a later flush.
|
|
150
|
+
*/
|
|
151
|
+
function stageSpool(kind, data, scope, meta = {}) {
|
|
152
|
+
if (!config.spool) return null
|
|
153
|
+
try {
|
|
154
|
+
ensureDir()
|
|
155
|
+
const file = claimName(path.join(spoolDir, newName(data)))
|
|
156
|
+
writePrivate(file, { kind, data, scope, ...meta, queuedAt: new Date(now()).toISOString() }, "wx")
|
|
157
|
+
return file
|
|
158
|
+
} catch (error) {
|
|
159
|
+
log.error("spool write failed", errorMessage(error))
|
|
160
|
+
return null
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function removeSpool(file) {
|
|
165
|
+
if (!file) return
|
|
166
|
+
try {
|
|
167
|
+
fs.unlinkSync(file)
|
|
168
|
+
} catch {
|
|
169
|
+
/* ignore */
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** Atomically take a pending entry. Null when another sender got it first. */
|
|
174
|
+
function claim(file) {
|
|
175
|
+
const claimed = claimName(file)
|
|
176
|
+
try {
|
|
177
|
+
fs.renameSync(file, claimed)
|
|
178
|
+
return claimed
|
|
179
|
+
} catch (error) {
|
|
180
|
+
if (error?.code !== "ENOENT") log.warn("spool claim failed", { file: path.basename(file), error: error?.code })
|
|
181
|
+
return null
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Move an entry aside with the reason so the rest of the queue drains. */
|
|
186
|
+
function quarantine(claimed, entry, reason, raw) {
|
|
187
|
+
const target = `${pendingOf(claimed)}${FAILED}`
|
|
188
|
+
const at = new Date(now()).toISOString()
|
|
189
|
+
const record =
|
|
190
|
+
entry && typeof entry === "object" && !Array.isArray(entry)
|
|
191
|
+
? { ...entry, quarantine: { reason, at } }
|
|
192
|
+
: { quarantine: { reason, at }, raw: String(raw ?? "") }
|
|
193
|
+
try {
|
|
194
|
+
writePrivate(claimed, record)
|
|
195
|
+
fs.renameSync(claimed, target)
|
|
196
|
+
log.warn("spool entry quarantined", { file: path.basename(target), reason })
|
|
197
|
+
return true
|
|
198
|
+
} catch (error) {
|
|
199
|
+
log.warn("spool quarantine failed", { file: path.basename(claimed), error: errorMessage(error) })
|
|
200
|
+
return false
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Record a failed send on a claimed entry: quarantine a permanent failure or
|
|
206
|
+
* an exhausted entry; otherwise schedule the next attempt and return the
|
|
207
|
+
* entry to the queue.
|
|
208
|
+
*
|
|
209
|
+
* @returns {"quarantined" | "kept"}
|
|
210
|
+
*/
|
|
211
|
+
function recordFailure(claimed, entry, error) {
|
|
212
|
+
const kind = classifyFailure(error)
|
|
213
|
+
const status = Number(error?.status) || 0
|
|
214
|
+
const message = oneLine(redactSecrets(errorMessage(error)), 300)
|
|
215
|
+
const attempts = (Number(entry.attempts) || 0) + 1
|
|
216
|
+
const queuedAt = Date.parse(entry.queuedAt ?? "")
|
|
217
|
+
const age = Number.isFinite(queuedAt) ? now() - queuedAt : 0
|
|
218
|
+
const updated = {
|
|
219
|
+
...entry,
|
|
220
|
+
attempts,
|
|
221
|
+
lastError: message,
|
|
222
|
+
lastStatus: status,
|
|
223
|
+
lastAttemptAt: new Date(now()).toISOString(),
|
|
224
|
+
nextAttemptAt: new Date(now() + backoffMs(attempts)).toISOString(),
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
let reason = null
|
|
228
|
+
if (kind === "permanent") reason = `rejected (${status}): ${message}`
|
|
229
|
+
else if (attempts >= MAX_ATTEMPTS) reason = `gave up after ${attempts} attempts: ${message}`
|
|
230
|
+
else if (age >= MAX_AGE_MS) reason = `gave up after ${Math.floor(age / 86_400_000)} days: ${message}`
|
|
231
|
+
if (reason && quarantine(claimed, updated, reason)) return "quarantined"
|
|
232
|
+
|
|
233
|
+
try {
|
|
234
|
+
writePrivate(claimed, updated)
|
|
235
|
+
fs.renameSync(claimed, pendingOf(claimed))
|
|
236
|
+
} catch (releaseError) {
|
|
237
|
+
log.warn("spool release failed", { file: path.basename(claimed), error: errorMessage(releaseError) })
|
|
238
|
+
}
|
|
239
|
+
log.warn("spool send failed; kept", { kind, status, attempts, error: message })
|
|
240
|
+
return "kept"
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Return a staged capture after a failed live send. */
|
|
244
|
+
function releaseSpool(claimed, error) {
|
|
245
|
+
if (!claimed) return null
|
|
246
|
+
try {
|
|
247
|
+
const entry = JSON.parse(fs.readFileSync(claimed, "utf8"))
|
|
248
|
+
return recordFailure(claimed, entry, error)
|
|
249
|
+
} catch (readError) {
|
|
250
|
+
log.warn("spool release failed", { file: path.basename(claimed), error: errorMessage(readError) })
|
|
251
|
+
return null
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
function invalidEntry(entry) {
|
|
256
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return "entry is not an object"
|
|
257
|
+
if (entry.kind !== "session" && entry.kind !== "conversation") return `unknown kind ${String(entry.kind)}`
|
|
258
|
+
if (!entry.data || typeof entry.data !== "object") return "entry has no data"
|
|
259
|
+
if (entry.kind === "conversation") {
|
|
260
|
+
if (!entry.data.scope || typeof entry.data.scope !== "object") return "conversation entry has no scope"
|
|
261
|
+
if (!entry.data.sessionID) return "conversation entry has no session"
|
|
262
|
+
}
|
|
263
|
+
if (entry.kind === "session") {
|
|
264
|
+
if (captureTurn) {
|
|
265
|
+
if (!turnSessionOf(entry)) return "session entry has no session id"
|
|
266
|
+
} else if (!entry.scope) {
|
|
267
|
+
return "session entry has no scope, and there is no control plane to place it"
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
return null
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** The session a turn entry belongs to, from this version's entry or an older one's scope. */
|
|
274
|
+
function turnSessionOf(entry) {
|
|
275
|
+
return entry.sessionID ?? entry.scope?.session_id ?? null
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Cross-process, cross-instance claim on a capture. Multiple plugin instances
|
|
280
|
+
* can observe the same global event stream, so turn identity has to be
|
|
281
|
+
* deduplicated outside memory.
|
|
282
|
+
*/
|
|
283
|
+
function claimCapture(key) {
|
|
284
|
+
try {
|
|
285
|
+
fs.mkdirSync(claimsDir, { recursive: true })
|
|
286
|
+
const file = path.join(claimsDir, `${shortHash(key, 24)}.claim`)
|
|
287
|
+
fs.closeSync(fs.openSync(file, "wx"))
|
|
288
|
+
return true
|
|
289
|
+
} catch (error) {
|
|
290
|
+
if (error?.code === "EEXIST") return false
|
|
291
|
+
log.warn("capture claim failed", errorMessage(error))
|
|
292
|
+
return true
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/** Give a claim back after a failed write so the next event can retry it. */
|
|
297
|
+
function releaseCapture(key) {
|
|
298
|
+
try {
|
|
299
|
+
fs.unlinkSync(path.join(claimsDir, `${shortHash(key, 24)}.claim`))
|
|
300
|
+
} catch {
|
|
301
|
+
/* ignore */
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function checkpointMark(sessionID) {
|
|
306
|
+
return path.join(claimsDir, `checkpoint-${shortHash(sessionID, 24)}.claim`)
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/** Record, for every process, that a pre-compaction checkpoint was just written. */
|
|
310
|
+
function markCheckpoint(sessionID) {
|
|
311
|
+
if (!sessionID) return
|
|
312
|
+
try {
|
|
313
|
+
fs.mkdirSync(claimsDir, { recursive: true })
|
|
314
|
+
const file = checkpointMark(sessionID)
|
|
315
|
+
fs.closeSync(fs.openSync(file, "a"))
|
|
316
|
+
const at = now() / 1000
|
|
317
|
+
fs.utimesSync(file, at, at)
|
|
318
|
+
} catch (error) {
|
|
319
|
+
log.debug("checkpoint mark failed", errorMessage(error))
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/** Whether any process wrote this session's pre-compaction checkpoint recently. */
|
|
324
|
+
function recentCheckpoint(sessionID, withinMs = 10 * 60_000) {
|
|
325
|
+
if (!sessionID) return false
|
|
326
|
+
try {
|
|
327
|
+
return now() - fs.statSync(checkpointMark(sessionID)).mtimeMs < withinMs
|
|
328
|
+
} catch {
|
|
329
|
+
return false
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
function pruneClaims(maxAgeDays = 7) {
|
|
334
|
+
try {
|
|
335
|
+
const cutoff = Date.now() - maxAgeDays * 86_400_000
|
|
336
|
+
for (const name of fs.readdirSync(claimsDir)) {
|
|
337
|
+
const file = path.join(claimsDir, name)
|
|
338
|
+
try {
|
|
339
|
+
if (fs.statSync(file).mtimeMs < cutoff) fs.unlinkSync(file)
|
|
340
|
+
} catch {
|
|
341
|
+
/* ignore */
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
} catch {
|
|
345
|
+
/* ignore */
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function enqueueSpool(kind, data, scope) {
|
|
350
|
+
const file = writeSpool(kind, data, scope)
|
|
351
|
+
if (file) log.warn("capture spooled", { file: path.basename(file), kind })
|
|
352
|
+
return file
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/** Return claims left by a process that died mid-send to the queue. */
|
|
356
|
+
function recoverStaleClaims(names) {
|
|
357
|
+
for (const name of names) {
|
|
358
|
+
const match = name.match(CLAIM_SUFFIX)
|
|
359
|
+
if (!match || !name.endsWith(INFLIGHT)) continue
|
|
360
|
+
if (now() - Number(match[1]) < CLAIM_STALE_MS) continue
|
|
361
|
+
const claimed = path.join(spoolDir, name)
|
|
362
|
+
try {
|
|
363
|
+
fs.renameSync(claimed, pendingOf(claimed))
|
|
364
|
+
log.info("stale spool claim recovered", { file: path.basename(pendingOf(claimed)) })
|
|
365
|
+
} catch {
|
|
366
|
+
/* another process recovered or finished it */
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
async function send(entry) {
|
|
372
|
+
if (entry.kind === "session") {
|
|
373
|
+
if (captureTurn) {
|
|
374
|
+
// Control mode: the gateway places the turn from the visibility it
|
|
375
|
+
// was captured with. An entry without one (written before visibility
|
|
376
|
+
// was recorded) replays as private, the narrower choice. A container
|
|
377
|
+
// in an older entry's scope is never sent.
|
|
378
|
+
await captureTurn({
|
|
379
|
+
sessionID: turnSessionOf(entry),
|
|
380
|
+
agent: entry.agent ?? entry.scope?.agent_id,
|
|
381
|
+
visibility: entry.visibility === "team" ? "team" : "private",
|
|
382
|
+
task: entry.task ?? entry.scope?.task_id,
|
|
383
|
+
payload: entry.data,
|
|
384
|
+
})
|
|
385
|
+
return
|
|
386
|
+
}
|
|
387
|
+
await backend.addSessionTurn({ payload: entry.data, scope: entry.scope })
|
|
388
|
+
} else {
|
|
389
|
+
await backend.addConversation(entry.data)
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/**
|
|
394
|
+
* Send what is due. Every pending entry is considered, oldest first, so a
|
|
395
|
+
* failing entry never blocks the ones behind it. `force` ignores the retry
|
|
396
|
+
* backoff (for `/mem-sync`).
|
|
397
|
+
*
|
|
398
|
+
* @param {{ force?: boolean }} [options]
|
|
399
|
+
*/
|
|
400
|
+
async function flushSpool(options = {}) {
|
|
401
|
+
const result = { flushed: 0, failed: 0, quarantined: 0, deferred: 0 }
|
|
402
|
+
if (!config.spool) return result
|
|
403
|
+
let names = []
|
|
404
|
+
try {
|
|
405
|
+
names = fs.readdirSync(spoolDir)
|
|
406
|
+
} catch {
|
|
407
|
+
return result
|
|
408
|
+
}
|
|
409
|
+
recoverStaleClaims(names)
|
|
410
|
+
try {
|
|
411
|
+
names = fs.readdirSync(spoolDir).filter((name) => name.endsWith(PENDING)).sort()
|
|
412
|
+
} catch {
|
|
413
|
+
return result
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
let sends = 0
|
|
417
|
+
let unreachable = 0
|
|
418
|
+
for (const name of names) {
|
|
419
|
+
if (sends >= MAX_SENDS_PER_FLUSH) break
|
|
420
|
+
const file = path.join(spoolDir, name)
|
|
421
|
+
|
|
422
|
+
// Peek before claiming so an entry that is not due costs no rename.
|
|
423
|
+
let peeked = null
|
|
424
|
+
try {
|
|
425
|
+
peeked = JSON.parse(fs.readFileSync(file, "utf8"))
|
|
426
|
+
} catch (error) {
|
|
427
|
+
if (error?.code === "ENOENT") continue
|
|
428
|
+
}
|
|
429
|
+
const requeued = Boolean(peeked?.quarantine)
|
|
430
|
+
if (!requeued && !options.force && peeked?.nextAttemptAt && Date.parse(peeked.nextAttemptAt) > now()) {
|
|
431
|
+
result.deferred += 1
|
|
432
|
+
continue
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
const claimed = claim(file)
|
|
436
|
+
if (!claimed) continue
|
|
437
|
+
|
|
438
|
+
let raw = ""
|
|
439
|
+
let entry
|
|
440
|
+
try {
|
|
441
|
+
raw = fs.readFileSync(claimed, "utf8")
|
|
442
|
+
entry = JSON.parse(raw)
|
|
443
|
+
if (entry?.quarantine) {
|
|
444
|
+
// An operator renamed a quarantined entry back: start it afresh.
|
|
445
|
+
entry = { ...entry, attempts: 0 }
|
|
446
|
+
delete entry.quarantine
|
|
447
|
+
delete entry.nextAttemptAt
|
|
448
|
+
}
|
|
449
|
+
} catch (error) {
|
|
450
|
+
result.failed += 1
|
|
451
|
+
if (quarantine(claimed, null, `unreadable: ${errorMessage(error)}`, raw)) result.quarantined += 1
|
|
452
|
+
continue
|
|
453
|
+
}
|
|
454
|
+
const invalid = invalidEntry(entry)
|
|
455
|
+
if (invalid) {
|
|
456
|
+
result.failed += 1
|
|
457
|
+
if (quarantine(claimed, entry, `invalid: ${invalid}`)) result.quarantined += 1
|
|
458
|
+
continue
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
sends += 1
|
|
462
|
+
try {
|
|
463
|
+
await send(entry)
|
|
464
|
+
fs.unlinkSync(claimed)
|
|
465
|
+
result.flushed += 1
|
|
466
|
+
unreachable = 0
|
|
467
|
+
} catch (error) {
|
|
468
|
+
result.failed += 1
|
|
469
|
+
if (recordFailure(claimed, entry, error) === "quarantined") result.quarantined += 1
|
|
470
|
+
if (classifyFailure(error) === "unreachable") {
|
|
471
|
+
unreachable += 1
|
|
472
|
+
if (unreachable >= UNREACHABLE_STOP) {
|
|
473
|
+
log.warn("spool flush stopped: backend unreachable", { sent: sends })
|
|
474
|
+
break
|
|
475
|
+
}
|
|
476
|
+
} else {
|
|
477
|
+
unreachable = 0
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
return result
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
/** Queue depth by state, for mem_status and /mem-sync. */
|
|
485
|
+
function spoolStats() {
|
|
486
|
+
const stats = { pending: 0, inflight: 0, quarantined: 0, oldest: null }
|
|
487
|
+
let names = []
|
|
488
|
+
try {
|
|
489
|
+
names = fs.readdirSync(spoolDir)
|
|
490
|
+
} catch {
|
|
491
|
+
return stats
|
|
492
|
+
}
|
|
493
|
+
let oldest = Number.POSITIVE_INFINITY
|
|
494
|
+
for (const name of names) {
|
|
495
|
+
if (name.endsWith(PENDING)) {
|
|
496
|
+
stats.pending += 1
|
|
497
|
+
const written = Number.parseInt(name, 10)
|
|
498
|
+
if (Number.isFinite(written) && written < oldest) oldest = written
|
|
499
|
+
} else if (name.endsWith(INFLIGHT)) stats.inflight += 1
|
|
500
|
+
else if (name.endsWith(FAILED)) stats.quarantined += 1
|
|
501
|
+
}
|
|
502
|
+
if (Number.isFinite(oldest)) stats.oldest = new Date(oldest).toISOString()
|
|
503
|
+
return stats
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
return {
|
|
507
|
+
writeSpool,
|
|
508
|
+
stageSpool,
|
|
509
|
+
releaseSpool,
|
|
510
|
+
removeSpool,
|
|
511
|
+
claimCapture,
|
|
512
|
+
releaseCapture,
|
|
513
|
+
markCheckpoint,
|
|
514
|
+
recentCheckpoint,
|
|
515
|
+
pruneClaims,
|
|
516
|
+
enqueueSpool,
|
|
517
|
+
flushSpool,
|
|
518
|
+
spoolStats,
|
|
519
|
+
}
|
|
520
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { createToolkit } from "./shared.js"
|
|
2
|
+
import { registerRecall } from "./recall.js"
|
|
3
|
+
import { registerWiki } from "./wiki.js"
|
|
4
|
+
import { registerMemory } from "./memory.js"
|
|
5
|
+
import { registerStatus } from "./status.js"
|
|
6
|
+
import { registerSpaces } from "./spaces.js"
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Register the `mem_*` tool family.
|
|
10
|
+
*
|
|
11
|
+
* Recall is deliberately a tool rather than an automatic injection: pull-based
|
|
12
|
+
* retrieval is cheaper, attributable, and does not invalidate prompt caches.
|
|
13
|
+
*
|
|
14
|
+
* The tools are registered even when memory is unavailable (a gateway or
|
|
15
|
+
* Jelly install with no credential): each then answers the one sentence
|
|
16
|
+
* `runtime.unavailable()` gives, `mem_status` included, and makes no request.
|
|
17
|
+
*
|
|
18
|
+
* @param {any} ctx
|
|
19
|
+
* @param {any} runtime
|
|
20
|
+
*/
|
|
21
|
+
export async function registerTools(ctx, runtime) {
|
|
22
|
+
if (!runtime.config.tools) return null
|
|
23
|
+
const toolkit = createToolkit(ctx, runtime)
|
|
24
|
+
return ctx.tool.transform((editor) => {
|
|
25
|
+
editor.namespace({ name: "mem", description: "Durable private and shared-space memory" })
|
|
26
|
+
const gated = gateEditor(editor, runtime)
|
|
27
|
+
registerRecall(gated, toolkit)
|
|
28
|
+
registerWiki(gated, toolkit)
|
|
29
|
+
registerMemory(gated, toolkit)
|
|
30
|
+
registerStatus(gated, toolkit)
|
|
31
|
+
registerSpaces(gated, toolkit)
|
|
32
|
+
})
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** An editor whose added tools answer the unavailable sentence before running. */
|
|
36
|
+
function gateEditor(editor, runtime) {
|
|
37
|
+
return new Proxy(editor, {
|
|
38
|
+
get(target, key, receiver) {
|
|
39
|
+
if (key !== "add") return Reflect.get(target, key, receiver)
|
|
40
|
+
return (definition) =>
|
|
41
|
+
target.add({
|
|
42
|
+
...definition,
|
|
43
|
+
execute: async (input, context) => {
|
|
44
|
+
const reason = runtime.unavailable?.() ?? ""
|
|
45
|
+
if (reason) return { content: reason }
|
|
46
|
+
return definition.execute(input, context)
|
|
47
|
+
},
|
|
48
|
+
})
|
|
49
|
+
},
|
|
50
|
+
})
|
|
51
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { errorMessage, neutralizeFenceClosers, oneLine } from "../util.js"
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Discover the wiki and code-graph assets reachable from a scope. The asset
|
|
5
|
+
* table is the index, but wiki and code-graph services are also first-class and
|
|
6
|
+
* are not always mirrored into it, so both sources are merged.
|
|
7
|
+
*/
|
|
8
|
+
export async function discoverKnowledge(backend, log, assetType, scope) {
|
|
9
|
+
const ids = []
|
|
10
|
+
try {
|
|
11
|
+
const data = await backend.listAccessibleAssets({ scope, assetType })
|
|
12
|
+
for (const asset of data?.items ?? []) {
|
|
13
|
+
const knowledgeType = asset.asset_type === "llm_wiki" || asset.asset_type === "code_graph"
|
|
14
|
+
if (!knowledgeType) continue
|
|
15
|
+
if (assetType && asset.asset_type !== assetType) continue
|
|
16
|
+
const id = asset.content_ref ?? asset.asset_id
|
|
17
|
+
if (id) ids.push(id)
|
|
18
|
+
}
|
|
19
|
+
} catch (error) {
|
|
20
|
+
log.debug("knowledge asset discovery failed", { assetType, error: errorMessage(error) })
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
if (!assetType || assetType === "llm_wiki") {
|
|
24
|
+
try {
|
|
25
|
+
const data = await backend.listWiki({ scope })
|
|
26
|
+
for (const item of data?.items ?? []) if (item.wiki_id) ids.push(item.wiki_id)
|
|
27
|
+
} catch (error) {
|
|
28
|
+
log.debug("wiki discovery failed", { error: errorMessage(error) })
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (!assetType || assetType === "code_graph") {
|
|
32
|
+
try {
|
|
33
|
+
const data = await backend.listCodeGraph({ scope })
|
|
34
|
+
for (const item of data?.items ?? []) if (item.code_graph_id) ids.push(item.code_graph_id)
|
|
35
|
+
} catch (error) {
|
|
36
|
+
log.debug("code-graph discovery failed", { error: errorMessage(error) })
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return [...new Set(ids)]
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const DEFAULT_KNOWLEDGE_CHARS = 6000
|
|
43
|
+
const DEFAULT_ASSET_CHARS = 3000
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Search knowledge assets and render the results as fenced data.
|
|
47
|
+
*
|
|
48
|
+
* The total output is bounded, not just each page: three assets at the old
|
|
49
|
+
* per-page cap could add 9,000 characters to every recall. Assets are read in
|
|
50
|
+
* order until the budget is spent, so the caller gets whole early results
|
|
51
|
+
* instead of three equally truncated ones.
|
|
52
|
+
*/
|
|
53
|
+
export async function searchKnowledge(backend, ids, query, scoped, options = {}) {
|
|
54
|
+
const maxChars = Number.isFinite(options.maxChars) && options.maxChars > 0 ? Math.floor(options.maxChars) : DEFAULT_KNOWLEDGE_CHARS
|
|
55
|
+
const perAssetChars =
|
|
56
|
+
Number.isFinite(options.perAssetChars) && options.perAssetChars > 0
|
|
57
|
+
? Math.min(Math.floor(options.perAssetChars), maxChars)
|
|
58
|
+
: Math.min(DEFAULT_ASSET_CHARS, maxChars)
|
|
59
|
+
const blocks = []
|
|
60
|
+
let used = 0
|
|
61
|
+
for (const knowledgeID of ids) {
|
|
62
|
+
const remaining = maxChars - used
|
|
63
|
+
const source = neutralizeFenceClosers(oneLine(knowledgeID, 80))
|
|
64
|
+
// The fence and the separator are part of the budget; a page must not push
|
|
65
|
+
// the block past maxChars when the caller asked for a bound.
|
|
66
|
+
const fenceChars = `<knowledge source="${source}">\n\n</knowledge>`.length
|
|
67
|
+
if (remaining < fenceChars + 122) break
|
|
68
|
+
const bodyRoom = Math.max(120, Math.min(perAssetChars, remaining - fenceChars - 2))
|
|
69
|
+
try {
|
|
70
|
+
const result = await backend.knowledgeCall({
|
|
71
|
+
knowledgeID: String(knowledgeID),
|
|
72
|
+
toolName: "search",
|
|
73
|
+
params: { query, limit: 5 },
|
|
74
|
+
scope: scoped,
|
|
75
|
+
})
|
|
76
|
+
const rendered = typeof result === "string" ? result : JSON.stringify(result, null, 2)
|
|
77
|
+
// Truncate first, then neutralize: the closing tag is appended after,
|
|
78
|
+
// so neither a long page nor a crafted one can escape the fence.
|
|
79
|
+
const body = neutralizeFenceClosers(rendered.slice(0, bodyRoom))
|
|
80
|
+
const block = `<knowledge source="${source}">\n${body}\n</knowledge>`
|
|
81
|
+
blocks.push(block)
|
|
82
|
+
used += block.length + 2
|
|
83
|
+
} catch (error) {
|
|
84
|
+
const block = neutralizeFenceClosers(`(knowledge ${oneLine(knowledgeID, 60)} failed: ${errorMessage(error)})`)
|
|
85
|
+
blocks.push(block)
|
|
86
|
+
used += block.length + 2
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return blocks.join("\n\n")
|
|
90
|
+
}
|