@joe-broadhead/opencode-mem 0.0.0-stage → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/spool.js ADDED
@@ -0,0 +1,520 @@
1
+ import fs from "node:fs"
2
+ import path from "node:path"
3
+ import { errorMessage, oneLine, redactSecrets, shortHash } from "./util.js"
4
+
5
+ /** At most this many sends per flush; entries that are not due do not count. */
6
+ const MAX_SENDS_PER_FLUSH = 25
7
+ /** A transient failure is retried with backoff until either cap is reached. */
8
+ const MAX_ATTEMPTS = 50
9
+ const MAX_AGE_MS = 7 * 86_400_000
10
+ const BACKOFF_BASE_MS = 30_000
11
+ const BACKOFF_MAX_MS = 60 * 60_000
12
+ /** A claim older than this belongs to a process that died mid-send. */
13
+ const CLAIM_STALE_MS = 5 * 60_000
14
+ /** Stop a flush after this many consecutive "backend unreachable" failures. */
15
+ const UNREACHABLE_STOP = 3
16
+
17
+ const PENDING = ".json"
18
+ const INFLIGHT = ".inflight"
19
+ const FAILED = ".failed"
20
+ const CLAIM_SUFFIX = /\.(\d+)-(\d+)\.inflight$/
21
+
22
+ /**
23
+ * Classify a failed send.
24
+ *
25
+ * - `permanent`: the server rejected this entry (a 4xx such as 400, 403, 409,
26
+ * 413, or 422). Retrying the same bytes cannot succeed, so the entry is
27
+ * quarantined with the reason.
28
+ * - `unreachable`: no answer from the backend (transport error, timeout, 502,
29
+ * 503, 504). The rest of the queue would fail the same way.
30
+ * - `transient`: anything else, including 401 (a credential problem, fixed by
31
+ * the operator, not by the entry), 404 (a route the edge does not forward),
32
+ * 408, 425, 429, other 5xx, and every control-plane failure: placement is
33
+ * resolved again on the next attempt.
34
+ *
35
+ * @param {any} error
36
+ * @returns {"permanent" | "unreachable" | "transient"}
37
+ */
38
+ export function classifyFailure(error) {
39
+ const status = Number(error?.status)
40
+ if (!Number.isFinite(status) || status === 0 || status === 408) return "unreachable"
41
+ if (status === 502 || status === 503 || status === 504) return "unreachable"
42
+ if (error?.name === "ControlError") return "transient"
43
+ if (status >= 400 && status < 500 && ![401, 404, 425, 429].includes(status)) return "permanent"
44
+ return "transient"
45
+ }
46
+
47
+ function backoffMs(attempts) {
48
+ return Math.min(BACKOFF_MAX_MS, BACKOFF_BASE_MS * 2 ** Math.max(0, attempts - 1))
49
+ }
50
+
51
+ /**
52
+ * Durable local queue for captures that could not reach the backend, plus the
53
+ * cross-process claims that keep duplicate event streams from capturing a turn
54
+ * twice.
55
+ *
56
+ * The spool holds memory text, so the directory is private to the user (0700)
57
+ * and each entry is readable only by them (0600).
58
+ *
59
+ * Entry life cycle:
60
+ *
61
+ * - `<name>.json` is pending.
62
+ * - `<name>.json.<ms>-<pid>.inflight` is claimed by one sender. The claim is an
63
+ * atomic rename, so two processes, or a flush racing a live capture, never
64
+ * send the same entry twice. A claim older than five minutes is returned to
65
+ * pending.
66
+ * - `<name>.json.failed` is quarantined. It holds the entry plus
67
+ * `quarantine: { reason, at }`. Rename it back to `.json` to retry it; the
68
+ * attempt count restarts.
69
+ *
70
+ * In control mode a turn entry records the session, agent, task, and the
71
+ * visibility it was captured with, never a container. Its replay goes through
72
+ * the gateway's capture route with that visibility (`captureTurn`), so the
73
+ * gateway places it again, and a private turn can never land in a shared space
74
+ * that was bound while it waited. A container stored by an older plugin
75
+ * version is ignored on replay. In local direct mode the entry carries the
76
+ * fixed local scope and is written to `conversation/add`.
77
+ *
78
+ * @param {{ config: any, log: any, backend: any, captureTurn?: Function | null, now?: () => number }} input
79
+ */
80
+ export function createSpool(input) {
81
+ const { config, log, backend } = input
82
+ const captureTurn = typeof input.captureTurn === "function" ? input.captureTurn : null
83
+ const now = typeof input.now === "function" ? input.now : Date.now
84
+ const spoolDir = path.join(config.stateDir, "spool")
85
+ const claimsDir = path.join(config.stateDir, "claims")
86
+ let sequence = 0
87
+
88
+ function ensureDir() {
89
+ fs.mkdirSync(spoolDir, { recursive: true, mode: 0o700 })
90
+ try {
91
+ fs.chmodSync(spoolDir, 0o700)
92
+ } catch {
93
+ /* best effort on platforms without POSIX modes */
94
+ }
95
+ }
96
+
97
+ function writePrivate(file, record, flag = "w") {
98
+ fs.writeFileSync(file, JSON.stringify(record, null, 2), { mode: 0o600, flag })
99
+ try {
100
+ fs.chmodSync(file, 0o600)
101
+ } catch {
102
+ /* best effort on platforms without POSIX modes */
103
+ }
104
+ }
105
+
106
+ /**
107
+ * Names sort in enqueue order, which is what "oldest first" relies on: the
108
+ * timestamp is fixed-width for centuries and the per-process sequence is
109
+ * zero-padded, so `...-10-` cannot sort before `...-9-` within a millisecond.
110
+ */
111
+ function newName(data) {
112
+ sequence += 1
113
+ const stamp = String(now()).padStart(13, "0")
114
+ return `${stamp}-${process.pid}-${String(sequence).padStart(9, "0")}-${shortHash(JSON.stringify(data), 8)}${PENDING}`
115
+ }
116
+
117
+ function claimName(file) {
118
+ return `${file}.${now()}-${process.pid}${INFLIGHT}`
119
+ }
120
+
121
+ /** The pending path a claimed or quarantined file belongs to. */
122
+ function pendingOf(file) {
123
+ return file.replace(CLAIM_SUFFIX, "").replace(/\.failed$/, "")
124
+ }
125
+
126
+ /**
127
+ * Persist one capture locally as pending. Returns the file path on success
128
+ * and null when spooling is disabled or the write fails; callers must not
129
+ * claim a capture is queued otherwise.
130
+ */
131
+ function writeSpool(kind, data, scope, meta = {}) {
132
+ if (!config.spool) return null
133
+ try {
134
+ ensureDir()
135
+ const file = path.join(spoolDir, newName(data))
136
+ writePrivate(file, { kind, data, scope, ...meta, queuedAt: new Date(now()).toISOString() }, "wx")
137
+ return file
138
+ } catch (error) {
139
+ log.error("spool write failed", errorMessage(error))
140
+ return null
141
+ }
142
+ }
143
+
144
+ /**
145
+ * Persist a capture that is about to be sent by this process. It is written
146
+ * already claimed, so a concurrent flush cannot send it too. On success the
147
+ * caller removes it; on failure `releaseSpool` returns it to the queue or
148
+ * quarantines it. If the process exits mid-send, the stale claim is
149
+ * recovered by a later flush.
150
+ */
151
+ function stageSpool(kind, data, scope, meta = {}) {
152
+ if (!config.spool) return null
153
+ try {
154
+ ensureDir()
155
+ const file = claimName(path.join(spoolDir, newName(data)))
156
+ writePrivate(file, { kind, data, scope, ...meta, queuedAt: new Date(now()).toISOString() }, "wx")
157
+ return file
158
+ } catch (error) {
159
+ log.error("spool write failed", errorMessage(error))
160
+ return null
161
+ }
162
+ }
163
+
164
+ function removeSpool(file) {
165
+ if (!file) return
166
+ try {
167
+ fs.unlinkSync(file)
168
+ } catch {
169
+ /* ignore */
170
+ }
171
+ }
172
+
173
+ /** Atomically take a pending entry. Null when another sender got it first. */
174
+ function claim(file) {
175
+ const claimed = claimName(file)
176
+ try {
177
+ fs.renameSync(file, claimed)
178
+ return claimed
179
+ } catch (error) {
180
+ if (error?.code !== "ENOENT") log.warn("spool claim failed", { file: path.basename(file), error: error?.code })
181
+ return null
182
+ }
183
+ }
184
+
185
+ /** Move an entry aside with the reason so the rest of the queue drains. */
186
+ function quarantine(claimed, entry, reason, raw) {
187
+ const target = `${pendingOf(claimed)}${FAILED}`
188
+ const at = new Date(now()).toISOString()
189
+ const record =
190
+ entry && typeof entry === "object" && !Array.isArray(entry)
191
+ ? { ...entry, quarantine: { reason, at } }
192
+ : { quarantine: { reason, at }, raw: String(raw ?? "") }
193
+ try {
194
+ writePrivate(claimed, record)
195
+ fs.renameSync(claimed, target)
196
+ log.warn("spool entry quarantined", { file: path.basename(target), reason })
197
+ return true
198
+ } catch (error) {
199
+ log.warn("spool quarantine failed", { file: path.basename(claimed), error: errorMessage(error) })
200
+ return false
201
+ }
202
+ }
203
+
204
+ /**
205
+ * Record a failed send on a claimed entry: quarantine a permanent failure or
206
+ * an exhausted entry; otherwise schedule the next attempt and return the
207
+ * entry to the queue.
208
+ *
209
+ * @returns {"quarantined" | "kept"}
210
+ */
211
+ function recordFailure(claimed, entry, error) {
212
+ const kind = classifyFailure(error)
213
+ const status = Number(error?.status) || 0
214
+ const message = oneLine(redactSecrets(errorMessage(error)), 300)
215
+ const attempts = (Number(entry.attempts) || 0) + 1
216
+ const queuedAt = Date.parse(entry.queuedAt ?? "")
217
+ const age = Number.isFinite(queuedAt) ? now() - queuedAt : 0
218
+ const updated = {
219
+ ...entry,
220
+ attempts,
221
+ lastError: message,
222
+ lastStatus: status,
223
+ lastAttemptAt: new Date(now()).toISOString(),
224
+ nextAttemptAt: new Date(now() + backoffMs(attempts)).toISOString(),
225
+ }
226
+
227
+ let reason = null
228
+ if (kind === "permanent") reason = `rejected (${status}): ${message}`
229
+ else if (attempts >= MAX_ATTEMPTS) reason = `gave up after ${attempts} attempts: ${message}`
230
+ else if (age >= MAX_AGE_MS) reason = `gave up after ${Math.floor(age / 86_400_000)} days: ${message}`
231
+ if (reason && quarantine(claimed, updated, reason)) return "quarantined"
232
+
233
+ try {
234
+ writePrivate(claimed, updated)
235
+ fs.renameSync(claimed, pendingOf(claimed))
236
+ } catch (releaseError) {
237
+ log.warn("spool release failed", { file: path.basename(claimed), error: errorMessage(releaseError) })
238
+ }
239
+ log.warn("spool send failed; kept", { kind, status, attempts, error: message })
240
+ return "kept"
241
+ }
242
+
243
+ /** Return a staged capture after a failed live send. */
244
+ function releaseSpool(claimed, error) {
245
+ if (!claimed) return null
246
+ try {
247
+ const entry = JSON.parse(fs.readFileSync(claimed, "utf8"))
248
+ return recordFailure(claimed, entry, error)
249
+ } catch (readError) {
250
+ log.warn("spool release failed", { file: path.basename(claimed), error: errorMessage(readError) })
251
+ return null
252
+ }
253
+ }
254
+
255
+ function invalidEntry(entry) {
256
+ if (!entry || typeof entry !== "object" || Array.isArray(entry)) return "entry is not an object"
257
+ if (entry.kind !== "session" && entry.kind !== "conversation") return `unknown kind ${String(entry.kind)}`
258
+ if (!entry.data || typeof entry.data !== "object") return "entry has no data"
259
+ if (entry.kind === "conversation") {
260
+ if (!entry.data.scope || typeof entry.data.scope !== "object") return "conversation entry has no scope"
261
+ if (!entry.data.sessionID) return "conversation entry has no session"
262
+ }
263
+ if (entry.kind === "session") {
264
+ if (captureTurn) {
265
+ if (!turnSessionOf(entry)) return "session entry has no session id"
266
+ } else if (!entry.scope) {
267
+ return "session entry has no scope, and there is no control plane to place it"
268
+ }
269
+ }
270
+ return null
271
+ }
272
+
273
+ /** The session a turn entry belongs to, from this version's entry or an older one's scope. */
274
+ function turnSessionOf(entry) {
275
+ return entry.sessionID ?? entry.scope?.session_id ?? null
276
+ }
277
+
278
+ /**
279
+ * Cross-process, cross-instance claim on a capture. Multiple plugin instances
280
+ * can observe the same global event stream, so turn identity has to be
281
+ * deduplicated outside memory.
282
+ */
283
+ function claimCapture(key) {
284
+ try {
285
+ fs.mkdirSync(claimsDir, { recursive: true })
286
+ const file = path.join(claimsDir, `${shortHash(key, 24)}.claim`)
287
+ fs.closeSync(fs.openSync(file, "wx"))
288
+ return true
289
+ } catch (error) {
290
+ if (error?.code === "EEXIST") return false
291
+ log.warn("capture claim failed", errorMessage(error))
292
+ return true
293
+ }
294
+ }
295
+
296
+ /** Give a claim back after a failed write so the next event can retry it. */
297
+ function releaseCapture(key) {
298
+ try {
299
+ fs.unlinkSync(path.join(claimsDir, `${shortHash(key, 24)}.claim`))
300
+ } catch {
301
+ /* ignore */
302
+ }
303
+ }
304
+
305
+ function checkpointMark(sessionID) {
306
+ return path.join(claimsDir, `checkpoint-${shortHash(sessionID, 24)}.claim`)
307
+ }
308
+
309
+ /** Record, for every process, that a pre-compaction checkpoint was just written. */
310
+ function markCheckpoint(sessionID) {
311
+ if (!sessionID) return
312
+ try {
313
+ fs.mkdirSync(claimsDir, { recursive: true })
314
+ const file = checkpointMark(sessionID)
315
+ fs.closeSync(fs.openSync(file, "a"))
316
+ const at = now() / 1000
317
+ fs.utimesSync(file, at, at)
318
+ } catch (error) {
319
+ log.debug("checkpoint mark failed", errorMessage(error))
320
+ }
321
+ }
322
+
323
+ /** Whether any process wrote this session's pre-compaction checkpoint recently. */
324
+ function recentCheckpoint(sessionID, withinMs = 10 * 60_000) {
325
+ if (!sessionID) return false
326
+ try {
327
+ return now() - fs.statSync(checkpointMark(sessionID)).mtimeMs < withinMs
328
+ } catch {
329
+ return false
330
+ }
331
+ }
332
+
333
+ function pruneClaims(maxAgeDays = 7) {
334
+ try {
335
+ const cutoff = Date.now() - maxAgeDays * 86_400_000
336
+ for (const name of fs.readdirSync(claimsDir)) {
337
+ const file = path.join(claimsDir, name)
338
+ try {
339
+ if (fs.statSync(file).mtimeMs < cutoff) fs.unlinkSync(file)
340
+ } catch {
341
+ /* ignore */
342
+ }
343
+ }
344
+ } catch {
345
+ /* ignore */
346
+ }
347
+ }
348
+
349
+ function enqueueSpool(kind, data, scope) {
350
+ const file = writeSpool(kind, data, scope)
351
+ if (file) log.warn("capture spooled", { file: path.basename(file), kind })
352
+ return file
353
+ }
354
+
355
+ /** Return claims left by a process that died mid-send to the queue. */
356
+ function recoverStaleClaims(names) {
357
+ for (const name of names) {
358
+ const match = name.match(CLAIM_SUFFIX)
359
+ if (!match || !name.endsWith(INFLIGHT)) continue
360
+ if (now() - Number(match[1]) < CLAIM_STALE_MS) continue
361
+ const claimed = path.join(spoolDir, name)
362
+ try {
363
+ fs.renameSync(claimed, pendingOf(claimed))
364
+ log.info("stale spool claim recovered", { file: path.basename(pendingOf(claimed)) })
365
+ } catch {
366
+ /* another process recovered or finished it */
367
+ }
368
+ }
369
+ }
370
+
371
+ async function send(entry) {
372
+ if (entry.kind === "session") {
373
+ if (captureTurn) {
374
+ // Control mode: the gateway places the turn from the visibility it
375
+ // was captured with. An entry without one (written before visibility
376
+ // was recorded) replays as private, the narrower choice. A container
377
+ // in an older entry's scope is never sent.
378
+ await captureTurn({
379
+ sessionID: turnSessionOf(entry),
380
+ agent: entry.agent ?? entry.scope?.agent_id,
381
+ visibility: entry.visibility === "team" ? "team" : "private",
382
+ task: entry.task ?? entry.scope?.task_id,
383
+ payload: entry.data,
384
+ })
385
+ return
386
+ }
387
+ await backend.addSessionTurn({ payload: entry.data, scope: entry.scope })
388
+ } else {
389
+ await backend.addConversation(entry.data)
390
+ }
391
+ }
392
+
393
+ /**
394
+ * Send what is due. Every pending entry is considered, oldest first, so a
395
+ * failing entry never blocks the ones behind it. `force` ignores the retry
396
+ * backoff (for `/mem-sync`).
397
+ *
398
+ * @param {{ force?: boolean }} [options]
399
+ */
400
+ async function flushSpool(options = {}) {
401
+ const result = { flushed: 0, failed: 0, quarantined: 0, deferred: 0 }
402
+ if (!config.spool) return result
403
+ let names = []
404
+ try {
405
+ names = fs.readdirSync(spoolDir)
406
+ } catch {
407
+ return result
408
+ }
409
+ recoverStaleClaims(names)
410
+ try {
411
+ names = fs.readdirSync(spoolDir).filter((name) => name.endsWith(PENDING)).sort()
412
+ } catch {
413
+ return result
414
+ }
415
+
416
+ let sends = 0
417
+ let unreachable = 0
418
+ for (const name of names) {
419
+ if (sends >= MAX_SENDS_PER_FLUSH) break
420
+ const file = path.join(spoolDir, name)
421
+
422
+ // Peek before claiming so an entry that is not due costs no rename.
423
+ let peeked = null
424
+ try {
425
+ peeked = JSON.parse(fs.readFileSync(file, "utf8"))
426
+ } catch (error) {
427
+ if (error?.code === "ENOENT") continue
428
+ }
429
+ const requeued = Boolean(peeked?.quarantine)
430
+ if (!requeued && !options.force && peeked?.nextAttemptAt && Date.parse(peeked.nextAttemptAt) > now()) {
431
+ result.deferred += 1
432
+ continue
433
+ }
434
+
435
+ const claimed = claim(file)
436
+ if (!claimed) continue
437
+
438
+ let raw = ""
439
+ let entry
440
+ try {
441
+ raw = fs.readFileSync(claimed, "utf8")
442
+ entry = JSON.parse(raw)
443
+ if (entry?.quarantine) {
444
+ // An operator renamed a quarantined entry back: start it afresh.
445
+ entry = { ...entry, attempts: 0 }
446
+ delete entry.quarantine
447
+ delete entry.nextAttemptAt
448
+ }
449
+ } catch (error) {
450
+ result.failed += 1
451
+ if (quarantine(claimed, null, `unreadable: ${errorMessage(error)}`, raw)) result.quarantined += 1
452
+ continue
453
+ }
454
+ const invalid = invalidEntry(entry)
455
+ if (invalid) {
456
+ result.failed += 1
457
+ if (quarantine(claimed, entry, `invalid: ${invalid}`)) result.quarantined += 1
458
+ continue
459
+ }
460
+
461
+ sends += 1
462
+ try {
463
+ await send(entry)
464
+ fs.unlinkSync(claimed)
465
+ result.flushed += 1
466
+ unreachable = 0
467
+ } catch (error) {
468
+ result.failed += 1
469
+ if (recordFailure(claimed, entry, error) === "quarantined") result.quarantined += 1
470
+ if (classifyFailure(error) === "unreachable") {
471
+ unreachable += 1
472
+ if (unreachable >= UNREACHABLE_STOP) {
473
+ log.warn("spool flush stopped: backend unreachable", { sent: sends })
474
+ break
475
+ }
476
+ } else {
477
+ unreachable = 0
478
+ }
479
+ }
480
+ }
481
+ return result
482
+ }
483
+
484
+ /** Queue depth by state, for mem_status and /mem-sync. */
485
+ function spoolStats() {
486
+ const stats = { pending: 0, inflight: 0, quarantined: 0, oldest: null }
487
+ let names = []
488
+ try {
489
+ names = fs.readdirSync(spoolDir)
490
+ } catch {
491
+ return stats
492
+ }
493
+ let oldest = Number.POSITIVE_INFINITY
494
+ for (const name of names) {
495
+ if (name.endsWith(PENDING)) {
496
+ stats.pending += 1
497
+ const written = Number.parseInt(name, 10)
498
+ if (Number.isFinite(written) && written < oldest) oldest = written
499
+ } else if (name.endsWith(INFLIGHT)) stats.inflight += 1
500
+ else if (name.endsWith(FAILED)) stats.quarantined += 1
501
+ }
502
+ if (Number.isFinite(oldest)) stats.oldest = new Date(oldest).toISOString()
503
+ return stats
504
+ }
505
+
506
+ return {
507
+ writeSpool,
508
+ stageSpool,
509
+ releaseSpool,
510
+ removeSpool,
511
+ claimCapture,
512
+ releaseCapture,
513
+ markCheckpoint,
514
+ recentCheckpoint,
515
+ pruneClaims,
516
+ enqueueSpool,
517
+ flushSpool,
518
+ spoolStats,
519
+ }
520
+ }
@@ -0,0 +1,51 @@
1
+ import { createToolkit } from "./shared.js"
2
+ import { registerRecall } from "./recall.js"
3
+ import { registerWiki } from "./wiki.js"
4
+ import { registerMemory } from "./memory.js"
5
+ import { registerStatus } from "./status.js"
6
+ import { registerSpaces } from "./spaces.js"
7
+
8
+ /**
9
+ * Register the `mem_*` tool family.
10
+ *
11
+ * Recall is deliberately a tool rather than an automatic injection: pull-based
12
+ * retrieval is cheaper, attributable, and does not invalidate prompt caches.
13
+ *
14
+ * The tools are registered even when memory is unavailable (a gateway or
15
+ * Jelly install with no credential): each then answers the one sentence
16
+ * `runtime.unavailable()` gives, `mem_status` included, and makes no request.
17
+ *
18
+ * @param {any} ctx
19
+ * @param {any} runtime
20
+ */
21
+ export async function registerTools(ctx, runtime) {
22
+ if (!runtime.config.tools) return null
23
+ const toolkit = createToolkit(ctx, runtime)
24
+ return ctx.tool.transform((editor) => {
25
+ editor.namespace({ name: "mem", description: "Durable private and shared-space memory" })
26
+ const gated = gateEditor(editor, runtime)
27
+ registerRecall(gated, toolkit)
28
+ registerWiki(gated, toolkit)
29
+ registerMemory(gated, toolkit)
30
+ registerStatus(gated, toolkit)
31
+ registerSpaces(gated, toolkit)
32
+ })
33
+ }
34
+
35
+ /** An editor whose added tools answer the unavailable sentence before running. */
36
+ function gateEditor(editor, runtime) {
37
+ return new Proxy(editor, {
38
+ get(target, key, receiver) {
39
+ if (key !== "add") return Reflect.get(target, key, receiver)
40
+ return (definition) =>
41
+ target.add({
42
+ ...definition,
43
+ execute: async (input, context) => {
44
+ const reason = runtime.unavailable?.() ?? ""
45
+ if (reason) return { content: reason }
46
+ return definition.execute(input, context)
47
+ },
48
+ })
49
+ },
50
+ })
51
+ }
@@ -0,0 +1,90 @@
1
+ import { errorMessage, neutralizeFenceClosers, oneLine } from "../util.js"
2
+
3
+ /**
4
+ * Discover the wiki and code-graph assets reachable from a scope. The asset
5
+ * table is the index, but wiki and code-graph services are also first-class and
6
+ * are not always mirrored into it, so both sources are merged.
7
+ */
8
+ export async function discoverKnowledge(backend, log, assetType, scope) {
9
+ const ids = []
10
+ try {
11
+ const data = await backend.listAccessibleAssets({ scope, assetType })
12
+ for (const asset of data?.items ?? []) {
13
+ const knowledgeType = asset.asset_type === "llm_wiki" || asset.asset_type === "code_graph"
14
+ if (!knowledgeType) continue
15
+ if (assetType && asset.asset_type !== assetType) continue
16
+ const id = asset.content_ref ?? asset.asset_id
17
+ if (id) ids.push(id)
18
+ }
19
+ } catch (error) {
20
+ log.debug("knowledge asset discovery failed", { assetType, error: errorMessage(error) })
21
+ }
22
+
23
+ if (!assetType || assetType === "llm_wiki") {
24
+ try {
25
+ const data = await backend.listWiki({ scope })
26
+ for (const item of data?.items ?? []) if (item.wiki_id) ids.push(item.wiki_id)
27
+ } catch (error) {
28
+ log.debug("wiki discovery failed", { error: errorMessage(error) })
29
+ }
30
+ }
31
+ if (!assetType || assetType === "code_graph") {
32
+ try {
33
+ const data = await backend.listCodeGraph({ scope })
34
+ for (const item of data?.items ?? []) if (item.code_graph_id) ids.push(item.code_graph_id)
35
+ } catch (error) {
36
+ log.debug("code-graph discovery failed", { error: errorMessage(error) })
37
+ }
38
+ }
39
+ return [...new Set(ids)]
40
+ }
41
+
42
+ const DEFAULT_KNOWLEDGE_CHARS = 6000
43
+ const DEFAULT_ASSET_CHARS = 3000
44
+
45
+ /**
46
+ * Search knowledge assets and render the results as fenced data.
47
+ *
48
+ * The total output is bounded, not just each page: three assets at the old
49
+ * per-page cap could add 9,000 characters to every recall. Assets are read in
50
+ * order until the budget is spent, so the caller gets whole early results
51
+ * instead of three equally truncated ones.
52
+ */
53
+ export async function searchKnowledge(backend, ids, query, scoped, options = {}) {
54
+ const maxChars = Number.isFinite(options.maxChars) && options.maxChars > 0 ? Math.floor(options.maxChars) : DEFAULT_KNOWLEDGE_CHARS
55
+ const perAssetChars =
56
+ Number.isFinite(options.perAssetChars) && options.perAssetChars > 0
57
+ ? Math.min(Math.floor(options.perAssetChars), maxChars)
58
+ : Math.min(DEFAULT_ASSET_CHARS, maxChars)
59
+ const blocks = []
60
+ let used = 0
61
+ for (const knowledgeID of ids) {
62
+ const remaining = maxChars - used
63
+ const source = neutralizeFenceClosers(oneLine(knowledgeID, 80))
64
+ // The fence and the separator are part of the budget; a page must not push
65
+ // the block past maxChars when the caller asked for a bound.
66
+ const fenceChars = `<knowledge source="${source}">\n\n</knowledge>`.length
67
+ if (remaining < fenceChars + 122) break
68
+ const bodyRoom = Math.max(120, Math.min(perAssetChars, remaining - fenceChars - 2))
69
+ try {
70
+ const result = await backend.knowledgeCall({
71
+ knowledgeID: String(knowledgeID),
72
+ toolName: "search",
73
+ params: { query, limit: 5 },
74
+ scope: scoped,
75
+ })
76
+ const rendered = typeof result === "string" ? result : JSON.stringify(result, null, 2)
77
+ // Truncate first, then neutralize: the closing tag is appended after,
78
+ // so neither a long page nor a crafted one can escape the fence.
79
+ const body = neutralizeFenceClosers(rendered.slice(0, bodyRoom))
80
+ const block = `<knowledge source="${source}">\n${body}\n</knowledge>`
81
+ blocks.push(block)
82
+ used += block.length + 2
83
+ } catch (error) {
84
+ const block = neutralizeFenceClosers(`(knowledge ${oneLine(knowledgeID, 60)} failed: ${errorMessage(error)})`)
85
+ blocks.push(block)
86
+ used += block.length + 2
87
+ }
88
+ }
89
+ return blocks.join("\n\n")
90
+ }