@joe-broadhead/opencode-mem 0.8.4 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CHANGELOG.md +455 -0
  2. package/README.md +114 -61
  3. package/docs/agent-protocol.md +67 -177
  4. package/package.json +1351 -98
  5. package/skills/mem.pages/SKILL.md +143 -0
  6. package/skills/mem.sharing/SKILL.md +82 -0
  7. package/skills/mem.sources/SKILL.md +93 -0
  8. package/skills/mem.usage/SKILL.md +118 -492
  9. package/src/backend/records.js +285 -0
  10. package/src/bootstrap.js +49 -45
  11. package/src/call-shapes.js +41 -10
  12. package/src/collector-lock.js +162 -0
  13. package/src/collector-service.js +441 -0
  14. package/src/collector.js +381 -0
  15. package/src/commands.js +3 -3
  16. package/src/config.js +12 -1
  17. package/src/curation.js +1 -1
  18. package/src/permissions.js +18 -2
  19. package/src/protocol.js +55 -24
  20. package/src/recall.js +104 -27
  21. package/src/records-store.js +280 -120
  22. package/src/records.js +63 -11
  23. package/src/runtime.js +23 -14
  24. package/src/skills.js +44 -18
  25. package/src/sources.js +58 -0
  26. package/src/spaces.js +93 -10
  27. package/src/tools/aliases.js +163 -0
  28. package/src/tools/commitments.js +12 -9
  29. package/src/tools/index.js +88 -19
  30. package/src/tools/ingest.js +277 -0
  31. package/src/tools/knowledge.js +6 -1
  32. package/src/tools/memory.js +640 -210
  33. package/src/tools/page.js +411 -0
  34. package/src/tools/recall.js +363 -112
  35. package/src/tools/records-kit.js +142 -0
  36. package/src/tools/share.js +1012 -0
  37. package/src/tools/shared.js +8 -1
  38. package/src/tools/spaces.js +431 -143
  39. package/src/tools/status.js +87 -58
  40. package/src/tools/wiki.js +2 -2
  41. package/src/views/contracts.js +273 -3
  42. package/src/views/records-reads.js +675 -0
  43. package/src/views/rpc.js +17 -100
  44. package/src/views/shape.js +61 -0
  45. package/views/commitments.html +4 -0
  46. package/views/graph.html +753 -0
  47. package/views/home.html +11 -5
  48. package/views/journal.html +621 -0
  49. package/views/memory.html +7 -1
  50. package/views/pages.html +636 -0
  51. package/views/review.html +406 -0
  52. package/views/today.html +309 -0
@@ -1,18 +1,19 @@
1
1
  import { isCheckpointMessage, renderRecallBlock } from "../recall.js"
2
- import { addWithheld, partitionAtoms, partitionTurns } from "../curation.js"
3
- import { errorMessage } from "../util.js"
2
+ import { addWithheld, isInstructionShaped, isStatusShaped, partitionAtoms, partitionTurns, stripInstructions } from "../curation.js"
3
+ import { errorMessage, neutralizeFenceClosers, oneLine } from "../util.js"
4
4
  import { discoverKnowledge, searchKnowledge } from "./knowledge.js"
5
5
  import { TASK_INPUT } from "./shared.js"
6
+ import { MEMORY_TYPES, PAGE_TYPES, callRecords, grantedSpace, noteRecordSpace } from "./records-kit.js"
7
+ import { isRecordsUnavailable } from "../backend/records.js"
6
8
 
7
- /** Upper bound on atoms per recall; the character budget still applies. */
9
+ /** Upper bound on hits per layer; the character budget still applies. */
8
10
  const MAX_LIMIT = 25
9
- /** Raw turns rendered per recall. Atoms carry the distilled answer. */
11
+ /** Raw turns rendered per recall. Records and atoms carry the distilled answer. */
10
12
  const MAX_TURNS = 4
11
- /**
12
- * Upstream L1 types a caller may restrict a search to. `work_task` is absent on
13
- * purpose: task descriptions are withheld from recall (see curation.js).
14
- */
15
- const ATOM_TYPES = Object.freeze(["persona", "episodic", "instruction", "work_fact", "work_method", "work_artifact"])
13
+ /** Where recall looks (contract §4.1). */
14
+ export const RECALL_IN = Object.freeze(["all", "memories", "pages", "transcripts", "documents"])
15
+ /** Record types `types` filters on: the kinds remember writes, page types, and commitments. */
16
+ export const RECORD_TYPES = Object.freeze([...new Set([...MEMORY_TYPES, ...PAGE_TYPES, "commitment"])])
16
17
 
17
18
  /** Whether upstream returned a task field for this row (L1 hits do, L0 search hits do not). */
18
19
  function hasTaskField(row) {
@@ -26,7 +27,51 @@ function rowTask(row) {
26
27
  }
27
28
 
28
29
  /**
29
- * `mem_recall`: pull-based retrieval across memory, wiki, and code graph.
30
+ * The record hits recall may serve. Memories are curated at write, but a seed
31
+ * or an import is not, so a status- or instruction-shaped memory is withheld
32
+ * and counted as an atom would be. A page or commitment keeps its Status
33
+ * section (that is its home); a planted command in its text is removed and
34
+ * marked, and a hit that is nothing else is withheld.
35
+ */
36
+ export function partitionRecords(hits) {
37
+ const served = []
38
+ const withheld = { status: 0, instruction: 0, task: 0 }
39
+ for (const hit of hits ?? []) {
40
+ const text = `${hit?.title ?? ""}\n${hit?.snippet ?? ""}`
41
+ if (hit?.kind === "memory" && isStatusShaped(text)) {
42
+ withheld.status += 1
43
+ continue
44
+ }
45
+ if (!isInstructionShaped(text)) {
46
+ served.push(hit)
47
+ continue
48
+ }
49
+ if (hit?.kind === "memory") {
50
+ withheld.instruction += 1
51
+ continue
52
+ }
53
+ const snippet = stripInstructions(hit.snippet ?? "")
54
+ const title = stripInstructions(hit.title ?? "")
55
+ if (!snippet.text && !title.text) {
56
+ withheld.instruction += 1
57
+ continue
58
+ }
59
+ served.push({ ...hit, title: title.text || hit.path || hit.id, snippet: snippet.text })
60
+ }
61
+ return { served, withheld }
62
+ }
63
+
64
+ /** Lists as given, or a comma-separated string, trimmed and de-duplicated. */
65
+ function listOf(value) {
66
+ const raw = Array.isArray(value) ? value : typeof value === "string" ? value.split(",") : []
67
+ return [...new Set(raw.map((entry) => String(entry ?? "").trim()).filter(Boolean))]
68
+ }
69
+
70
+ /**
71
+ * `mem_recall`: pull-based retrieval (contract §4.1). The records service
72
+ * first (memories, pages, commitments, by full text), then the core's atoms
73
+ * and turns as before, then documents through the hub's wiki search. One line
74
+ * per hit, with its id, inside the `<recalled_memory>` fence.
30
75
  *
31
76
  * @param {any} editor
32
77
  * @param {ReturnType<import("./shared.js").createToolkit>} toolkit
@@ -34,20 +79,67 @@ function rowTask(row) {
34
79
  export function registerRecall(editor, toolkit) {
35
80
  const { backend, config, log, control, text, fail, sharedScope, resolveReadSpace, displayScope, noteLocations } = toolkit
36
81
 
82
+ // Resolve a kind only for recall, never for writes. A bound shared space wins;
83
+ // otherwise a single accessible shared space is unambiguous. Control still
84
+ // resolves and authorizes the selected id, without binding the session.
85
+ async function recallSpace(input, context, visibility) {
86
+ const requested = String(input?.space ?? "").trim()
87
+ if (requested === "personal") {
88
+ if (visibility === "team") return { invalid: "visibility=team cannot search the personal space." }
89
+ if (!control) return { error: "control plane not configured; cannot select your personal space" }
90
+ return resolveReadSpace({}, context, "private")
91
+ }
92
+ if (requested !== "shared" && (requested || visibility !== "team" || !control)) {
93
+ return resolveReadSpace(input, context, visibility)
94
+ }
95
+ if (visibility === "private") return { invalid: "visibility=private cannot search a shared space." }
96
+ if (!control) return { error: "control plane not configured; cannot select a shared space" }
97
+ try {
98
+ const active = await toolkit.runtime.sessionContext(context?.sessionID, context?.agent)
99
+ if (active?.space_kind === "shared") return { context: active }
100
+ const spaces = (await control.listSpaces({ userId: config.user })).filter((space) => space.kind === "shared")
101
+ if (spaces.length !== 1) return { invalid: spaces.length ? "More than one shared space is available; ask which space to search and pass its id." : "No shared space is available to search; do not infer what it contains." }
102
+ const selected = await toolkit.runtime.resolveContext({ space: spaces[0].space_id, agent: context?.agent })
103
+ if (!selected || selected.space_kind !== "shared" || selected.space_id !== spaces[0].space_id) return { error: "the control plane did not confirm the selected shared space" }
104
+ return { context: selected }
105
+ } catch (error) {
106
+ return { error: errorMessage(error) }
107
+ }
108
+ }
109
+
37
110
  /**
38
111
  * Reads describe the whole space, not one session. L0 writes from
39
- * `mem_remember` and capture live under synthetic session ids, so scoping a
40
- * recall query by the current session hides the very memories just written.
112
+ * capture live under synthetic session ids, so scoping a recall query by
113
+ * the current session hides the very turns just written.
41
114
  */
42
115
  function readScope(scoped) {
43
116
  const { session_id, ...rest } = scoped
44
117
  return rest
45
118
  }
46
119
 
120
+ /**
121
+ * Label record hits by the task they were written under: `task` on the hit
122
+ * when the service sends it, else read once with `records.get`. A failed
123
+ * read labels nothing as the task's, so the hits stay, as scope=space.
124
+ */
125
+ async function labelRecordTasks(hits, key, spaceContext, scope) {
126
+ const tasks = new Map(hits.filter((hit) => Object.hasOwn(hit, "task")).map((hit) => [hit.id, hit.task ?? null]))
127
+ const unknown = hits.filter((hit) => !tasks.has(hit.id)).map((hit) => hit.id)
128
+ if (unknown.length > 0) {
129
+ try {
130
+ const data = await callRecords(toolkit, "records.get", { ids: unknown.slice(0, 50) }, spaceContext, scope)
131
+ for (const record of data?.records ?? []) tasks.set(record.id, record.task ?? null)
132
+ } catch (error) {
133
+ log.debug("record tasks unread", { error: errorMessage(error) })
134
+ }
135
+ }
136
+ return hits.map((hit) => ({ ...hit, _bucket: tasks.get(hit.id) === key ? "task" : "space" }))
137
+ }
138
+
47
139
  editor.add({
48
140
  name: "recall",
49
141
  description:
50
- "Search durable memory (L1 atoms and L0 turns of one space; optionally wiki and code graph) before re-deriving context: when the user refers to past work, decisions, constraints, or preferences, when you start work in an area with history, or before asking something already answered. With a task it searches that task first and widens to the whole space only on a miss; hits are labelled scope=task or scope=space. It cannot see the L2 scenarios or L3 persona. Task descriptions, status, and instruction-shaped records are withheld and counted. Results may be stale: verify before acting.",
142
+ 'Search memory: remembered memories, pages, commitments, past conversations and documents. Call it when past context is needed. One recall answers most questions; a miss is an answer, not a reason to search pages, wiki, spaces or commitments again. If recall answers the question, answer it; do not double-check pages, wiki or spaces unless the person asks or results conflict. Search another space only when the person names it. Before deciding if an action is allowed, including sharing, recall the recorded rule rather than relying on a default. Never present a rule, date, decision, exception or fact as recorded or in force unless recall or another tool returned it this session. If no rule is found, say so; do not deny on an invented rule or missing permission. General advice must be labelled as such. For the person\'s own private memory pass visibility "private"; otherwise recall searches the session\'s space, which may be shared. Never treat another person\'s private memory as the caller\'s. The dashboard page and the Due list are already in context. Records come first, then atoms and past turns, then up to two document indexes; in narrows the search. Read a named document with mem_wiki when its full text is needed. Every hit is one line with its id or path, kind/type, status, date, text and links. Superseded and retired records appear only with status "all". Status-shaped memories and instruction-shaped text are withheld and counted. Check any unmatched records shown; if none answers, say it is not recorded. Report incomplete searches, never claim absence. Results may be stale: verify before acting. Recall only reads; it never stores anything.',
51
143
  input: {
52
144
  type: "object",
53
145
  properties: {
@@ -60,25 +152,28 @@ export function registerRecall(editor, toolkit) {
60
152
  visibility: {
61
153
  type: "string",
62
154
  enum: ["auto", "private", "team"],
63
- description:
64
- "auto (default): the session's space, a bound shared space or else personal; private: your personal space; team: a shared space, bound or passed with space",
65
- },
66
- limit: {
67
- type: "number",
68
- description: `Atoms to return (default ${config.recallLimit}, max ${MAX_LIMIT}); up to ${MAX_TURNS} turns are added; the character budget applies`,
155
+ description: "auto (default): the session's space, a bound shared space or else personal; private: the caller's own personal space; team: the bound shared space or the only accessible shared space, otherwise asks for an id. Use private for the person's own private memory, not another person's.",
69
156
  },
70
- type: {
157
+ in: {
71
158
  type: "string",
72
- enum: [...ATOM_TYPES],
73
- description: "Only atoms of this upstream L1 type; turns are then not searched",
159
+ enum: [...RECALL_IN],
160
+ description: "all (default): records, atoms, turns and documents; memories: remembered memories and atoms; pages: pages only; transcripts: past turns only; documents: uploaded files and repositories (the hub's wikis and code graphs)",
161
+ },
162
+ types: {
163
+ type: "array",
164
+ items: { type: "string", enum: [...RECORD_TYPES] },
165
+ description: "Filter records strictly by these types. Memory types also search legacy atoms without kind filtering, labelled by their core type; turns are excluded. Page and commitment types do not search atoms.",
74
166
  },
75
- sources: {
167
+ status: {
76
168
  type: "string",
77
- enum: ["memory", "knowledge", "all"],
78
- description: "memory (default) searches stored memories; knowledge searches wiki/code-graph; all searches both",
169
+ enum: ["current", "all"],
170
+ description: 'current (default): current and contested records; all: superseded and retired ones too, for "what was it before"',
79
171
  },
80
- knowledge_id: { type: "string", description: "Wiki or code-graph asset id; discovered automatically when omitted" },
81
- space: { type: "string", description: "Space id to search (control plane); does not rebind the session" },
172
+ limit: {
173
+ type: "number",
174
+ description: `Hits per layer (default ${config.recallLimit}, max ${MAX_LIMIT}); up to ${MAX_TURNS} turns are added; the character budget applies`,
175
+ },
176
+ space: { type: "string", description: 'Space id, "personal" for your own personal space, or "shared" for the bound or only accessible shared space. Ambiguous shared selection asks for an id. Defaults to the session space; never rebinds.' },
82
177
  },
83
178
  required: ["query"],
84
179
  additionalProperties: false,
@@ -88,57 +183,102 @@ export function registerRecall(editor, toolkit) {
88
183
  const query = String(input?.query ?? "").trim()
89
184
  if (!query) return text("nothing to recall: query is empty")
90
185
  const limit = Math.max(1, Math.min(Math.floor(Number(input?.limit)) || config.recallLimit, MAX_LIMIT))
186
+ // The session's space (personal, or a shared one bound to it), or the
187
+ // space named; pre-0.9 callers' visibility still selects as it did.
91
188
  const visibility = ["auto", "private", "team"].includes(String(input?.visibility)) ? String(input.visibility) : "auto"
92
- const sources = input?.sources ?? "memory"
93
- const type = input?.type === undefined || input?.type === null || input?.type === "" ? undefined : String(input.type)
94
- if (type && !ATOM_TYPES.includes(type)) {
95
- return text(
96
- type === "work_task"
97
- ? "work_task atoms are task descriptions; recall withholds them. Read the tracker for tasks."
98
- : `unknown atom type ${type}; use one of ${ATOM_TYPES.join(", ")}.`,
99
- )
189
+ const where = input?.in === undefined || input?.in === null || input?.in === "" ? "all" : String(input.in)
190
+ if (!RECALL_IN.includes(where)) return text(`in must be one of ${RECALL_IN.join(", ")}.`)
191
+ const types = listOf(input?.types)
192
+ const badTypes = types.filter((value) => !RECORD_TYPES.includes(value))
193
+ if (badTypes.includes("work_task")) return text("work_task atoms are task descriptions; recall withholds them. Read the tracker or mem_commitments for things to do.")
194
+ if (badTypes.length > 0) return text(`unknown type ${badTypes.join(", ")}; types are ${RECORD_TYPES.join(", ")}.`)
195
+ const status = input?.status === "all" ? "all" : input?.status === undefined || input?.status === null || input?.status === "" || input?.status === "current" ? "current" : null
196
+ if (!status) return text('status is "current" (default) or "all".')
197
+
198
+ // Which layers this call reads.
199
+ const layers = {
200
+ records: ["all", "memories", "pages"].includes(where),
201
+ atoms: (types.length === 0 || types.some((type) => MEMORY_TYPES.includes(type))) && ["all", "memories"].includes(where),
202
+ turns: types.length === 0 && ["all", "transcripts"].includes(where),
203
+ documents: where === "documents" || (where === "all" && types.length === 0),
100
204
  }
205
+ const recordKinds = where === "memories" ? ["memory"] : where === "pages" ? ["page"] : ["memory", "page", "commitment"]
101
206
 
102
207
  const task = toolkit.taskFor(input, context)
103
208
  if (task.invalid) return text(task.invalid)
104
209
 
105
- const resolved = await resolveReadSpace(input, context, visibility)
210
+ // A space the person reaches only through a page grant has no role, so
211
+ // control refuses its context; its records are read through the
212
+ // gateway, which resolves the grant. The core is not readable there.
213
+ const readSpace = await recallSpace(input, context, visibility)
214
+ // Private means the caller's own space, never the page-grant fallback
215
+ // for some other space, even when that page would otherwise be readable.
216
+ const resolved = visibility === "private" || ["personal", "shared"].includes(String(input?.space ?? "").trim()) ? readSpace : grantedSpace(input, readSpace)
106
217
  if (resolved.invalid) return text(resolved.invalid)
107
218
  if (resolved.error) return fail(resolved.error)
108
219
  const spaceContext = resolved.context
220
+ if (resolved.granted) {
221
+ layers.atoms = false
222
+ layers.turns = false
223
+ layers.documents = false
224
+ }
109
225
  const scoped = spaceContext
110
226
  ? toolkit.scopeFor(spaceContext, { agent: context?.agent })
111
227
  : toolkit.scope({ agent: context?.agent })
112
228
 
113
229
  try {
114
- // Over-fetch atoms: curation can withhold part of a page, and the
115
- // caller asked for `limit` servable atoms, not `limit` candidates.
116
230
  const scope = readScope(scoped)
117
231
  const key = task.key
232
+
233
+ /** Records: full text over memories, pages and commitments, space-wide. */
234
+ const recordSearch = layers.records
235
+ ? callRecords(
236
+ toolkit,
237
+ "records.search",
238
+ { query, kinds: recordKinds, ...(types.length > 0 ? { types } : {}), status, limit: Math.min(50, limit * 2) },
239
+ spaceContext,
240
+ scope,
241
+ ).then(
242
+ (data) => ({ hits: Array.isArray(data?.hits) ? data.hits : [], failed: false }),
243
+ (error) => {
244
+ log.debug("records search failed", { error: errorMessage(error) })
245
+ // A server without records still permits a core search, but
246
+ // its result must be labelled incomplete.
247
+ if (isRecordsUnavailable(error)) return { hits: [], failed: false, unavailable: "server" }
248
+ if (error?.code === "records_missing") return { hits: [], failed: false, unavailable: "build" }
249
+ // Older gateways have no records route. A bare 404 at the
250
+ // search endpoint is not a search result or a missing record.
251
+ if (error?.status === 404 && !error?.code) return { hits: [], failed: true, missingEndpoint: true }
252
+ return { hits: [], failed: true }
253
+ },
254
+ )
255
+ : Promise.resolve({ hits: [], failed: false })
256
+
118
257
  async function searchIn(searchScope) {
119
- let failed = false
258
+ const failures = []
120
259
  const [atomPage, turnPage] = await Promise.all([
121
- backend.searchAtoms({ query, limit: Math.min(MAX_LIMIT, limit * 2), scope: searchScope, type }).catch(() => {
122
- failed = true
123
- return null
124
- }),
125
- type
126
- ? Promise.resolve(null)
127
- : backend.searchConversations({ query, limit: Math.min(MAX_TURNS * 2, limit * 2), scope: searchScope }).catch(() => {
128
- failed = true
260
+ layers.atoms
261
+ ? backend.searchAtoms({ query, limit: Math.min(MAX_LIMIT, limit * 2), scope: searchScope }).catch(() => {
262
+ failures.push("atoms")
129
263
  return null
130
- }),
264
+ })
265
+ : Promise.resolve(null),
266
+ layers.turns
267
+ ? backend.searchConversations({ query, limit: Math.min(MAX_TURNS * 2, limit * 2), scope: searchScope }).catch(() => {
268
+ failures.push("past turns")
269
+ return null
270
+ })
271
+ : Promise.resolve(null),
131
272
  ])
132
- // Remember where each hit was written, so mem_forget can address the
273
+ // Remember where each hit was written, so a forget can address the
133
274
  // delete to the row's own agent and session.
134
275
  noteLocations(spaceContext, "l1", atomPage?.items, { agent_id: scope.agent_id })
135
276
  noteLocations(spaceContext, "l0", turnPage?.messages, { agent_id: scope.agent_id })
136
277
  // A hit that carries its task is labelled by it, so a backend that
137
278
  // ignored the header cannot make another task's record look like this
138
- // one's. Upstream L0 search hits carry no task field at all (only id,
139
- // role, content, score, timestamp); for those the request decides:
140
- // upstream filters a search sent with x-tdai-task-id to that task, so
141
- // its hits are the task's, and a widened search's hits are not.
279
+ // one's. Upstream L0 search hits carry no task field at all; for those
280
+ // the request decides: upstream filters a search sent with
281
+ // x-tdai-task-id to that task, so its hits are the task's.
142
282
  const filtered = Boolean(searchScope.task_id)
143
283
  const bucketOf = (row) => {
144
284
  if (!hasTaskField(row)) return filtered ? "task" : "space"
@@ -151,35 +291,65 @@ export function registerRecall(editor, toolkit) {
151
291
  atoms: atomPart.served.map(label),
152
292
  conversations: turnPart.served.map(label),
153
293
  withheld: addWithheld(atomPart.withheld, turnPart.withheld),
154
- failed,
294
+ failures,
155
295
  }
156
296
  }
157
297
 
158
- let result = { atoms: [], conversations: [], withheld: addWithheld(), failed: false }
159
- let scopeNote
160
- if (sources !== "knowledge") {
298
+ let result = { atoms: [], conversations: [], withheld: addWithheld(), failures: [] }
299
+ let taskHit = false
300
+ if (layers.atoms || layers.turns) {
161
301
  if (key) {
162
302
  const first = await searchIn({ ...scope, task_id: key })
163
303
  const rows = [...first.atoms, ...first.conversations]
164
304
  if (rows.some((row) => row._bucket === "task")) {
165
305
  result = first
166
- scopeNote = rows.some((row) => row._bucket === "space")
167
- ? `Searched task "${key}". scope=task hits belong to it; scope=space hits were recorded outside it and are still valid memory of this space.`
168
- : `Searched task "${key}"; every hit belongs to it.`
306
+ taskHit = true
169
307
  } else {
170
308
  result = await searchIn(scope)
171
- result.failed = result.failed || first.failed
172
- scopeNote =
173
- result.atoms.length + result.conversations.length > 0
174
- ? `No match in task "${key}"; widened to the whole space. scope=space hits were recorded outside this task and are still valid memory of this space: use them, and say where they came from.`
175
- : `Searched task "${key}", then the whole space.`
309
+ result.failures = [...new Set([...result.failures, ...first.failures])]
176
310
  }
177
311
  } else {
178
312
  result = await searchIn(scope)
179
- scopeNote = task.wholeSpace ? "Searched the whole space." : "No task declared; searched the whole space."
180
313
  }
181
314
  }
182
- const { atoms, conversations, failed, withheld } = result
315
+ const found = await recordSearch
316
+ // No grant reached a page either: the space is not readable, and the
317
+ // control plane's refusal says why.
318
+ if (resolved.granted && found.failed) return fail(resolved.refusal)
319
+ // Records are searched space-wide; with a task declared, the hits of
320
+ // that task are served first, as the core's are: the others only
321
+ // when the task has none, labelled scope=space.
322
+ let recordHits = found.hits
323
+ if (key && recordHits.length > 0) {
324
+ const labelled = await labelRecordTasks(recordHits, key, spaceContext, scope)
325
+ recordHits = labelled.some((hit) => hit._bucket === "task") ? labelled.filter((hit) => hit._bucket === "task") : labelled
326
+ if (labelled.some((hit) => hit._bucket === "task")) taskHit = true
327
+ }
328
+ let scopeNote
329
+ if (key) {
330
+ const rows = [...recordHits, ...result.atoms, ...result.conversations]
331
+ scopeNote = taskHit
332
+ ? rows.some((row) => row._bucket === "space")
333
+ ? `Searched task "${key}". scope=task hits belong to it; scope=space hits were recorded outside it and are still valid memory of this space.`
334
+ : `Searched task "${key}"; every hit belongs to it.`
335
+ : rows.length > 0
336
+ ? `No match in task "${key}"; widened to the whole space. scope=space hits were recorded outside this task and are still valid memory of this space: use them, and say where they came from.`
337
+ : `Searched task "${key}", then the whole space.`
338
+ } else if (layers.atoms || layers.turns || layers.records) {
339
+ scopeNote = task.wholeSpace ? "Searched the whole space." : "No task declared; searched the whole space."
340
+ }
341
+ const recordPart = partitionRecords(recordHits)
342
+ const records = recordPart.served
343
+ for (const record of records) noteRecordSpace(toolkit, record.id, spaceContext)
344
+ const { atoms, conversations } = result
345
+ const failures = [...result.failures, ...(found.failed || found.unavailable ? [found.missingEndpoint ? "records endpoint (404)" : "records"] : [])]
346
+ const failureDetail = found.unavailable
347
+ ? `${found.unavailable === "build" ? "This plugin build has no records client" : "Records are not enabled on this Memory server yet"}, so remembered memories, pages and commitments were not searched${layers.atoms && layers.turns && result.failures.length === 0 ? "; past turns and atoms were" : ""}.`
348
+ : undefined
349
+ const withheld = addWithheld(result.withheld, recordPart.withheld)
350
+ if (layers.records && types.length > 0) scopeNote = `Only records of type ${types.join(", ")} were searched; ${layers.atoms ? "legacy atoms were searched without kind filtering; turns were not searched" : "atoms and turns were not"}.`
351
+ if (layers.records && status === "all") scopeNote = `${scopeNote ? `${scopeNote} ` : ""}Superseded and retired records are included (status "all").`
352
+ if (resolved.granted) scopeNote = `${scopeNote ? `${scopeNote} ` : ""}This space is shared with you by page grant: only the pages granted were searched.`
183
353
 
184
354
  for (const atom of atoms) {
185
355
  if (atom?.asset_id) void backend.touchAsset({ assetID: atom.asset_id, scope: sharedScope(spaceContext) }).catch(() => {})
@@ -187,82 +357,156 @@ export function registerRecall(editor, toolkit) {
187
357
 
188
358
  // An empty result is ambiguous: nothing stored, nothing matching these
189
359
  // words, everything withheld, or a failed search. Say which one, so the
190
- // agent can stop instead of fanning out across spaces, grants, and the
191
- // wiki.
360
+ // agent can stop instead of fanning out across spaces and tools.
192
361
  const BROWSE_MAX_RECORDS = 12
193
362
  const BROWSE_ATOMS = 6
194
363
  const BROWSE_TURNS = 4
364
+ const BROWSE_RECORDS = 6
195
365
  let census
196
366
  let browse
197
- if (sources !== "knowledge" && atoms.length === 0 && conversations.length === 0) {
198
- const [atomsCount, conversationsCount] = await Promise.all([
199
- backend.countAtoms({ scope }).then((data) => Number(data?.total) || 0).catch(() => null),
200
- backend.countConversations({ scope }).then((data) => Number(data?.total) || 0).catch(() => null),
367
+ let unmatchedRecords = []
368
+ const memoryLayers = layers.records || layers.atoms || layers.turns
369
+ // A lexical records miss can coexist with a core hit on a nearby
370
+ // topic. In a very small store, show those records as unranked context
371
+ // rather than letting that hit suppress the records fallback entirely.
372
+ // The service applies the same space, role, kind and status filters.
373
+ if (layers.records && !found.failed && !found.unavailable && recordHits.length === 0 && atoms.length + conversations.length > 0) {
374
+ try {
375
+ const page = await callRecords(toolkit, "records.list", {
376
+ kind: recordKinds,
377
+ ...(types.length > 0 ? { type: types } : {}),
378
+ status,
379
+ limit: 4,
380
+ }, spaceContext, scope)
381
+ if (!page?.cursor && Array.isArray(page?.records) && page.records.length <= 3) {
382
+ unmatchedRecords = partitionRecords(page.records.map((record) => ({ ...record, snippet: record.body }))).served
383
+ for (const record of unmatchedRecords) noteRecordSpace(toolkit, record.id, spaceContext)
384
+ }
385
+ } catch (error) {
386
+ log.debug("records fallback unavailable", { error: errorMessage(error) })
387
+ }
388
+ }
389
+ if (memoryLayers && records.length === 0 && atoms.length === 0 && conversations.length === 0) {
390
+ const [atomsCount, conversationsCount, recordsCount] = await Promise.all([
391
+ layers.atoms ? backend.countAtoms({ scope }).then((data) => Number(data?.total) || 0).catch(() => null) : Promise.resolve(0),
392
+ layers.turns ? backend.countConversations({ scope }).then((data) => Number(data?.total) || 0).catch(() => null) : Promise.resolve(0),
393
+ // A page grant cannot read space.stats (it has no path, so the
394
+ // service answers 403). That refusal is not a failed search, and a
395
+ // count would tell the caller how much they cannot see.
396
+ layers.records && !found.unavailable && !found.missingEndpoint && !resolved.granted
397
+ ? callRecords(toolkit, "space.stats", {}, spaceContext, scope).then(
398
+ (data) => recordCount(data, recordKinds),
399
+ () => null,
400
+ )
401
+ : Promise.resolve(0),
201
402
  ])
403
+ if (atomsCount === null) failures.push("atom count")
404
+ if (conversationsCount === null) failures.push("past-turn count")
405
+ if (recordsCount === null) failures.push("records count")
202
406
  census = {
203
407
  atoms: atomsCount ?? 0,
204
408
  conversations: conversationsCount ?? 0,
205
- failed: failed || atomsCount === null || conversationsCount === null,
409
+ records: recordsCount ?? 0,
410
+ failed: failures.length > 0,
411
+ grant: Boolean(resolved.granted),
412
+ coreEmpty: layers.atoms && layers.turns && !result.failures.length && atomsCount === 0 && conversationsCount === 0,
206
413
  otherScopesSearched: !(control && spaceContext),
207
414
  }
208
415
 
209
- // A small scope can be described when a query misses: a question and a
210
- // record often share no word at all. Above the cap, report the count
211
- // and do not dump the store. Nothing here is scored for relevance, and
212
- // it stays inside the space that was searched.
213
- const totalRecords = census.atoms + census.conversations
416
+ // A small scope can be described when a query misses: a question and
417
+ // a record often share no word at all. Above the cap, report the
418
+ // count and do not dump the store.
419
+ const totalRecords = census.atoms + census.conversations + census.records
214
420
  if (!census.failed && totalRecords > 0 && totalRecords <= BROWSE_MAX_RECORDS) {
215
- // A backend older than the query endpoints simply has no browse;
216
- // that must not turn a miss into an error.
217
421
  const listRecent = (read) => Promise.resolve().then(read).catch(() => null)
218
- const [recentAtoms, recentTurns] = await Promise.all([
219
- listRecent(() => backend.queryAtoms({ scope, limit: BROWSE_ATOMS, type })),
220
- type ? Promise.resolve(null) : listRecent(() => backend.queryConversations({ scope, limit: BROWSE_TURNS })),
422
+ const [recentAtoms, recentTurns, recentRecords] = await Promise.all([
423
+ layers.atoms ? listRecent(() => backend.queryAtoms({ scope, limit: BROWSE_ATOMS })) : null,
424
+ layers.turns ? listRecent(() => backend.queryConversations({ scope, limit: BROWSE_TURNS })) : null,
425
+ layers.records && census.records > 0
426
+ ? listRecent(() => callRecords(toolkit, "records.list", { limit: BROWSE_RECORDS * 2, ...(recordKinds.length === 1 ? { kind: recordKinds[0] } : {}) }, spaceContext, scope))
427
+ : null,
221
428
  ])
222
429
  // The browse path reads the scope directly, so it applies the same
223
- // curation guards as the search path. Without this, a withheld
224
- // record returns through atomic/query and the filter is decorative.
430
+ // curation guards as the search path.
225
431
  noteLocations(spaceContext, "l1", recentAtoms?.items, { agent_id: scope.agent_id })
226
432
  noteLocations(spaceContext, "l0", recentTurns?.messages, { agent_id: scope.agent_id })
227
433
  const browseAtoms = partitionAtoms(recentAtoms?.items ?? []).served.slice(0, BROWSE_ATOMS)
228
- const browseConversations = partitionTurns(recentTurns?.messages ?? [], { isCheckpoint: isCheckpointMessage })
229
- .served.slice(0, BROWSE_TURNS)
230
- if (browseAtoms.length + browseConversations.length > 0) browse = { atoms: browseAtoms, conversations: browseConversations }
434
+ const browseConversations = partitionTurns(recentTurns?.messages ?? [], { isCheckpoint: isCheckpointMessage }).served.slice(0, BROWSE_TURNS)
435
+ const listed = (recentRecords?.records ?? []).filter((record) => recordKinds.includes(record.kind) && (types.length === 0 || types.includes(record.type)))
436
+ const browseRecords = partitionRecords(listed.map((record) => ({ ...record, snippet: record.body }))).served.slice(0, BROWSE_RECORDS)
437
+ if (browseAtoms.length + browseConversations.length + browseRecords.length > 0) {
438
+ browse = { atoms: browseAtoms, conversations: browseConversations, records: browseRecords }
439
+ }
231
440
  }
232
441
  }
233
442
 
234
- let block = renderRecallBlock({
235
- atoms,
236
- conversations,
237
- scope: scoped,
238
- scopeLabel: spaceContext ? displayScope(spaceContext, scoped) : undefined,
239
- maxChars: config.recallChars,
240
- maxAtoms: limit,
241
- maxTurns: MAX_TURNS,
242
- query,
243
- census,
244
- browse,
245
- withheld,
246
- scopeNote,
247
- })
443
+ const renderMemory = (maxChars) => memoryLayers
444
+ ? renderRecallBlock({
445
+ records,
446
+ atoms,
447
+ conversations,
448
+ scope: scoped,
449
+ scopeLabel: spaceContext ? displayScope(spaceContext, scoped) : undefined,
450
+ maxChars,
451
+ maxAtoms: limit,
452
+ maxRecords: limit,
453
+ maxTurns: MAX_TURNS,
454
+ query,
455
+ census,
456
+ browse,
457
+ withheld,
458
+ scopeNote,
459
+ failures,
460
+ failureDetail,
461
+ unmatchedRecords,
462
+ })
463
+ : ""
248
464
 
249
- if (sources === "knowledge" || sources === "all") {
465
+ const searched = []
466
+ if (layers.records && !found.failed && !found.unavailable) searched.push(recordKinds.map((kind) => kind === "memory" ? "memories" : `${kind}s`).join(", "))
467
+ if (layers.atoms && !result.failures.includes("atoms")) searched.push("atoms")
468
+ if (layers.turns && !result.failures.includes("past turns")) searched.push("past conversations")
469
+ let documentCoverage = ""
470
+ let documents = ""
471
+ let documentsEmpty = false
472
+ if (layers.documents) {
250
473
  const knowledgeScope = sharedScope(spaceContext)
251
- const ids = input?.knowledge_id
252
- ? [String(input.knowledge_id)]
253
- : await discoverKnowledge(backend, log, undefined, knowledgeScope)
254
- const knowledge =
474
+ const discoveryFailures = []
475
+ const unavailable = []
476
+ const ids = await discoverKnowledge(backend, log, undefined, knowledgeScope, { onFailure: (name, error) => (Number(error?.status) === 404 ? unavailable : discoveryFailures).push(name) })
477
+ let searchedDocuments = 0
478
+ let documentFailures = 0
479
+ documents =
255
480
  ids.length === 0
256
481
  ? ""
257
- : await searchKnowledge(backend, ids.slice(0, 2), query, knowledgeScope, {
258
- maxChars: Math.max(1500, Math.floor(config.recallChars * 0.5)),
259
- })
260
- block += knowledge ? `\n\n${knowledge}` : "\n\n(no accessible wiki assets found)"
482
+ : await searchKnowledge(backend, ids.slice(0, 2), query, knowledgeScope, {
483
+ maxChars: Math.max(1500, Math.floor(config.recallChars * (where === "documents" ? 1 : 0.5))),
484
+ onSearch: (_id, ok) => { if (ok) searchedDocuments += 1; else documentFailures += 1 },
485
+ })
486
+ if (!documents && where === "documents" && !discoveryFailures.length && !unavailable.length) documents = "(no accessible wiki assets found)"
487
+ documentCoverage = discoveryFailures.length || documentFailures
488
+ ? `Document search incomplete: ${[...discoveryFailures, ...(documentFailures ? ["document search"] : [])].join(", ")} failed; do not infer absence.`
489
+ : ids.length === 0
490
+ ? unavailable.length ? "Documents: no indexes found in available catalogs." : "Documents: no accessible wiki or code-graph indexes found."
491
+ : `Documents: searched ${searchedDocuments} of ${ids.length} accessible wiki or code-graph indexes.${searchedDocuments < ids.length ? " For a named document not covered, use mem_wiki." : ""}`
492
+ if (unavailable.length) documentCoverage += ` ${unavailable.join(", ")} unavailable (404).`
493
+ documentsEmpty = ids.length === 0 && discoveryFailures.length === 0 && unavailable.length === 0
261
494
  }
262
495
 
496
+ const label = neutralizeFenceClosers(oneLine(spaceContext ? displayScope(spaceContext, scoped) : `${scoped.team_id}/${scoped.agent_id}`, 200))
497
+ const coverage = `Searched in ${label}: ${searched.length ? searched.join(", ") : "no memory layers"}. ${documentCoverage}`.trim()
498
+ const empty = where === "all" && types.length === 0 && !resolved.granted && census && !census.failed && !withheld.status && !withheld.instruction && !withheld.task && !withheld.redacted && census.records + census.atoms + census.conversations === 0 && documentsEmpty
499
+ const memoryEmpty = where === "all" && types.length === 0 && !resolved.granted && census && !census.failed && !withheld.status && !withheld.instruction && !withheld.task && !withheld.redacted && census.records + census.atoms + census.conversations === 0
500
+ const next = empty ? "Nothing is recorded in this space. Answer that it is not recorded, or continue the requested work. Do not search again in pages, wiki, spaces, commitments or other tools; a named document can be read directly when requested." : memoryEmpty ? "Nothing is recorded in the searched memory layers of this space. Report any document search limitation above; do not claim unsearched documents are empty. Do not repeat the search in pages, wiki, spaces or commitments." : ""
501
+ const memoryBudget = Math.max(100, config.recallChars - coverage.length - 1 - (documents ? documents.length + 2 : 0) - (next ? next.length + 1 : 0))
502
+ const memory = renderMemory(memoryBudget)
503
+ const block = [coverage, next, memory, documents].filter(Boolean).join("\n")
504
+
263
505
  log.debug("recall", {
264
506
  scope: displayScope(spaceContext, scoped),
507
+ in: where,
265
508
  task: Boolean(key),
509
+ records: records.length,
266
510
  atoms: atoms.length,
267
511
  conversations: conversations.length,
268
512
  withheld,
@@ -275,3 +519,10 @@ export function registerRecall(editor, toolkit) {
275
519
  },
276
520
  })
277
521
  }
522
+
523
+ /** The records `space.stats` counts (`records.by_kind[kind][status]`) for the kinds a recall searched. */
524
+ function recordCount(stats, kinds) {
525
+ const byKind = stats?.records?.by_kind
526
+ if (!byKind || typeof byKind !== "object") return 0
527
+ return kinds.reduce((sum, kind) => sum + Object.values(byKind[kind] ?? {}).reduce((total, count) => total + (Number(count) || 0), 0), 0)
528
+ }