@joe-broadhead/opencode-mem 0.8.5 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +429 -0
- package/README.md +114 -61
- package/docs/agent-protocol.md +67 -177
- package/package.json +1351 -98
- package/skills/mem.pages/SKILL.md +143 -0
- package/skills/mem.sharing/SKILL.md +82 -0
- package/skills/mem.sources/SKILL.md +93 -0
- package/skills/mem.usage/SKILL.md +118 -492
- package/src/backend/records.js +285 -0
- package/src/bootstrap.js +49 -45
- package/src/call-shapes.js +41 -10
- package/src/collector-lock.js +162 -0
- package/src/collector-service.js +441 -0
- package/src/collector.js +381 -0
- package/src/commands.js +3 -3
- package/src/config.js +12 -1
- package/src/curation.js +1 -1
- package/src/permissions.js +18 -2
- package/src/protocol.js +29 -20
- package/src/recall.js +104 -27
- package/src/records-store.js +280 -120
- package/src/records.js +63 -11
- package/src/runtime.js +23 -14
- package/src/skills.js +44 -18
- package/src/sources.js +58 -0
- package/src/spaces.js +93 -10
- package/src/tools/aliases.js +163 -0
- package/src/tools/commitments.js +12 -9
- package/src/tools/index.js +88 -19
- package/src/tools/ingest.js +277 -0
- package/src/tools/knowledge.js +6 -1
- package/src/tools/memory.js +640 -210
- package/src/tools/page.js +411 -0
- package/src/tools/recall.js +363 -112
- package/src/tools/records-kit.js +142 -0
- package/src/tools/share.js +1012 -0
- package/src/tools/shared.js +8 -1
- package/src/tools/spaces.js +431 -143
- package/src/tools/status.js +87 -58
- package/src/tools/wiki.js +2 -2
- package/src/views/contracts.js +273 -3
- package/src/views/records-reads.js +675 -0
- package/src/views/rpc.js +17 -100
- package/src/views/shape.js +61 -0
- package/views/commitments.html +4 -0
- package/views/graph.html +753 -0
- package/views/home.html +11 -5
- package/views/journal.html +621 -0
- package/views/memory.html +7 -1
- package/views/pages.html +636 -0
- package/views/review.html +406 -0
- package/views/today.html +309 -0
package/src/tools/recall.js
CHANGED
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
import { isCheckpointMessage, renderRecallBlock } from "../recall.js"
|
|
2
|
-
import { addWithheld, partitionAtoms, partitionTurns } from "../curation.js"
|
|
3
|
-
import { errorMessage } from "../util.js"
|
|
2
|
+
import { addWithheld, isInstructionShaped, isStatusShaped, partitionAtoms, partitionTurns, stripInstructions } from "../curation.js"
|
|
3
|
+
import { errorMessage, neutralizeFenceClosers, oneLine } from "../util.js"
|
|
4
4
|
import { discoverKnowledge, searchKnowledge } from "./knowledge.js"
|
|
5
5
|
import { TASK_INPUT } from "./shared.js"
|
|
6
|
+
import { MEMORY_TYPES, PAGE_TYPES, callRecords, grantedSpace, noteRecordSpace } from "./records-kit.js"
|
|
7
|
+
import { isRecordsUnavailable } from "../backend/records.js"
|
|
6
8
|
|
|
7
|
-
/** Upper bound on
|
|
9
|
+
/** Upper bound on hits per layer; the character budget still applies. */
|
|
8
10
|
const MAX_LIMIT = 25
|
|
9
|
-
/** Raw turns rendered per recall.
|
|
11
|
+
/** Raw turns rendered per recall. Records and atoms carry the distilled answer. */
|
|
10
12
|
const MAX_TURNS = 4
|
|
11
|
-
/**
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
const ATOM_TYPES = Object.freeze(["persona", "episodic", "instruction", "work_fact", "work_method", "work_artifact"])
|
|
13
|
+
/** Where recall looks (contract §4.1). */
|
|
14
|
+
export const RECALL_IN = Object.freeze(["all", "memories", "pages", "transcripts", "documents"])
|
|
15
|
+
/** Record types `types` filters on: the kinds remember writes, page types, and commitments. */
|
|
16
|
+
export const RECORD_TYPES = Object.freeze([...new Set([...MEMORY_TYPES, ...PAGE_TYPES, "commitment"])])
|
|
16
17
|
|
|
17
18
|
/** Whether upstream returned a task field for this row (L1 hits do, L0 search hits do not). */
|
|
18
19
|
function hasTaskField(row) {
|
|
@@ -26,7 +27,51 @@ function rowTask(row) {
|
|
|
26
27
|
}
|
|
27
28
|
|
|
28
29
|
/**
|
|
29
|
-
*
|
|
30
|
+
* The record hits recall may serve. Memories are curated at write, but a seed
|
|
31
|
+
* or an import is not, so a status- or instruction-shaped memory is withheld
|
|
32
|
+
* and counted as an atom would be. A page or commitment keeps its Status
|
|
33
|
+
* section (that is its home); a planted command in its text is removed and
|
|
34
|
+
* marked, and a hit that is nothing else is withheld.
|
|
35
|
+
*/
|
|
36
|
+
export function partitionRecords(hits) {
|
|
37
|
+
const served = []
|
|
38
|
+
const withheld = { status: 0, instruction: 0, task: 0 }
|
|
39
|
+
for (const hit of hits ?? []) {
|
|
40
|
+
const text = `${hit?.title ?? ""}\n${hit?.snippet ?? ""}`
|
|
41
|
+
if (hit?.kind === "memory" && isStatusShaped(text)) {
|
|
42
|
+
withheld.status += 1
|
|
43
|
+
continue
|
|
44
|
+
}
|
|
45
|
+
if (!isInstructionShaped(text)) {
|
|
46
|
+
served.push(hit)
|
|
47
|
+
continue
|
|
48
|
+
}
|
|
49
|
+
if (hit?.kind === "memory") {
|
|
50
|
+
withheld.instruction += 1
|
|
51
|
+
continue
|
|
52
|
+
}
|
|
53
|
+
const snippet = stripInstructions(hit.snippet ?? "")
|
|
54
|
+
const title = stripInstructions(hit.title ?? "")
|
|
55
|
+
if (!snippet.text && !title.text) {
|
|
56
|
+
withheld.instruction += 1
|
|
57
|
+
continue
|
|
58
|
+
}
|
|
59
|
+
served.push({ ...hit, title: title.text || hit.path || hit.id, snippet: snippet.text })
|
|
60
|
+
}
|
|
61
|
+
return { served, withheld }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Lists as given, or a comma-separated string, trimmed and de-duplicated. */
|
|
65
|
+
function listOf(value) {
|
|
66
|
+
const raw = Array.isArray(value) ? value : typeof value === "string" ? value.split(",") : []
|
|
67
|
+
return [...new Set(raw.map((entry) => String(entry ?? "").trim()).filter(Boolean))]
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* `mem_recall`: pull-based retrieval (contract §4.1). The records service
|
|
72
|
+
* first (memories, pages, commitments, by full text), then the core's atoms
|
|
73
|
+
* and turns as before, then documents through the hub's wiki search. One line
|
|
74
|
+
* per hit, with its id, inside the `<recalled_memory>` fence.
|
|
30
75
|
*
|
|
31
76
|
* @param {any} editor
|
|
32
77
|
* @param {ReturnType<import("./shared.js").createToolkit>} toolkit
|
|
@@ -34,20 +79,67 @@ function rowTask(row) {
|
|
|
34
79
|
export function registerRecall(editor, toolkit) {
|
|
35
80
|
const { backend, config, log, control, text, fail, sharedScope, resolveReadSpace, displayScope, noteLocations } = toolkit
|
|
36
81
|
|
|
82
|
+
// Resolve a kind only for recall, never for writes. A bound shared space wins;
|
|
83
|
+
// otherwise a single accessible shared space is unambiguous. Control still
|
|
84
|
+
// resolves and authorizes the selected id, without binding the session.
|
|
85
|
+
async function recallSpace(input, context, visibility) {
|
|
86
|
+
const requested = String(input?.space ?? "").trim()
|
|
87
|
+
if (requested === "personal") {
|
|
88
|
+
if (visibility === "team") return { invalid: "visibility=team cannot search the personal space." }
|
|
89
|
+
if (!control) return { error: "control plane not configured; cannot select your personal space" }
|
|
90
|
+
return resolveReadSpace({}, context, "private")
|
|
91
|
+
}
|
|
92
|
+
if (requested !== "shared" && (requested || visibility !== "team" || !control)) {
|
|
93
|
+
return resolveReadSpace(input, context, visibility)
|
|
94
|
+
}
|
|
95
|
+
if (visibility === "private") return { invalid: "visibility=private cannot search a shared space." }
|
|
96
|
+
if (!control) return { error: "control plane not configured; cannot select a shared space" }
|
|
97
|
+
try {
|
|
98
|
+
const active = await toolkit.runtime.sessionContext(context?.sessionID, context?.agent)
|
|
99
|
+
if (active?.space_kind === "shared") return { context: active }
|
|
100
|
+
const spaces = (await control.listSpaces({ userId: config.user })).filter((space) => space.kind === "shared")
|
|
101
|
+
if (spaces.length !== 1) return { invalid: spaces.length ? "More than one shared space is available; ask which space to search and pass its id." : "No shared space is available to search; do not infer what it contains." }
|
|
102
|
+
const selected = await toolkit.runtime.resolveContext({ space: spaces[0].space_id, agent: context?.agent })
|
|
103
|
+
if (!selected || selected.space_kind !== "shared" || selected.space_id !== spaces[0].space_id) return { error: "the control plane did not confirm the selected shared space" }
|
|
104
|
+
return { context: selected }
|
|
105
|
+
} catch (error) {
|
|
106
|
+
return { error: errorMessage(error) }
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
37
110
|
/**
|
|
38
111
|
* Reads describe the whole space, not one session. L0 writes from
|
|
39
|
-
*
|
|
40
|
-
*
|
|
112
|
+
* capture live under synthetic session ids, so scoping a recall query by
|
|
113
|
+
* the current session hides the very turns just written.
|
|
41
114
|
*/
|
|
42
115
|
function readScope(scoped) {
|
|
43
116
|
const { session_id, ...rest } = scoped
|
|
44
117
|
return rest
|
|
45
118
|
}
|
|
46
119
|
|
|
120
|
+
/**
|
|
121
|
+
* Label record hits by the task they were written under: `task` on the hit
|
|
122
|
+
* when the service sends it, else read once with `records.get`. A failed
|
|
123
|
+
* read labels nothing as the task's, so the hits stay, as scope=space.
|
|
124
|
+
*/
|
|
125
|
+
async function labelRecordTasks(hits, key, spaceContext, scope) {
|
|
126
|
+
const tasks = new Map(hits.filter((hit) => Object.hasOwn(hit, "task")).map((hit) => [hit.id, hit.task ?? null]))
|
|
127
|
+
const unknown = hits.filter((hit) => !tasks.has(hit.id)).map((hit) => hit.id)
|
|
128
|
+
if (unknown.length > 0) {
|
|
129
|
+
try {
|
|
130
|
+
const data = await callRecords(toolkit, "records.get", { ids: unknown.slice(0, 50) }, spaceContext, scope)
|
|
131
|
+
for (const record of data?.records ?? []) tasks.set(record.id, record.task ?? null)
|
|
132
|
+
} catch (error) {
|
|
133
|
+
log.debug("record tasks unread", { error: errorMessage(error) })
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return hits.map((hit) => ({ ...hit, _bucket: tasks.get(hit.id) === key ? "task" : "space" }))
|
|
137
|
+
}
|
|
138
|
+
|
|
47
139
|
editor.add({
|
|
48
140
|
name: "recall",
|
|
49
141
|
description:
|
|
50
|
-
|
|
142
|
+
'Search memory: remembered memories, pages, commitments, past conversations and documents. Call it when past context is needed. One recall answers most questions; a miss is an answer, not a reason to search pages, wiki, spaces or commitments again. If recall answers the question, answer it; do not double-check pages, wiki or spaces unless the person asks or results conflict. Search another space only when the person names it. Before deciding if an action is allowed, including sharing, recall the recorded rule rather than relying on a default. Never present a rule, date, decision, exception or fact as recorded or in force unless recall or another tool returned it this session. If no rule is found, say so; do not deny on an invented rule or missing permission. General advice must be labelled as such. For the person\'s own private memory pass visibility "private"; otherwise recall searches the session\'s space, which may be shared. Never treat another person\'s private memory as the caller\'s. The dashboard page and the Due list are already in context. Records come first, then atoms and past turns, then up to two document indexes; in narrows the search. Read a named document with mem_wiki when its full text is needed. Every hit is one line with its id or path, kind/type, status, date, text and links. Superseded and retired records appear only with status "all". Status-shaped memories and instruction-shaped text are withheld and counted. Check any unmatched records shown; if none answers, say it is not recorded. Report incomplete searches, never claim absence. Results may be stale: verify before acting. Recall only reads; it never stores anything.',
|
|
51
143
|
input: {
|
|
52
144
|
type: "object",
|
|
53
145
|
properties: {
|
|
@@ -60,25 +152,28 @@ export function registerRecall(editor, toolkit) {
|
|
|
60
152
|
visibility: {
|
|
61
153
|
type: "string",
|
|
62
154
|
enum: ["auto", "private", "team"],
|
|
63
|
-
description:
|
|
64
|
-
"auto (default): the session's space, a bound shared space or else personal; private: your personal space; team: a shared space, bound or passed with space",
|
|
65
|
-
},
|
|
66
|
-
limit: {
|
|
67
|
-
type: "number",
|
|
68
|
-
description: `Atoms to return (default ${config.recallLimit}, max ${MAX_LIMIT}); up to ${MAX_TURNS} turns are added; the character budget applies`,
|
|
155
|
+
description: "auto (default): the session's space, a bound shared space or else personal; private: the caller's own personal space; team: the bound shared space or the only accessible shared space, otherwise asks for an id. Use private for the person's own private memory, not another person's.",
|
|
69
156
|
},
|
|
70
|
-
|
|
157
|
+
in: {
|
|
71
158
|
type: "string",
|
|
72
|
-
enum: [...
|
|
73
|
-
description: "
|
|
159
|
+
enum: [...RECALL_IN],
|
|
160
|
+
description: "all (default): records, atoms, turns and documents; memories: remembered memories and atoms; pages: pages only; transcripts: past turns only; documents: uploaded files and repositories (the hub's wikis and code graphs)",
|
|
161
|
+
},
|
|
162
|
+
types: {
|
|
163
|
+
type: "array",
|
|
164
|
+
items: { type: "string", enum: [...RECORD_TYPES] },
|
|
165
|
+
description: "Filter records strictly by these types. Memory types also search legacy atoms without kind filtering, labelled by their core type; turns are excluded. Page and commitment types do not search atoms.",
|
|
74
166
|
},
|
|
75
|
-
|
|
167
|
+
status: {
|
|
76
168
|
type: "string",
|
|
77
|
-
enum: ["
|
|
78
|
-
description:
|
|
169
|
+
enum: ["current", "all"],
|
|
170
|
+
description: 'current (default): current and contested records; all: superseded and retired ones too, for "what was it before"',
|
|
79
171
|
},
|
|
80
|
-
|
|
81
|
-
|
|
172
|
+
limit: {
|
|
173
|
+
type: "number",
|
|
174
|
+
description: `Hits per layer (default ${config.recallLimit}, max ${MAX_LIMIT}); up to ${MAX_TURNS} turns are added; the character budget applies`,
|
|
175
|
+
},
|
|
176
|
+
space: { type: "string", description: 'Space id, "personal" for your own personal space, or "shared" for the bound or only accessible shared space. Ambiguous shared selection asks for an id. Defaults to the session space; never rebinds.' },
|
|
82
177
|
},
|
|
83
178
|
required: ["query"],
|
|
84
179
|
additionalProperties: false,
|
|
@@ -88,57 +183,102 @@ export function registerRecall(editor, toolkit) {
|
|
|
88
183
|
const query = String(input?.query ?? "").trim()
|
|
89
184
|
if (!query) return text("nothing to recall: query is empty")
|
|
90
185
|
const limit = Math.max(1, Math.min(Math.floor(Number(input?.limit)) || config.recallLimit, MAX_LIMIT))
|
|
186
|
+
// The session's space (personal, or a shared one bound to it), or the
|
|
187
|
+
// space named; pre-0.9 callers' visibility still selects as it did.
|
|
91
188
|
const visibility = ["auto", "private", "team"].includes(String(input?.visibility)) ? String(input.visibility) : "auto"
|
|
92
|
-
const
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
189
|
+
const where = input?.in === undefined || input?.in === null || input?.in === "" ? "all" : String(input.in)
|
|
190
|
+
if (!RECALL_IN.includes(where)) return text(`in must be one of ${RECALL_IN.join(", ")}.`)
|
|
191
|
+
const types = listOf(input?.types)
|
|
192
|
+
const badTypes = types.filter((value) => !RECORD_TYPES.includes(value))
|
|
193
|
+
if (badTypes.includes("work_task")) return text("work_task atoms are task descriptions; recall withholds them. Read the tracker or mem_commitments for things to do.")
|
|
194
|
+
if (badTypes.length > 0) return text(`unknown type ${badTypes.join(", ")}; types are ${RECORD_TYPES.join(", ")}.`)
|
|
195
|
+
const status = input?.status === "all" ? "all" : input?.status === undefined || input?.status === null || input?.status === "" || input?.status === "current" ? "current" : null
|
|
196
|
+
if (!status) return text('status is "current" (default) or "all".')
|
|
197
|
+
|
|
198
|
+
// Which layers this call reads.
|
|
199
|
+
const layers = {
|
|
200
|
+
records: ["all", "memories", "pages"].includes(where),
|
|
201
|
+
atoms: (types.length === 0 || types.some((type) => MEMORY_TYPES.includes(type))) && ["all", "memories"].includes(where),
|
|
202
|
+
turns: types.length === 0 && ["all", "transcripts"].includes(where),
|
|
203
|
+
documents: where === "documents" || (where === "all" && types.length === 0),
|
|
100
204
|
}
|
|
205
|
+
const recordKinds = where === "memories" ? ["memory"] : where === "pages" ? ["page"] : ["memory", "page", "commitment"]
|
|
101
206
|
|
|
102
207
|
const task = toolkit.taskFor(input, context)
|
|
103
208
|
if (task.invalid) return text(task.invalid)
|
|
104
209
|
|
|
105
|
-
|
|
210
|
+
// A space the person reaches only through a page grant has no role, so
|
|
211
|
+
// control refuses its context; its records are read through the
|
|
212
|
+
// gateway, which resolves the grant. The core is not readable there.
|
|
213
|
+
const readSpace = await recallSpace(input, context, visibility)
|
|
214
|
+
// Private means the caller's own space, never the page-grant fallback
|
|
215
|
+
// for some other space, even when that page would otherwise be readable.
|
|
216
|
+
const resolved = visibility === "private" || ["personal", "shared"].includes(String(input?.space ?? "").trim()) ? readSpace : grantedSpace(input, readSpace)
|
|
106
217
|
if (resolved.invalid) return text(resolved.invalid)
|
|
107
218
|
if (resolved.error) return fail(resolved.error)
|
|
108
219
|
const spaceContext = resolved.context
|
|
220
|
+
if (resolved.granted) {
|
|
221
|
+
layers.atoms = false
|
|
222
|
+
layers.turns = false
|
|
223
|
+
layers.documents = false
|
|
224
|
+
}
|
|
109
225
|
const scoped = spaceContext
|
|
110
226
|
? toolkit.scopeFor(spaceContext, { agent: context?.agent })
|
|
111
227
|
: toolkit.scope({ agent: context?.agent })
|
|
112
228
|
|
|
113
229
|
try {
|
|
114
|
-
// Over-fetch atoms: curation can withhold part of a page, and the
|
|
115
|
-
// caller asked for `limit` servable atoms, not `limit` candidates.
|
|
116
230
|
const scope = readScope(scoped)
|
|
117
231
|
const key = task.key
|
|
232
|
+
|
|
233
|
+
/** Records: full text over memories, pages and commitments, space-wide. */
|
|
234
|
+
const recordSearch = layers.records
|
|
235
|
+
? callRecords(
|
|
236
|
+
toolkit,
|
|
237
|
+
"records.search",
|
|
238
|
+
{ query, kinds: recordKinds, ...(types.length > 0 ? { types } : {}), status, limit: Math.min(50, limit * 2) },
|
|
239
|
+
spaceContext,
|
|
240
|
+
scope,
|
|
241
|
+
).then(
|
|
242
|
+
(data) => ({ hits: Array.isArray(data?.hits) ? data.hits : [], failed: false }),
|
|
243
|
+
(error) => {
|
|
244
|
+
log.debug("records search failed", { error: errorMessage(error) })
|
|
245
|
+
// A server without records still permits a core search, but
|
|
246
|
+
// its result must be labelled incomplete.
|
|
247
|
+
if (isRecordsUnavailable(error)) return { hits: [], failed: false, unavailable: "server" }
|
|
248
|
+
if (error?.code === "records_missing") return { hits: [], failed: false, unavailable: "build" }
|
|
249
|
+
// Older gateways have no records route. A bare 404 at the
|
|
250
|
+
// search endpoint is not a search result or a missing record.
|
|
251
|
+
if (error?.status === 404 && !error?.code) return { hits: [], failed: true, missingEndpoint: true }
|
|
252
|
+
return { hits: [], failed: true }
|
|
253
|
+
},
|
|
254
|
+
)
|
|
255
|
+
: Promise.resolve({ hits: [], failed: false })
|
|
256
|
+
|
|
118
257
|
async function searchIn(searchScope) {
|
|
119
|
-
|
|
258
|
+
const failures = []
|
|
120
259
|
const [atomPage, turnPage] = await Promise.all([
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
}),
|
|
125
|
-
type
|
|
126
|
-
? Promise.resolve(null)
|
|
127
|
-
: backend.searchConversations({ query, limit: Math.min(MAX_TURNS * 2, limit * 2), scope: searchScope }).catch(() => {
|
|
128
|
-
failed = true
|
|
260
|
+
layers.atoms
|
|
261
|
+
? backend.searchAtoms({ query, limit: Math.min(MAX_LIMIT, limit * 2), scope: searchScope }).catch(() => {
|
|
262
|
+
failures.push("atoms")
|
|
129
263
|
return null
|
|
130
|
-
})
|
|
264
|
+
})
|
|
265
|
+
: Promise.resolve(null),
|
|
266
|
+
layers.turns
|
|
267
|
+
? backend.searchConversations({ query, limit: Math.min(MAX_TURNS * 2, limit * 2), scope: searchScope }).catch(() => {
|
|
268
|
+
failures.push("past turns")
|
|
269
|
+
return null
|
|
270
|
+
})
|
|
271
|
+
: Promise.resolve(null),
|
|
131
272
|
])
|
|
132
|
-
// Remember where each hit was written, so
|
|
273
|
+
// Remember where each hit was written, so a forget can address the
|
|
133
274
|
// delete to the row's own agent and session.
|
|
134
275
|
noteLocations(spaceContext, "l1", atomPage?.items, { agent_id: scope.agent_id })
|
|
135
276
|
noteLocations(spaceContext, "l0", turnPage?.messages, { agent_id: scope.agent_id })
|
|
136
277
|
// A hit that carries its task is labelled by it, so a backend that
|
|
137
278
|
// ignored the header cannot make another task's record look like this
|
|
138
|
-
// one's. Upstream L0 search hits carry no task field at all
|
|
139
|
-
//
|
|
140
|
-
//
|
|
141
|
-
// its hits are the task's, and a widened search's hits are not.
|
|
279
|
+
// one's. Upstream L0 search hits carry no task field at all; for those
|
|
280
|
+
// the request decides: upstream filters a search sent with
|
|
281
|
+
// x-tdai-task-id to that task, so its hits are the task's.
|
|
142
282
|
const filtered = Boolean(searchScope.task_id)
|
|
143
283
|
const bucketOf = (row) => {
|
|
144
284
|
if (!hasTaskField(row)) return filtered ? "task" : "space"
|
|
@@ -151,35 +291,65 @@ export function registerRecall(editor, toolkit) {
|
|
|
151
291
|
atoms: atomPart.served.map(label),
|
|
152
292
|
conversations: turnPart.served.map(label),
|
|
153
293
|
withheld: addWithheld(atomPart.withheld, turnPart.withheld),
|
|
154
|
-
|
|
294
|
+
failures,
|
|
155
295
|
}
|
|
156
296
|
}
|
|
157
297
|
|
|
158
|
-
let result = { atoms: [], conversations: [], withheld: addWithheld(),
|
|
159
|
-
let
|
|
160
|
-
if (
|
|
298
|
+
let result = { atoms: [], conversations: [], withheld: addWithheld(), failures: [] }
|
|
299
|
+
let taskHit = false
|
|
300
|
+
if (layers.atoms || layers.turns) {
|
|
161
301
|
if (key) {
|
|
162
302
|
const first = await searchIn({ ...scope, task_id: key })
|
|
163
303
|
const rows = [...first.atoms, ...first.conversations]
|
|
164
304
|
if (rows.some((row) => row._bucket === "task")) {
|
|
165
305
|
result = first
|
|
166
|
-
|
|
167
|
-
? `Searched task "${key}". scope=task hits belong to it; scope=space hits were recorded outside it and are still valid memory of this space.`
|
|
168
|
-
: `Searched task "${key}"; every hit belongs to it.`
|
|
306
|
+
taskHit = true
|
|
169
307
|
} else {
|
|
170
308
|
result = await searchIn(scope)
|
|
171
|
-
result.
|
|
172
|
-
scopeNote =
|
|
173
|
-
result.atoms.length + result.conversations.length > 0
|
|
174
|
-
? `No match in task "${key}"; widened to the whole space. scope=space hits were recorded outside this task and are still valid memory of this space: use them, and say where they came from.`
|
|
175
|
-
: `Searched task "${key}", then the whole space.`
|
|
309
|
+
result.failures = [...new Set([...result.failures, ...first.failures])]
|
|
176
310
|
}
|
|
177
311
|
} else {
|
|
178
312
|
result = await searchIn(scope)
|
|
179
|
-
scopeNote = task.wholeSpace ? "Searched the whole space." : "No task declared; searched the whole space."
|
|
180
313
|
}
|
|
181
314
|
}
|
|
182
|
-
const
|
|
315
|
+
const found = await recordSearch
|
|
316
|
+
// No grant reached a page either: the space is not readable, and the
|
|
317
|
+
// control plane's refusal says why.
|
|
318
|
+
if (resolved.granted && found.failed) return fail(resolved.refusal)
|
|
319
|
+
// Records are searched space-wide; with a task declared, the hits of
|
|
320
|
+
// that task are served first, as the core's are: the others only
|
|
321
|
+
// when the task has none, labelled scope=space.
|
|
322
|
+
let recordHits = found.hits
|
|
323
|
+
if (key && recordHits.length > 0) {
|
|
324
|
+
const labelled = await labelRecordTasks(recordHits, key, spaceContext, scope)
|
|
325
|
+
recordHits = labelled.some((hit) => hit._bucket === "task") ? labelled.filter((hit) => hit._bucket === "task") : labelled
|
|
326
|
+
if (labelled.some((hit) => hit._bucket === "task")) taskHit = true
|
|
327
|
+
}
|
|
328
|
+
let scopeNote
|
|
329
|
+
if (key) {
|
|
330
|
+
const rows = [...recordHits, ...result.atoms, ...result.conversations]
|
|
331
|
+
scopeNote = taskHit
|
|
332
|
+
? rows.some((row) => row._bucket === "space")
|
|
333
|
+
? `Searched task "${key}". scope=task hits belong to it; scope=space hits were recorded outside it and are still valid memory of this space.`
|
|
334
|
+
: `Searched task "${key}"; every hit belongs to it.`
|
|
335
|
+
: rows.length > 0
|
|
336
|
+
? `No match in task "${key}"; widened to the whole space. scope=space hits were recorded outside this task and are still valid memory of this space: use them, and say where they came from.`
|
|
337
|
+
: `Searched task "${key}", then the whole space.`
|
|
338
|
+
} else if (layers.atoms || layers.turns || layers.records) {
|
|
339
|
+
scopeNote = task.wholeSpace ? "Searched the whole space." : "No task declared; searched the whole space."
|
|
340
|
+
}
|
|
341
|
+
const recordPart = partitionRecords(recordHits)
|
|
342
|
+
const records = recordPart.served
|
|
343
|
+
for (const record of records) noteRecordSpace(toolkit, record.id, spaceContext)
|
|
344
|
+
const { atoms, conversations } = result
|
|
345
|
+
const failures = [...result.failures, ...(found.failed || found.unavailable ? [found.missingEndpoint ? "records endpoint (404)" : "records"] : [])]
|
|
346
|
+
const failureDetail = found.unavailable
|
|
347
|
+
? `${found.unavailable === "build" ? "This plugin build has no records client" : "Records are not enabled on this Memory server yet"}, so remembered memories, pages and commitments were not searched${layers.atoms && layers.turns && result.failures.length === 0 ? "; past turns and atoms were" : ""}.`
|
|
348
|
+
: undefined
|
|
349
|
+
const withheld = addWithheld(result.withheld, recordPart.withheld)
|
|
350
|
+
if (layers.records && types.length > 0) scopeNote = `Only records of type ${types.join(", ")} were searched; ${layers.atoms ? "legacy atoms were searched without kind filtering; turns were not searched" : "atoms and turns were not"}.`
|
|
351
|
+
if (layers.records && status === "all") scopeNote = `${scopeNote ? `${scopeNote} ` : ""}Superseded and retired records are included (status "all").`
|
|
352
|
+
if (resolved.granted) scopeNote = `${scopeNote ? `${scopeNote} ` : ""}This space is shared with you by page grant: only the pages granted were searched.`
|
|
183
353
|
|
|
184
354
|
for (const atom of atoms) {
|
|
185
355
|
if (atom?.asset_id) void backend.touchAsset({ assetID: atom.asset_id, scope: sharedScope(spaceContext) }).catch(() => {})
|
|
@@ -187,82 +357,156 @@ export function registerRecall(editor, toolkit) {
|
|
|
187
357
|
|
|
188
358
|
// An empty result is ambiguous: nothing stored, nothing matching these
|
|
189
359
|
// words, everything withheld, or a failed search. Say which one, so the
|
|
190
|
-
// agent can stop instead of fanning out across spaces
|
|
191
|
-
// wiki.
|
|
360
|
+
// agent can stop instead of fanning out across spaces and tools.
|
|
192
361
|
const BROWSE_MAX_RECORDS = 12
|
|
193
362
|
const BROWSE_ATOMS = 6
|
|
194
363
|
const BROWSE_TURNS = 4
|
|
364
|
+
const BROWSE_RECORDS = 6
|
|
195
365
|
let census
|
|
196
366
|
let browse
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
367
|
+
let unmatchedRecords = []
|
|
368
|
+
const memoryLayers = layers.records || layers.atoms || layers.turns
|
|
369
|
+
// A lexical records miss can coexist with a core hit on a nearby
|
|
370
|
+
// topic. In a very small store, show those records as unranked context
|
|
371
|
+
// rather than letting that hit suppress the records fallback entirely.
|
|
372
|
+
// The service applies the same space, role, kind and status filters.
|
|
373
|
+
if (layers.records && !found.failed && !found.unavailable && recordHits.length === 0 && atoms.length + conversations.length > 0) {
|
|
374
|
+
try {
|
|
375
|
+
const page = await callRecords(toolkit, "records.list", {
|
|
376
|
+
kind: recordKinds,
|
|
377
|
+
...(types.length > 0 ? { type: types } : {}),
|
|
378
|
+
status,
|
|
379
|
+
limit: 4,
|
|
380
|
+
}, spaceContext, scope)
|
|
381
|
+
if (!page?.cursor && Array.isArray(page?.records) && page.records.length <= 3) {
|
|
382
|
+
unmatchedRecords = partitionRecords(page.records.map((record) => ({ ...record, snippet: record.body }))).served
|
|
383
|
+
for (const record of unmatchedRecords) noteRecordSpace(toolkit, record.id, spaceContext)
|
|
384
|
+
}
|
|
385
|
+
} catch (error) {
|
|
386
|
+
log.debug("records fallback unavailable", { error: errorMessage(error) })
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
if (memoryLayers && records.length === 0 && atoms.length === 0 && conversations.length === 0) {
|
|
390
|
+
const [atomsCount, conversationsCount, recordsCount] = await Promise.all([
|
|
391
|
+
layers.atoms ? backend.countAtoms({ scope }).then((data) => Number(data?.total) || 0).catch(() => null) : Promise.resolve(0),
|
|
392
|
+
layers.turns ? backend.countConversations({ scope }).then((data) => Number(data?.total) || 0).catch(() => null) : Promise.resolve(0),
|
|
393
|
+
// A page grant cannot read space.stats (it has no path, so the
|
|
394
|
+
// service answers 403). That refusal is not a failed search, and a
|
|
395
|
+
// count would tell the caller how much they cannot see.
|
|
396
|
+
layers.records && !found.unavailable && !found.missingEndpoint && !resolved.granted
|
|
397
|
+
? callRecords(toolkit, "space.stats", {}, spaceContext, scope).then(
|
|
398
|
+
(data) => recordCount(data, recordKinds),
|
|
399
|
+
() => null,
|
|
400
|
+
)
|
|
401
|
+
: Promise.resolve(0),
|
|
201
402
|
])
|
|
403
|
+
if (atomsCount === null) failures.push("atom count")
|
|
404
|
+
if (conversationsCount === null) failures.push("past-turn count")
|
|
405
|
+
if (recordsCount === null) failures.push("records count")
|
|
202
406
|
census = {
|
|
203
407
|
atoms: atomsCount ?? 0,
|
|
204
408
|
conversations: conversationsCount ?? 0,
|
|
205
|
-
|
|
409
|
+
records: recordsCount ?? 0,
|
|
410
|
+
failed: failures.length > 0,
|
|
411
|
+
grant: Boolean(resolved.granted),
|
|
412
|
+
coreEmpty: layers.atoms && layers.turns && !result.failures.length && atomsCount === 0 && conversationsCount === 0,
|
|
206
413
|
otherScopesSearched: !(control && spaceContext),
|
|
207
414
|
}
|
|
208
415
|
|
|
209
|
-
// A small scope can be described when a query misses: a question and
|
|
210
|
-
// record often share no word at all. Above the cap, report the
|
|
211
|
-
// and do not dump the store.
|
|
212
|
-
|
|
213
|
-
const totalRecords = census.atoms + census.conversations
|
|
416
|
+
// A small scope can be described when a query misses: a question and
|
|
417
|
+
// a record often share no word at all. Above the cap, report the
|
|
418
|
+
// count and do not dump the store.
|
|
419
|
+
const totalRecords = census.atoms + census.conversations + census.records
|
|
214
420
|
if (!census.failed && totalRecords > 0 && totalRecords <= BROWSE_MAX_RECORDS) {
|
|
215
|
-
// A backend older than the query endpoints simply has no browse;
|
|
216
|
-
// that must not turn a miss into an error.
|
|
217
421
|
const listRecent = (read) => Promise.resolve().then(read).catch(() => null)
|
|
218
|
-
const [recentAtoms, recentTurns] = await Promise.all([
|
|
219
|
-
listRecent(() => backend.queryAtoms({ scope, limit: BROWSE_ATOMS
|
|
220
|
-
|
|
422
|
+
const [recentAtoms, recentTurns, recentRecords] = await Promise.all([
|
|
423
|
+
layers.atoms ? listRecent(() => backend.queryAtoms({ scope, limit: BROWSE_ATOMS })) : null,
|
|
424
|
+
layers.turns ? listRecent(() => backend.queryConversations({ scope, limit: BROWSE_TURNS })) : null,
|
|
425
|
+
layers.records && census.records > 0
|
|
426
|
+
? listRecent(() => callRecords(toolkit, "records.list", { limit: BROWSE_RECORDS * 2, ...(recordKinds.length === 1 ? { kind: recordKinds[0] } : {}) }, spaceContext, scope))
|
|
427
|
+
: null,
|
|
221
428
|
])
|
|
222
429
|
// The browse path reads the scope directly, so it applies the same
|
|
223
|
-
// curation guards as the search path.
|
|
224
|
-
// record returns through atomic/query and the filter is decorative.
|
|
430
|
+
// curation guards as the search path.
|
|
225
431
|
noteLocations(spaceContext, "l1", recentAtoms?.items, { agent_id: scope.agent_id })
|
|
226
432
|
noteLocations(spaceContext, "l0", recentTurns?.messages, { agent_id: scope.agent_id })
|
|
227
433
|
const browseAtoms = partitionAtoms(recentAtoms?.items ?? []).served.slice(0, BROWSE_ATOMS)
|
|
228
|
-
const browseConversations = partitionTurns(recentTurns?.messages ?? [], { isCheckpoint: isCheckpointMessage })
|
|
229
|
-
|
|
230
|
-
|
|
434
|
+
const browseConversations = partitionTurns(recentTurns?.messages ?? [], { isCheckpoint: isCheckpointMessage }).served.slice(0, BROWSE_TURNS)
|
|
435
|
+
const listed = (recentRecords?.records ?? []).filter((record) => recordKinds.includes(record.kind) && (types.length === 0 || types.includes(record.type)))
|
|
436
|
+
const browseRecords = partitionRecords(listed.map((record) => ({ ...record, snippet: record.body }))).served.slice(0, BROWSE_RECORDS)
|
|
437
|
+
if (browseAtoms.length + browseConversations.length + browseRecords.length > 0) {
|
|
438
|
+
browse = { atoms: browseAtoms, conversations: browseConversations, records: browseRecords }
|
|
439
|
+
}
|
|
231
440
|
}
|
|
232
441
|
}
|
|
233
442
|
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
443
|
+
const renderMemory = (maxChars) => memoryLayers
|
|
444
|
+
? renderRecallBlock({
|
|
445
|
+
records,
|
|
446
|
+
atoms,
|
|
447
|
+
conversations,
|
|
448
|
+
scope: scoped,
|
|
449
|
+
scopeLabel: spaceContext ? displayScope(spaceContext, scoped) : undefined,
|
|
450
|
+
maxChars,
|
|
451
|
+
maxAtoms: limit,
|
|
452
|
+
maxRecords: limit,
|
|
453
|
+
maxTurns: MAX_TURNS,
|
|
454
|
+
query,
|
|
455
|
+
census,
|
|
456
|
+
browse,
|
|
457
|
+
withheld,
|
|
458
|
+
scopeNote,
|
|
459
|
+
failures,
|
|
460
|
+
failureDetail,
|
|
461
|
+
unmatchedRecords,
|
|
462
|
+
})
|
|
463
|
+
: ""
|
|
248
464
|
|
|
249
|
-
|
|
465
|
+
const searched = []
|
|
466
|
+
if (layers.records && !found.failed && !found.unavailable) searched.push(recordKinds.map((kind) => kind === "memory" ? "memories" : `${kind}s`).join(", "))
|
|
467
|
+
if (layers.atoms && !result.failures.includes("atoms")) searched.push("atoms")
|
|
468
|
+
if (layers.turns && !result.failures.includes("past turns")) searched.push("past conversations")
|
|
469
|
+
let documentCoverage = ""
|
|
470
|
+
let documents = ""
|
|
471
|
+
let documentsEmpty = false
|
|
472
|
+
if (layers.documents) {
|
|
250
473
|
const knowledgeScope = sharedScope(spaceContext)
|
|
251
|
-
const
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
474
|
+
const discoveryFailures = []
|
|
475
|
+
const unavailable = []
|
|
476
|
+
const ids = await discoverKnowledge(backend, log, undefined, knowledgeScope, { onFailure: (name, error) => (Number(error?.status) === 404 ? unavailable : discoveryFailures).push(name) })
|
|
477
|
+
let searchedDocuments = 0
|
|
478
|
+
let documentFailures = 0
|
|
479
|
+
documents =
|
|
255
480
|
ids.length === 0
|
|
256
481
|
? ""
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
482
|
+
: await searchKnowledge(backend, ids.slice(0, 2), query, knowledgeScope, {
|
|
483
|
+
maxChars: Math.max(1500, Math.floor(config.recallChars * (where === "documents" ? 1 : 0.5))),
|
|
484
|
+
onSearch: (_id, ok) => { if (ok) searchedDocuments += 1; else documentFailures += 1 },
|
|
485
|
+
})
|
|
486
|
+
if (!documents && where === "documents" && !discoveryFailures.length && !unavailable.length) documents = "(no accessible wiki assets found)"
|
|
487
|
+
documentCoverage = discoveryFailures.length || documentFailures
|
|
488
|
+
? `Document search incomplete: ${[...discoveryFailures, ...(documentFailures ? ["document search"] : [])].join(", ")} failed; do not infer absence.`
|
|
489
|
+
: ids.length === 0
|
|
490
|
+
? unavailable.length ? "Documents: no indexes found in available catalogs." : "Documents: no accessible wiki or code-graph indexes found."
|
|
491
|
+
: `Documents: searched ${searchedDocuments} of ${ids.length} accessible wiki or code-graph indexes.${searchedDocuments < ids.length ? " For a named document not covered, use mem_wiki." : ""}`
|
|
492
|
+
if (unavailable.length) documentCoverage += ` ${unavailable.join(", ")} unavailable (404).`
|
|
493
|
+
documentsEmpty = ids.length === 0 && discoveryFailures.length === 0 && unavailable.length === 0
|
|
261
494
|
}
|
|
262
495
|
|
|
496
|
+
const label = neutralizeFenceClosers(oneLine(spaceContext ? displayScope(spaceContext, scoped) : `${scoped.team_id}/${scoped.agent_id}`, 200))
|
|
497
|
+
const coverage = `Searched in ${label}: ${searched.length ? searched.join(", ") : "no memory layers"}. ${documentCoverage}`.trim()
|
|
498
|
+
const empty = where === "all" && types.length === 0 && !resolved.granted && census && !census.failed && !withheld.status && !withheld.instruction && !withheld.task && !withheld.redacted && census.records + census.atoms + census.conversations === 0 && documentsEmpty
|
|
499
|
+
const memoryEmpty = where === "all" && types.length === 0 && !resolved.granted && census && !census.failed && !withheld.status && !withheld.instruction && !withheld.task && !withheld.redacted && census.records + census.atoms + census.conversations === 0
|
|
500
|
+
const next = empty ? "Nothing is recorded in this space. Answer that it is not recorded, or continue the requested work. Do not search again in pages, wiki, spaces, commitments or other tools; a named document can be read directly when requested." : memoryEmpty ? "Nothing is recorded in the searched memory layers of this space. Report any document search limitation above; do not claim unsearched documents are empty. Do not repeat the search in pages, wiki, spaces or commitments." : ""
|
|
501
|
+
const memoryBudget = Math.max(100, config.recallChars - coverage.length - 1 - (documents ? documents.length + 2 : 0) - (next ? next.length + 1 : 0))
|
|
502
|
+
const memory = renderMemory(memoryBudget)
|
|
503
|
+
const block = [coverage, next, memory, documents].filter(Boolean).join("\n")
|
|
504
|
+
|
|
263
505
|
log.debug("recall", {
|
|
264
506
|
scope: displayScope(spaceContext, scoped),
|
|
507
|
+
in: where,
|
|
265
508
|
task: Boolean(key),
|
|
509
|
+
records: records.length,
|
|
266
510
|
atoms: atoms.length,
|
|
267
511
|
conversations: conversations.length,
|
|
268
512
|
withheld,
|
|
@@ -275,3 +519,10 @@ export function registerRecall(editor, toolkit) {
|
|
|
275
519
|
},
|
|
276
520
|
})
|
|
277
521
|
}
|
|
522
|
+
|
|
523
|
+
/** The records `space.stats` counts (`records.by_kind[kind][status]`) for the kinds a recall searched. */
|
|
524
|
+
function recordCount(stats, kinds) {
|
|
525
|
+
const byKind = stats?.records?.by_kind
|
|
526
|
+
if (!byKind || typeof byKind !== "object") return 0
|
|
527
|
+
return kinds.reduce((sum, kind) => sum + Object.values(byKind[kind] ?? {}).reduce((total, count) => total + (Number(count) || 0), 0), 0)
|
|
528
|
+
}
|