@joe-broadhead/opencode-mem 0.8.5 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +512 -0
- package/README.md +127 -61
- package/docs/agent-protocol.md +67 -177
- package/package.json +1453 -100
- package/skills/mem.pages/SKILL.md +146 -0
- package/skills/mem.sharing/SKILL.md +91 -0
- package/skills/mem.sources/SKILL.md +94 -0
- package/skills/mem.usage/SKILL.md +129 -492
- package/src/backend/records.js +285 -0
- package/src/backend/v3.js +3 -0
- package/src/bootstrap.js +49 -45
- package/src/call-shapes.js +41 -10
- package/src/collector-lock.js +162 -0
- package/src/collector-service.js +441 -0
- package/src/collector.js +381 -0
- package/src/commands.js +3 -3
- package/src/config.js +15 -1
- package/src/credentials.js +26 -1
- package/src/curation.js +1 -1
- package/src/permissions.js +87 -10
- package/src/protocol.js +29 -20
- package/src/recall.js +104 -27
- package/src/records-store.js +267 -121
- package/src/records.js +63 -11
- package/src/runtime.js +23 -14
- package/src/skills.js +44 -18
- package/src/sources.js +58 -0
- package/src/spaces.js +93 -10
- package/src/tools/aliases.js +164 -0
- package/src/tools/commitments.js +12 -9
- package/src/tools/held-write.js +82 -0
- package/src/tools/index.js +100 -23
- package/src/tools/ingest.js +277 -0
- package/src/tools/knowledge.js +6 -1
- package/src/tools/memory.js +722 -229
- package/src/tools/page.js +423 -0
- package/src/tools/recall.js +371 -113
- package/src/tools/records-kit.js +142 -0
- package/src/tools/share.js +1042 -0
- package/src/tools/shared.js +16 -2
- package/src/tools/spaces.js +431 -143
- package/src/tools/status.js +87 -58
- package/src/tools/wiki.js +8 -4
- package/src/views/contracts.js +293 -3
- package/src/views/inbox-decision.js +59 -0
- package/src/views/recall-result.js +12 -0
- package/src/views/records-reads.js +675 -0
- package/src/views/rpc.js +58 -99
- package/src/views/shape.js +61 -0
- package/views/commitments.html +4 -0
- package/views/graph.html +753 -0
- package/views/home.html +11 -5
- package/views/journal.html +621 -0
- package/views/memory.html +7 -1
- package/views/pages.html +636 -0
- package/views/recall.html +123 -0
- package/views/review.html +406 -0
- package/views/today.html +309 -0
package/src/tools/recall.js
CHANGED
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
import { isCheckpointMessage, renderRecallBlock } from "../recall.js"
|
|
2
|
-
import { addWithheld, partitionAtoms, partitionTurns } from "../curation.js"
|
|
3
|
-
import { errorMessage } from "../util.js"
|
|
2
|
+
import { addWithheld, isInstructionShaped, isStatusShaped, partitionAtoms, partitionTurns, stripInstructions } from "../curation.js"
|
|
3
|
+
import { errorMessage, neutralizeFenceClosers, oneLine } from "../util.js"
|
|
4
4
|
import { discoverKnowledge, searchKnowledge } from "./knowledge.js"
|
|
5
5
|
import { TASK_INPUT } from "./shared.js"
|
|
6
|
+
import { MEMORY_TYPES, PAGE_TYPES, callRecords, grantedSpace, noteRecordSpace } from "./records-kit.js"
|
|
7
|
+
import { isRecordsUnavailable } from "../backend/records.js"
|
|
8
|
+
import { recallResultView } from "../views/recall-result.js"
|
|
6
9
|
|
|
7
|
-
/** Upper bound on
|
|
10
|
+
/** Upper bound on hits per layer; the character budget still applies. */
|
|
8
11
|
const MAX_LIMIT = 25
|
|
9
|
-
/** Raw turns rendered per recall.
|
|
12
|
+
/** Raw turns rendered per recall. Records and atoms carry the distilled answer. */
|
|
10
13
|
const MAX_TURNS = 4
|
|
11
|
-
/**
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
const ATOM_TYPES = Object.freeze(["persona", "episodic", "instruction", "work_fact", "work_method", "work_artifact"])
|
|
14
|
+
/** Where recall looks (contract §4.1). */
|
|
15
|
+
export const RECALL_IN = Object.freeze(["all", "memories", "pages", "transcripts", "documents"])
|
|
16
|
+
/** Record types `types` filters on: the kinds remember writes, page types, and commitments. */
|
|
17
|
+
export const RECORD_TYPES = Object.freeze([...new Set([...MEMORY_TYPES, ...PAGE_TYPES, "commitment"])])
|
|
16
18
|
|
|
17
19
|
/** Whether upstream returned a task field for this row (L1 hits do, L0 search hits do not). */
|
|
18
20
|
function hasTaskField(row) {
|
|
@@ -26,7 +28,51 @@ function rowTask(row) {
|
|
|
26
28
|
}
|
|
27
29
|
|
|
28
30
|
/**
|
|
29
|
-
*
|
|
31
|
+
* The record hits recall may serve. Memories are curated at write, but a seed
|
|
32
|
+
* or an import is not, so a status- or instruction-shaped memory is withheld
|
|
33
|
+
* and counted as an atom would be. A page or commitment keeps its Status
|
|
34
|
+
* section (that is its home); a planted command in its text is removed and
|
|
35
|
+
* marked, and a hit that is nothing else is withheld.
|
|
36
|
+
*/
|
|
37
|
+
export function partitionRecords(hits) {
|
|
38
|
+
const served = []
|
|
39
|
+
const withheld = { status: 0, instruction: 0, task: 0 }
|
|
40
|
+
for (const hit of hits ?? []) {
|
|
41
|
+
const text = `${hit?.title ?? ""}\n${hit?.snippet ?? ""}`
|
|
42
|
+
if (hit?.kind === "memory" && isStatusShaped(text)) {
|
|
43
|
+
withheld.status += 1
|
|
44
|
+
continue
|
|
45
|
+
}
|
|
46
|
+
if (!isInstructionShaped(text)) {
|
|
47
|
+
served.push(hit)
|
|
48
|
+
continue
|
|
49
|
+
}
|
|
50
|
+
if (hit?.kind === "memory") {
|
|
51
|
+
withheld.instruction += 1
|
|
52
|
+
continue
|
|
53
|
+
}
|
|
54
|
+
const snippet = stripInstructions(hit.snippet ?? "")
|
|
55
|
+
const title = stripInstructions(hit.title ?? "")
|
|
56
|
+
if (!snippet.text && !title.text) {
|
|
57
|
+
withheld.instruction += 1
|
|
58
|
+
continue
|
|
59
|
+
}
|
|
60
|
+
served.push({ ...hit, title: title.text || hit.path || hit.id, snippet: snippet.text })
|
|
61
|
+
}
|
|
62
|
+
return { served, withheld }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Lists as given, or a comma-separated string, trimmed and de-duplicated. */
|
|
66
|
+
function listOf(value) {
|
|
67
|
+
const raw = Array.isArray(value) ? value : typeof value === "string" ? value.split(",") : []
|
|
68
|
+
return [...new Set(raw.map((entry) => String(entry ?? "").trim()).filter(Boolean))]
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* `mem_recall`: pull-based retrieval (contract §4.1). The records service
|
|
73
|
+
* first (memories, pages, commitments, by full text), then the core's atoms
|
|
74
|
+
* and turns as before, then documents through the hub's wiki search. One line
|
|
75
|
+
* per hit, with its id, inside the `<recalled_memory>` fence.
|
|
30
76
|
*
|
|
31
77
|
* @param {any} editor
|
|
32
78
|
* @param {ReturnType<import("./shared.js").createToolkit>} toolkit
|
|
@@ -34,20 +80,67 @@ function rowTask(row) {
|
|
|
34
80
|
export function registerRecall(editor, toolkit) {
|
|
35
81
|
const { backend, config, log, control, text, fail, sharedScope, resolveReadSpace, displayScope, noteLocations } = toolkit
|
|
36
82
|
|
|
83
|
+
// Resolve a kind only for recall, never for writes. A bound shared space wins;
|
|
84
|
+
// otherwise a single accessible shared space is unambiguous. Control still
|
|
85
|
+
// resolves and authorizes the selected id, without binding the session.
|
|
86
|
+
async function recallSpace(input, context, visibility) {
|
|
87
|
+
const requested = String(input?.space ?? "").trim()
|
|
88
|
+
if (requested === "personal") {
|
|
89
|
+
if (visibility === "team") return { invalid: "visibility=team cannot search the personal space." }
|
|
90
|
+
if (!control) return { error: "control plane not configured; cannot select your personal space" }
|
|
91
|
+
return resolveReadSpace({}, context, "private")
|
|
92
|
+
}
|
|
93
|
+
if (requested !== "shared" && (requested || visibility !== "team" || !control)) {
|
|
94
|
+
return resolveReadSpace(input, context, visibility)
|
|
95
|
+
}
|
|
96
|
+
if (visibility === "private") return { invalid: "visibility=private cannot search a shared space." }
|
|
97
|
+
if (!control) return { error: "control plane not configured; cannot select a shared space" }
|
|
98
|
+
try {
|
|
99
|
+
const active = await toolkit.runtime.sessionContext(context?.sessionID, context?.agent)
|
|
100
|
+
if (active?.space_kind === "shared") return { context: active }
|
|
101
|
+
const spaces = (await control.listSpaces({ userId: config.user })).filter((space) => space.kind === "shared")
|
|
102
|
+
if (spaces.length !== 1) return { invalid: spaces.length ? "More than one shared space is available; ask which space to search and pass its id." : "No shared space is available to search; do not infer what it contains." }
|
|
103
|
+
const selected = await toolkit.runtime.resolveContext({ space: spaces[0].space_id, agent: context?.agent })
|
|
104
|
+
if (!selected || selected.space_kind !== "shared" || selected.space_id !== spaces[0].space_id) return { error: "the control plane did not confirm the selected shared space" }
|
|
105
|
+
return { context: selected }
|
|
106
|
+
} catch (error) {
|
|
107
|
+
return { error: errorMessage(error) }
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
37
111
|
/**
|
|
38
112
|
* Reads describe the whole space, not one session. L0 writes from
|
|
39
|
-
*
|
|
40
|
-
*
|
|
113
|
+
* capture live under synthetic session ids, so scoping a recall query by
|
|
114
|
+
* the current session hides the very turns just written.
|
|
41
115
|
*/
|
|
42
116
|
function readScope(scoped) {
|
|
43
117
|
const { session_id, ...rest } = scoped
|
|
44
118
|
return rest
|
|
45
119
|
}
|
|
46
120
|
|
|
121
|
+
/**
|
|
122
|
+
* Label record hits by the task they were written under: `task` on the hit
|
|
123
|
+
* when the service sends it, else read once with `records.get`. A failed
|
|
124
|
+
* read labels nothing as the task's, so the hits stay, as scope=space.
|
|
125
|
+
*/
|
|
126
|
+
async function labelRecordTasks(hits, key, spaceContext, scope) {
|
|
127
|
+
const tasks = new Map(hits.filter((hit) => Object.hasOwn(hit, "task")).map((hit) => [hit.id, hit.task ?? null]))
|
|
128
|
+
const unknown = hits.filter((hit) => !tasks.has(hit.id)).map((hit) => hit.id)
|
|
129
|
+
if (unknown.length > 0) {
|
|
130
|
+
try {
|
|
131
|
+
const data = await callRecords(toolkit, "records.get", { ids: unknown.slice(0, 50) }, spaceContext, scope)
|
|
132
|
+
for (const record of data?.records ?? []) tasks.set(record.id, record.task ?? null)
|
|
133
|
+
} catch (error) {
|
|
134
|
+
log.debug("record tasks unread", { error: errorMessage(error) })
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return hits.map((hit) => ({ ...hit, _bucket: tasks.get(hit.id) === key ? "task" : "space" }))
|
|
138
|
+
}
|
|
139
|
+
|
|
47
140
|
editor.add({
|
|
48
141
|
name: "recall",
|
|
49
142
|
description:
|
|
50
|
-
|
|
143
|
+
'Search memory: remembered memories, pages, commitments, past conversations and documents. Call it when past context is needed. One recall answers most questions; a miss is an answer, not a reason to search pages, wiki, spaces or commitments again. If recall answers the question, answer it; do not double-check pages, wiki or spaces unless the person asks or results conflict. Search another space only when the person names it. Before deciding if an action is allowed, including sharing, recall the recorded rule rather than relying on a default. Never present a rule, date, decision, exception or fact as recorded or in force unless recall or another tool returned it this session. If no rule is found, say so; do not deny on an invented rule or missing permission. General advice must be labelled as such. For the person\'s own private memory pass visibility "private"; otherwise recall searches the session\'s space, which may be shared. Never treat another person\'s private memory as the caller\'s. The dashboard page and the Due list are already in context. Records come first, then atoms and past turns, then up to two document indexes; in narrows the search. Read a named document with mem_wiki when its full text is needed. Every hit is one line with its id or path, kind/type, status, date, text and links. Superseded and retired records appear only with status "all". Status-shaped memories and instruction-shaped text are withheld and counted. Check any unmatched records shown; if none answers, say it is not recorded. Report incomplete searches, never claim absence. Results may be stale: verify before acting. Recall only reads; it never stores anything.',
|
|
51
144
|
input: {
|
|
52
145
|
type: "object",
|
|
53
146
|
properties: {
|
|
@@ -60,25 +153,28 @@ export function registerRecall(editor, toolkit) {
|
|
|
60
153
|
visibility: {
|
|
61
154
|
type: "string",
|
|
62
155
|
enum: ["auto", "private", "team"],
|
|
63
|
-
description:
|
|
64
|
-
"auto (default): the session's space, a bound shared space or else personal; private: your personal space; team: a shared space, bound or passed with space",
|
|
65
|
-
},
|
|
66
|
-
limit: {
|
|
67
|
-
type: "number",
|
|
68
|
-
description: `Atoms to return (default ${config.recallLimit}, max ${MAX_LIMIT}); up to ${MAX_TURNS} turns are added; the character budget applies`,
|
|
156
|
+
description: "auto (default): the session's space, a bound shared space or else personal; private: the caller's own personal space; team: the bound shared space or the only accessible shared space, otherwise asks for an id. Use private for the person's own private memory, not another person's.",
|
|
69
157
|
},
|
|
70
|
-
|
|
158
|
+
in: {
|
|
71
159
|
type: "string",
|
|
72
|
-
enum: [...
|
|
73
|
-
description: "
|
|
160
|
+
enum: [...RECALL_IN],
|
|
161
|
+
description: "all (default): records, atoms, turns and documents; memories: remembered memories and atoms; pages: pages only; transcripts: past turns only; documents: uploaded files and repositories (the hub's wikis and code graphs)",
|
|
162
|
+
},
|
|
163
|
+
types: {
|
|
164
|
+
type: "array",
|
|
165
|
+
items: { type: "string", enum: [...RECORD_TYPES] },
|
|
166
|
+
description: "Filter records strictly by these types. Memory types also search legacy atoms without kind filtering, labelled by their core type; turns are excluded. Page and commitment types do not search atoms.",
|
|
74
167
|
},
|
|
75
|
-
|
|
168
|
+
status: {
|
|
76
169
|
type: "string",
|
|
77
|
-
enum: ["
|
|
78
|
-
description:
|
|
170
|
+
enum: ["current", "all"],
|
|
171
|
+
description: 'current (default): current and contested records; all: superseded and retired ones too, for "what was it before"',
|
|
79
172
|
},
|
|
80
|
-
|
|
81
|
-
|
|
173
|
+
limit: {
|
|
174
|
+
type: "number",
|
|
175
|
+
description: `Hits per layer (default ${config.recallLimit}, max ${MAX_LIMIT}); up to ${MAX_TURNS} turns are added; the character budget applies`,
|
|
176
|
+
},
|
|
177
|
+
space: { type: "string", description: 'Space id, "personal" for your own personal space, or "shared" for the bound or only accessible shared space. Ambiguous shared selection asks for an id. Defaults to the session space; never rebinds.' },
|
|
82
178
|
},
|
|
83
179
|
required: ["query"],
|
|
84
180
|
additionalProperties: false,
|
|
@@ -88,57 +184,102 @@ export function registerRecall(editor, toolkit) {
|
|
|
88
184
|
const query = String(input?.query ?? "").trim()
|
|
89
185
|
if (!query) return text("nothing to recall: query is empty")
|
|
90
186
|
const limit = Math.max(1, Math.min(Math.floor(Number(input?.limit)) || config.recallLimit, MAX_LIMIT))
|
|
187
|
+
// The session's space (personal, or a shared one bound to it), or the
|
|
188
|
+
// space named; pre-0.9 callers' visibility still selects as it did.
|
|
91
189
|
const visibility = ["auto", "private", "team"].includes(String(input?.visibility)) ? String(input.visibility) : "auto"
|
|
92
|
-
const
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
190
|
+
const where = input?.in === undefined || input?.in === null || input?.in === "" ? "all" : String(input.in)
|
|
191
|
+
if (!RECALL_IN.includes(where)) return text(`in must be one of ${RECALL_IN.join(", ")}.`)
|
|
192
|
+
const types = listOf(input?.types)
|
|
193
|
+
const badTypes = types.filter((value) => !RECORD_TYPES.includes(value))
|
|
194
|
+
if (badTypes.includes("work_task")) return text("work_task atoms are task descriptions; recall withholds them. Read the tracker or mem_commitments for things to do.")
|
|
195
|
+
if (badTypes.length > 0) return text(`unknown type ${badTypes.join(", ")}; types are ${RECORD_TYPES.join(", ")}.`)
|
|
196
|
+
const status = input?.status === "all" ? "all" : input?.status === undefined || input?.status === null || input?.status === "" || input?.status === "current" ? "current" : null
|
|
197
|
+
if (!status) return text('status is "current" (default) or "all".')
|
|
198
|
+
|
|
199
|
+
// Which layers this call reads.
|
|
200
|
+
const layers = {
|
|
201
|
+
records: ["all", "memories", "pages"].includes(where),
|
|
202
|
+
atoms: (types.length === 0 || types.some((type) => MEMORY_TYPES.includes(type))) && ["all", "memories"].includes(where),
|
|
203
|
+
turns: types.length === 0 && ["all", "transcripts"].includes(where),
|
|
204
|
+
documents: where === "documents" || (where === "all" && types.length === 0),
|
|
100
205
|
}
|
|
206
|
+
const recordKinds = where === "memories" ? ["memory"] : where === "pages" ? ["page"] : ["memory", "page", "commitment"]
|
|
101
207
|
|
|
102
208
|
const task = toolkit.taskFor(input, context)
|
|
103
209
|
if (task.invalid) return text(task.invalid)
|
|
104
210
|
|
|
105
|
-
|
|
211
|
+
// A space the person reaches only through a page grant has no role, so
|
|
212
|
+
// control refuses its context; its records are read through the
|
|
213
|
+
// gateway, which resolves the grant. The core is not readable there.
|
|
214
|
+
const readSpace = await recallSpace(input, context, visibility)
|
|
215
|
+
// Private means the caller's own space, never the page-grant fallback
|
|
216
|
+
// for some other space, even when that page would otherwise be readable.
|
|
217
|
+
const resolved = visibility === "private" || ["personal", "shared"].includes(String(input?.space ?? "").trim()) ? readSpace : grantedSpace(input, readSpace)
|
|
106
218
|
if (resolved.invalid) return text(resolved.invalid)
|
|
107
219
|
if (resolved.error) return fail(resolved.error)
|
|
108
220
|
const spaceContext = resolved.context
|
|
221
|
+
if (resolved.granted) {
|
|
222
|
+
layers.atoms = false
|
|
223
|
+
layers.turns = false
|
|
224
|
+
layers.documents = false
|
|
225
|
+
}
|
|
109
226
|
const scoped = spaceContext
|
|
110
227
|
? toolkit.scopeFor(spaceContext, { agent: context?.agent })
|
|
111
228
|
: toolkit.scope({ agent: context?.agent })
|
|
112
229
|
|
|
113
230
|
try {
|
|
114
|
-
// Over-fetch atoms: curation can withhold part of a page, and the
|
|
115
|
-
// caller asked for `limit` servable atoms, not `limit` candidates.
|
|
116
231
|
const scope = readScope(scoped)
|
|
117
232
|
const key = task.key
|
|
233
|
+
|
|
234
|
+
/** Records: full text over memories, pages and commitments, space-wide. */
|
|
235
|
+
const recordSearch = layers.records
|
|
236
|
+
? callRecords(
|
|
237
|
+
toolkit,
|
|
238
|
+
"records.search",
|
|
239
|
+
{ query, kinds: recordKinds, ...(types.length > 0 ? { types } : {}), status, limit: Math.min(50, limit * 2) },
|
|
240
|
+
spaceContext,
|
|
241
|
+
scope,
|
|
242
|
+
).then(
|
|
243
|
+
(data) => ({ hits: Array.isArray(data?.hits) ? data.hits : [], failed: false }),
|
|
244
|
+
(error) => {
|
|
245
|
+
log.debug("records search failed", { error: errorMessage(error) })
|
|
246
|
+
// A server without records still permits a core search, but
|
|
247
|
+
// its result must be labelled incomplete.
|
|
248
|
+
if (isRecordsUnavailable(error)) return { hits: [], failed: false, unavailable: "server" }
|
|
249
|
+
if (error?.code === "records_missing") return { hits: [], failed: false, unavailable: "build" }
|
|
250
|
+
// Older gateways have no records route. A bare 404 at the
|
|
251
|
+
// search endpoint is not a search result or a missing record.
|
|
252
|
+
if (error?.status === 404 && !error?.code) return { hits: [], failed: true, missingEndpoint: true }
|
|
253
|
+
return { hits: [], failed: true }
|
|
254
|
+
},
|
|
255
|
+
)
|
|
256
|
+
: Promise.resolve({ hits: [], failed: false })
|
|
257
|
+
|
|
118
258
|
async function searchIn(searchScope) {
|
|
119
|
-
|
|
259
|
+
const failures = []
|
|
120
260
|
const [atomPage, turnPage] = await Promise.all([
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
}),
|
|
125
|
-
type
|
|
126
|
-
? Promise.resolve(null)
|
|
127
|
-
: backend.searchConversations({ query, limit: Math.min(MAX_TURNS * 2, limit * 2), scope: searchScope }).catch(() => {
|
|
128
|
-
failed = true
|
|
261
|
+
layers.atoms
|
|
262
|
+
? backend.searchAtoms({ query, limit: Math.min(MAX_LIMIT, limit * 2), scope: searchScope }).catch(() => {
|
|
263
|
+
failures.push("atoms")
|
|
129
264
|
return null
|
|
130
|
-
})
|
|
265
|
+
})
|
|
266
|
+
: Promise.resolve(null),
|
|
267
|
+
layers.turns
|
|
268
|
+
? backend.searchConversations({ query, limit: Math.min(MAX_TURNS * 2, limit * 2), scope: searchScope }).catch(() => {
|
|
269
|
+
failures.push("past turns")
|
|
270
|
+
return null
|
|
271
|
+
})
|
|
272
|
+
: Promise.resolve(null),
|
|
131
273
|
])
|
|
132
|
-
// Remember where each hit was written, so
|
|
274
|
+
// Remember where each hit was written, so a forget can address the
|
|
133
275
|
// delete to the row's own agent and session.
|
|
134
276
|
noteLocations(spaceContext, "l1", atomPage?.items, { agent_id: scope.agent_id })
|
|
135
277
|
noteLocations(spaceContext, "l0", turnPage?.messages, { agent_id: scope.agent_id })
|
|
136
278
|
// A hit that carries its task is labelled by it, so a backend that
|
|
137
279
|
// ignored the header cannot make another task's record look like this
|
|
138
|
-
// one's. Upstream L0 search hits carry no task field at all
|
|
139
|
-
//
|
|
140
|
-
//
|
|
141
|
-
// its hits are the task's, and a widened search's hits are not.
|
|
280
|
+
// one's. Upstream L0 search hits carry no task field at all; for those
|
|
281
|
+
// the request decides: upstream filters a search sent with
|
|
282
|
+
// x-tdai-task-id to that task, so its hits are the task's.
|
|
142
283
|
const filtered = Boolean(searchScope.task_id)
|
|
143
284
|
const bucketOf = (row) => {
|
|
144
285
|
if (!hasTaskField(row)) return filtered ? "task" : "space"
|
|
@@ -151,35 +292,65 @@ export function registerRecall(editor, toolkit) {
|
|
|
151
292
|
atoms: atomPart.served.map(label),
|
|
152
293
|
conversations: turnPart.served.map(label),
|
|
153
294
|
withheld: addWithheld(atomPart.withheld, turnPart.withheld),
|
|
154
|
-
|
|
295
|
+
failures,
|
|
155
296
|
}
|
|
156
297
|
}
|
|
157
298
|
|
|
158
|
-
let result = { atoms: [], conversations: [], withheld: addWithheld(),
|
|
159
|
-
let
|
|
160
|
-
if (
|
|
299
|
+
let result = { atoms: [], conversations: [], withheld: addWithheld(), failures: [] }
|
|
300
|
+
let taskHit = false
|
|
301
|
+
if (layers.atoms || layers.turns) {
|
|
161
302
|
if (key) {
|
|
162
303
|
const first = await searchIn({ ...scope, task_id: key })
|
|
163
304
|
const rows = [...first.atoms, ...first.conversations]
|
|
164
305
|
if (rows.some((row) => row._bucket === "task")) {
|
|
165
306
|
result = first
|
|
166
|
-
|
|
167
|
-
? `Searched task "${key}". scope=task hits belong to it; scope=space hits were recorded outside it and are still valid memory of this space.`
|
|
168
|
-
: `Searched task "${key}"; every hit belongs to it.`
|
|
307
|
+
taskHit = true
|
|
169
308
|
} else {
|
|
170
309
|
result = await searchIn(scope)
|
|
171
|
-
result.
|
|
172
|
-
scopeNote =
|
|
173
|
-
result.atoms.length + result.conversations.length > 0
|
|
174
|
-
? `No match in task "${key}"; widened to the whole space. scope=space hits were recorded outside this task and are still valid memory of this space: use them, and say where they came from.`
|
|
175
|
-
: `Searched task "${key}", then the whole space.`
|
|
310
|
+
result.failures = [...new Set([...result.failures, ...first.failures])]
|
|
176
311
|
}
|
|
177
312
|
} else {
|
|
178
313
|
result = await searchIn(scope)
|
|
179
|
-
scopeNote = task.wholeSpace ? "Searched the whole space." : "No task declared; searched the whole space."
|
|
180
314
|
}
|
|
181
315
|
}
|
|
182
|
-
const
|
|
316
|
+
const found = await recordSearch
|
|
317
|
+
// No grant reached a page either: the space is not readable, and the
|
|
318
|
+
// control plane's refusal says why.
|
|
319
|
+
if (resolved.granted && found.failed) return fail(resolved.refusal)
|
|
320
|
+
// Records are searched space-wide; with a task declared, the hits of
|
|
321
|
+
// that task are served first, as the core's are: the others only
|
|
322
|
+
// when the task has none, labelled scope=space.
|
|
323
|
+
let recordHits = found.hits
|
|
324
|
+
if (key && recordHits.length > 0) {
|
|
325
|
+
const labelled = await labelRecordTasks(recordHits, key, spaceContext, scope)
|
|
326
|
+
recordHits = labelled.some((hit) => hit._bucket === "task") ? labelled.filter((hit) => hit._bucket === "task") : labelled
|
|
327
|
+
if (labelled.some((hit) => hit._bucket === "task")) taskHit = true
|
|
328
|
+
}
|
|
329
|
+
let scopeNote
|
|
330
|
+
if (key) {
|
|
331
|
+
const rows = [...recordHits, ...result.atoms, ...result.conversations]
|
|
332
|
+
scopeNote = taskHit
|
|
333
|
+
? rows.some((row) => row._bucket === "space")
|
|
334
|
+
? `Searched task "${key}". scope=task hits belong to it; scope=space hits were recorded outside it and are still valid memory of this space.`
|
|
335
|
+
: `Searched task "${key}"; every hit belongs to it.`
|
|
336
|
+
: rows.length > 0
|
|
337
|
+
? `No match in task "${key}"; widened to the whole space. scope=space hits were recorded outside this task and are still valid memory of this space: use them, and say where they came from.`
|
|
338
|
+
: `Searched task "${key}", then the whole space.`
|
|
339
|
+
} else if (layers.atoms || layers.turns || layers.records) {
|
|
340
|
+
scopeNote = task.wholeSpace ? "Searched the whole space." : "No task declared; searched the whole space."
|
|
341
|
+
}
|
|
342
|
+
const recordPart = partitionRecords(recordHits)
|
|
343
|
+
const records = recordPart.served
|
|
344
|
+
for (const record of records) noteRecordSpace(toolkit, record.id, spaceContext)
|
|
345
|
+
const { atoms, conversations } = result
|
|
346
|
+
const failures = [...result.failures, ...(found.failed || found.unavailable ? [found.missingEndpoint ? "records endpoint (404)" : "records"] : [])]
|
|
347
|
+
const failureDetail = found.unavailable
|
|
348
|
+
? `${found.unavailable === "build" ? "This plugin build has no records client" : "Records are not enabled on this Memory server yet"}, so remembered memories, pages and commitments were not searched${layers.atoms && layers.turns && result.failures.length === 0 ? "; past turns and atoms were" : ""}.`
|
|
349
|
+
: undefined
|
|
350
|
+
const withheld = addWithheld(result.withheld, recordPart.withheld)
|
|
351
|
+
if (layers.records && types.length > 0) scopeNote = `Only records of type ${types.join(", ")} were searched; ${layers.atoms ? "legacy atoms were searched without kind filtering; turns were not searched" : "atoms and turns were not"}.`
|
|
352
|
+
if (layers.records && status === "all") scopeNote = `${scopeNote ? `${scopeNote} ` : ""}Superseded and retired records are included (status "all").`
|
|
353
|
+
if (resolved.granted) scopeNote = `${scopeNote ? `${scopeNote} ` : ""}This space is shared with you by page grant: only the pages granted were searched.`
|
|
183
354
|
|
|
184
355
|
for (const atom of atoms) {
|
|
185
356
|
if (atom?.asset_id) void backend.touchAsset({ assetID: atom.asset_id, scope: sharedScope(spaceContext) }).catch(() => {})
|
|
@@ -187,87 +358,167 @@ export function registerRecall(editor, toolkit) {
|
|
|
187
358
|
|
|
188
359
|
// An empty result is ambiguous: nothing stored, nothing matching these
|
|
189
360
|
// words, everything withheld, or a failed search. Say which one, so the
|
|
190
|
-
// agent can stop instead of fanning out across spaces
|
|
191
|
-
// wiki.
|
|
361
|
+
// agent can stop instead of fanning out across spaces and tools.
|
|
192
362
|
const BROWSE_MAX_RECORDS = 12
|
|
193
363
|
const BROWSE_ATOMS = 6
|
|
194
364
|
const BROWSE_TURNS = 4
|
|
365
|
+
const BROWSE_RECORDS = 6
|
|
195
366
|
let census
|
|
196
367
|
let browse
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
368
|
+
let unmatchedRecords = []
|
|
369
|
+
const memoryLayers = layers.records || layers.atoms || layers.turns
|
|
370
|
+
// A lexical records miss can coexist with a core hit on a nearby
|
|
371
|
+
// topic. In a very small store, show those records as unranked context
|
|
372
|
+
// rather than letting that hit suppress the records fallback entirely.
|
|
373
|
+
// The service applies the same space, role, kind and status filters.
|
|
374
|
+
if (layers.records && !found.failed && !found.unavailable && recordHits.length === 0 && atoms.length + conversations.length > 0) {
|
|
375
|
+
try {
|
|
376
|
+
const page = await callRecords(toolkit, "records.list", {
|
|
377
|
+
kind: recordKinds,
|
|
378
|
+
...(types.length > 0 ? { type: types } : {}),
|
|
379
|
+
status,
|
|
380
|
+
limit: 4,
|
|
381
|
+
}, spaceContext, scope)
|
|
382
|
+
if (!page?.cursor && Array.isArray(page?.records) && page.records.length <= 3) {
|
|
383
|
+
unmatchedRecords = partitionRecords(page.records.map((record) => ({ ...record, snippet: record.body }))).served
|
|
384
|
+
for (const record of unmatchedRecords) noteRecordSpace(toolkit, record.id, spaceContext)
|
|
385
|
+
}
|
|
386
|
+
} catch (error) {
|
|
387
|
+
log.debug("records fallback unavailable", { error: errorMessage(error) })
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
if (memoryLayers && records.length === 0 && atoms.length === 0 && conversations.length === 0) {
|
|
391
|
+
const [atomsCount, conversationsCount, recordsCount] = await Promise.all([
|
|
392
|
+
layers.atoms ? backend.countAtoms({ scope }).then((data) => Number(data?.total) || 0).catch(() => null) : Promise.resolve(0),
|
|
393
|
+
layers.turns ? backend.countConversations({ scope }).then((data) => Number(data?.total) || 0).catch(() => null) : Promise.resolve(0),
|
|
394
|
+
// A page grant cannot read space.stats (it has no path, so the
|
|
395
|
+
// service answers 403). That refusal is not a failed search, and a
|
|
396
|
+
// count would tell the caller how much they cannot see.
|
|
397
|
+
layers.records && !found.unavailable && !found.missingEndpoint && !resolved.granted
|
|
398
|
+
? callRecords(toolkit, "space.stats", {}, spaceContext, scope).then(
|
|
399
|
+
(data) => recordCount(data, recordKinds),
|
|
400
|
+
() => null,
|
|
401
|
+
)
|
|
402
|
+
: Promise.resolve(0),
|
|
201
403
|
])
|
|
404
|
+
if (atomsCount === null) failures.push("atom count")
|
|
405
|
+
if (conversationsCount === null) failures.push("past-turn count")
|
|
406
|
+
if (recordsCount === null) failures.push("records count")
|
|
202
407
|
census = {
|
|
203
408
|
atoms: atomsCount ?? 0,
|
|
204
409
|
conversations: conversationsCount ?? 0,
|
|
205
|
-
|
|
410
|
+
records: recordsCount ?? 0,
|
|
411
|
+
failed: failures.length > 0,
|
|
412
|
+
grant: Boolean(resolved.granted),
|
|
413
|
+
coreEmpty: layers.atoms && layers.turns && !result.failures.length && atomsCount === 0 && conversationsCount === 0,
|
|
206
414
|
otherScopesSearched: !(control && spaceContext),
|
|
207
415
|
}
|
|
208
416
|
|
|
209
|
-
// A small scope can be described when a query misses: a question and
|
|
210
|
-
// record often share no word at all. Above the cap, report the
|
|
211
|
-
// and do not dump the store.
|
|
212
|
-
|
|
213
|
-
const totalRecords = census.atoms + census.conversations
|
|
417
|
+
// A small scope can be described when a query misses: a question and
|
|
418
|
+
// a record often share no word at all. Above the cap, report the
|
|
419
|
+
// count and do not dump the store.
|
|
420
|
+
const totalRecords = census.atoms + census.conversations + census.records
|
|
214
421
|
if (!census.failed && totalRecords > 0 && totalRecords <= BROWSE_MAX_RECORDS) {
|
|
215
|
-
// A backend older than the query endpoints simply has no browse;
|
|
216
|
-
// that must not turn a miss into an error.
|
|
217
422
|
const listRecent = (read) => Promise.resolve().then(read).catch(() => null)
|
|
218
|
-
const [recentAtoms, recentTurns] = await Promise.all([
|
|
219
|
-
listRecent(() => backend.queryAtoms({ scope, limit: BROWSE_ATOMS
|
|
220
|
-
|
|
423
|
+
const [recentAtoms, recentTurns, recentRecords] = await Promise.all([
|
|
424
|
+
layers.atoms ? listRecent(() => backend.queryAtoms({ scope, limit: BROWSE_ATOMS })) : null,
|
|
425
|
+
layers.turns ? listRecent(() => backend.queryConversations({ scope, limit: BROWSE_TURNS })) : null,
|
|
426
|
+
layers.records && census.records > 0
|
|
427
|
+
? listRecent(() => callRecords(toolkit, "records.list", { limit: BROWSE_RECORDS * 2, ...(recordKinds.length === 1 ? { kind: recordKinds[0] } : {}) }, spaceContext, scope))
|
|
428
|
+
: null,
|
|
221
429
|
])
|
|
222
430
|
// The browse path reads the scope directly, so it applies the same
|
|
223
|
-
// curation guards as the search path.
|
|
224
|
-
// record returns through atomic/query and the filter is decorative.
|
|
431
|
+
// curation guards as the search path.
|
|
225
432
|
noteLocations(spaceContext, "l1", recentAtoms?.items, { agent_id: scope.agent_id })
|
|
226
433
|
noteLocations(spaceContext, "l0", recentTurns?.messages, { agent_id: scope.agent_id })
|
|
227
434
|
const browseAtoms = partitionAtoms(recentAtoms?.items ?? []).served.slice(0, BROWSE_ATOMS)
|
|
228
|
-
const browseConversations = partitionTurns(recentTurns?.messages ?? [], { isCheckpoint: isCheckpointMessage })
|
|
229
|
-
|
|
230
|
-
|
|
435
|
+
const browseConversations = partitionTurns(recentTurns?.messages ?? [], { isCheckpoint: isCheckpointMessage }).served.slice(0, BROWSE_TURNS)
|
|
436
|
+
const listed = (recentRecords?.records ?? []).filter((record) => recordKinds.includes(record.kind) && (types.length === 0 || types.includes(record.type)))
|
|
437
|
+
const browseRecords = partitionRecords(listed.map((record) => ({ ...record, snippet: record.body }))).served.slice(0, BROWSE_RECORDS)
|
|
438
|
+
if (browseAtoms.length + browseConversations.length + browseRecords.length > 0) {
|
|
439
|
+
browse = { atoms: browseAtoms, conversations: browseConversations, records: browseRecords }
|
|
440
|
+
}
|
|
231
441
|
}
|
|
232
442
|
}
|
|
233
443
|
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
444
|
+
const renderMemory = (maxChars) => memoryLayers
|
|
445
|
+
? renderRecallBlock({
|
|
446
|
+
records,
|
|
447
|
+
atoms,
|
|
448
|
+
conversations,
|
|
449
|
+
scope: scoped,
|
|
450
|
+
scopeLabel: spaceContext ? displayScope(spaceContext, scoped) : undefined,
|
|
451
|
+
maxChars,
|
|
452
|
+
maxAtoms: limit,
|
|
453
|
+
maxRecords: limit,
|
|
454
|
+
maxTurns: MAX_TURNS,
|
|
455
|
+
query,
|
|
456
|
+
census,
|
|
457
|
+
browse,
|
|
458
|
+
withheld,
|
|
459
|
+
scopeNote,
|
|
460
|
+
failures,
|
|
461
|
+
failureDetail,
|
|
462
|
+
unmatchedRecords,
|
|
463
|
+
})
|
|
464
|
+
: ""
|
|
248
465
|
|
|
249
|
-
|
|
466
|
+
const searched = []
|
|
467
|
+
if (layers.records && !found.failed && !found.unavailable) searched.push(recordKinds.map((kind) => kind === "memory" ? "memories" : `${kind}s`).join(", "))
|
|
468
|
+
if (layers.atoms && !result.failures.includes("atoms")) searched.push("atoms")
|
|
469
|
+
if (layers.turns && !result.failures.includes("past turns")) searched.push("past conversations")
|
|
470
|
+
let documentCoverage = ""
|
|
471
|
+
let documentIncomplete = false
|
|
472
|
+
let documents = ""
|
|
473
|
+
let documentsEmpty = false
|
|
474
|
+
if (layers.documents) {
|
|
250
475
|
const knowledgeScope = sharedScope(spaceContext)
|
|
251
|
-
const
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
476
|
+
const discoveryFailures = []
|
|
477
|
+
const unavailable = []
|
|
478
|
+
const ids = await discoverKnowledge(backend, log, undefined, knowledgeScope, { onFailure: (name, error) => (Number(error?.status) === 404 ? unavailable : discoveryFailures).push(name) })
|
|
479
|
+
let searchedDocuments = 0
|
|
480
|
+
let documentFailures = 0
|
|
481
|
+
documents =
|
|
255
482
|
ids.length === 0
|
|
256
483
|
? ""
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
484
|
+
: await searchKnowledge(backend, ids.slice(0, 2), query, knowledgeScope, {
|
|
485
|
+
maxChars: Math.max(1500, Math.floor(config.recallChars * (where === "documents" ? 1 : 0.5))),
|
|
486
|
+
onSearch: (_id, ok) => { if (ok) searchedDocuments += 1; else documentFailures += 1 },
|
|
487
|
+
})
|
|
488
|
+
if (!documents && where === "documents" && !discoveryFailures.length && !unavailable.length) documents = "(no accessible wiki assets found)"
|
|
489
|
+
documentCoverage = discoveryFailures.length || documentFailures
|
|
490
|
+
? `Document search incomplete: ${[...discoveryFailures, ...(documentFailures ? ["document search"] : [])].join(", ")} failed; do not infer absence.`
|
|
491
|
+
: ids.length === 0
|
|
492
|
+
? unavailable.length ? "Documents: no indexes found in available catalogs." : "Documents: no accessible wiki or code-graph indexes found."
|
|
493
|
+
: `Documents: searched ${searchedDocuments} of ${ids.length} accessible wiki or code-graph indexes.${searchedDocuments < ids.length ? " For a named document not covered, use mem_wiki." : ""}`
|
|
494
|
+
if (unavailable.length) documentCoverage += ` ${unavailable.join(", ")} unavailable (404).`
|
|
495
|
+
documentIncomplete = Boolean(discoveryFailures.length || documentFailures || unavailable.length || searchedDocuments < ids.length)
|
|
496
|
+
documentsEmpty = ids.length === 0 && discoveryFailures.length === 0 && unavailable.length === 0
|
|
261
497
|
}
|
|
262
498
|
|
|
499
|
+
const label = neutralizeFenceClosers(oneLine(spaceContext ? displayScope(spaceContext, scoped) : `${scoped.team_id}/${scoped.agent_id}`, 200))
|
|
500
|
+
const coverage = `Searched in ${label}: ${searched.length ? searched.join(", ") : "no memory layers"}. ${documentCoverage}`.trim()
|
|
501
|
+
const empty = where === "all" && types.length === 0 && !resolved.granted && census && !census.failed && !withheld.status && !withheld.instruction && !withheld.task && !withheld.redacted && census.records + census.atoms + census.conversations === 0 && documentsEmpty
|
|
502
|
+
const memoryEmpty = where === "all" && types.length === 0 && !resolved.granted && census && !census.failed && !withheld.status && !withheld.instruction && !withheld.task && !withheld.redacted && census.records + census.atoms + census.conversations === 0
|
|
503
|
+
const next = empty ? "Nothing is recorded in this space. Answer that it is not recorded, or continue the requested work. Do not search again in pages, wiki, spaces, commitments or other tools; a named document can be read directly when requested." : memoryEmpty ? "Nothing is recorded in the searched memory layers of this space. Report any document search limitation above; do not claim unsearched documents are empty. Do not repeat the search in pages, wiki, spaces or commitments." : ""
|
|
504
|
+
const memoryBudget = Math.max(100, config.recallChars - coverage.length - 1 - (documents ? documents.length + 2 : 0) - (next ? next.length + 1 : 0))
|
|
505
|
+
const memory = renderMemory(memoryBudget)
|
|
506
|
+
const block = [coverage, next, memory, documents].filter(Boolean).join("\n")
|
|
507
|
+
|
|
263
508
|
log.debug("recall", {
|
|
264
509
|
scope: displayScope(spaceContext, scoped),
|
|
510
|
+
in: where,
|
|
265
511
|
task: Boolean(key),
|
|
512
|
+
records: records.length,
|
|
266
513
|
atoms: atoms.length,
|
|
267
514
|
conversations: conversations.length,
|
|
268
515
|
withheld,
|
|
269
516
|
})
|
|
270
|
-
|
|
517
|
+
const shownScopeNote = neutralizeFenceClosers(oneLine(scopeNote ?? "", 240))
|
|
518
|
+
return { ...text(block), metadata: { memoryRecall: recallResultView({
|
|
519
|
+
coverage, scopeNote: shownScopeNote && block.includes(shownScopeNote) ? shownScopeNote : "",
|
|
520
|
+
records, atoms, conversations, failures, documentIncomplete,
|
|
521
|
+
}) } }
|
|
271
522
|
} catch (error) {
|
|
272
523
|
log.warn("recall failed", error)
|
|
273
524
|
return fail(errorMessage(error))
|
|
@@ -275,3 +526,10 @@ export function registerRecall(editor, toolkit) {
|
|
|
275
526
|
},
|
|
276
527
|
})
|
|
277
528
|
}
|
|
529
|
+
|
|
530
|
+
/** The records `space.stats` counts (`records.by_kind[kind][status]`) for the kinds a recall searched. */
|
|
531
|
+
function recordCount(stats, kinds) {
|
|
532
|
+
const byKind = stats?.records?.by_kind
|
|
533
|
+
if (!byKind || typeof byKind !== "object") return 0
|
|
534
|
+
return kinds.reduce((sum, kind) => sum + Object.values(byKind[kind] ?? {}).reduce((total, count) => total + (Number(count) || 0), 0), 0)
|
|
535
|
+
}
|