dsh-engram 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +331 -0
- package/README.zh.md +276 -0
- package/cordis.patch.yml +10 -0
- package/lib/api.js +418 -0
- package/lib/capture.js +208 -0
- package/lib/client.js +1438 -0
- package/lib/index-block.js +179 -0
- package/lib/index.js +322 -0
- package/lib/settings.js +95 -0
- package/lib/store.js +565 -0
- package/lib/tools.js +471 -0
- package/lib/usage.js +96 -0
- package/lib/util.js +119 -0
- package/package.json +92 -0
package/lib/store.js
ADDED
|
@@ -0,0 +1,565 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-engram: storage layer over DSH's own `ctx.storageDomain`.
|
|
3
|
+
*
|
|
4
|
+
* One domain `dsh_engram` (JSON unit `~/.dsh/storages/dsh_engram.json`) with three
|
|
5
|
+
* tables:
|
|
6
|
+
* - `memories` — the memory pool (kinds: decision/error/procedure/fact/
|
|
7
|
+
* insight/handoff/task), with provenance, signal strength, hit counts,
|
|
8
|
+
* optional entity anchor, and soft-expiry.
|
|
9
|
+
* - `tasks` — ESR-lite entities with a draft→active→stable lifecycle and
|
|
10
|
+
* an evidence set (artifact / evaluation / memoryRefs) required for stable.
|
|
11
|
+
* - `links` — typed entity relations (mini graph, ESR relation flavor).
|
|
12
|
+
* - `entities` — named graph nodes (esr_node): stable symbols for the things
|
|
13
|
+
* a workspace keeps referring to; links and memory entity-anchors point at
|
|
14
|
+
* these ids.
|
|
15
|
+
*
|
|
16
|
+
* No external server, no self-built SQLite, no model calls.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { z } from "zod";
|
|
20
|
+
import { defineDomain, domainTable } from "@deepseek-ai/dsh-storage-domain";
|
|
21
|
+
import { hashText, byRecency, uuid, now, tokenize as tokenizeLocal, scoreRecord as scoreRecordLocal, truncate as truncateText } from "./util.js";
|
|
22
|
+
|
|
23
|
+
export const DOMAIN_VERSION = 1;
|
|
24
|
+
|
|
25
|
+
/** Memory kinds, mapping to index prefixes. */
|
|
26
|
+
const MEMORY_KINDS = [
|
|
27
|
+
"decision",
|
|
28
|
+
"error",
|
|
29
|
+
"procedure",
|
|
30
|
+
"fact",
|
|
31
|
+
"insight",
|
|
32
|
+
"handoff",
|
|
33
|
+
"task",
|
|
34
|
+
];
|
|
35
|
+
|
|
36
|
+
const MemorySchema = z.object({
|
|
37
|
+
/** Stable id (table key). */
|
|
38
|
+
id: z.string().min(1),
|
|
39
|
+
/** Normalized workspace key (exact-cwd authorization). */
|
|
40
|
+
workspace: z.string().min(1),
|
|
41
|
+
kind: z.enum(MEMORY_KINDS),
|
|
42
|
+
/** Compact, information-dense text. */
|
|
43
|
+
text: z.string().min(1),
|
|
44
|
+
/** Free-form routing tags. */
|
|
45
|
+
tags: z.array(z.string()).default([]),
|
|
46
|
+
/** Optional opaque entity anchor (shared with pi-esr convention). */
|
|
47
|
+
entity: z.string().nullable().default(null),
|
|
48
|
+
/** Provenance: the session that (last) wrote it and the log seq observed. */
|
|
49
|
+
sessionId: z.string().min(1),
|
|
50
|
+
seq: z.number().int().nonnegative(),
|
|
51
|
+
createdAt: z.number().int().nonnegative(),
|
|
52
|
+
updatedAt: z.number().int().nonnegative(),
|
|
53
|
+
/** Recall hit count — used to promote low-signal auto-captures into the index. */
|
|
54
|
+
hits: z.number().int().nonnegative().default(0),
|
|
55
|
+
/** 0..1 static signal; index threshold is minIndexSignal. */
|
|
56
|
+
signal: z.number().min(0).max(1).default(0.5),
|
|
57
|
+
status: z.enum(["active", "archived"]).default("active"),
|
|
58
|
+
/** Absolute TTL expiry (epoch ms), null = no expiry. */
|
|
59
|
+
expiresAt: z.number().int().nullable().default(null),
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
const TaskSchema = z.object({
|
|
63
|
+
id: z.string().min(1),
|
|
64
|
+
workspace: z.string().min(1),
|
|
65
|
+
name: z.string().min(1),
|
|
66
|
+
description: z.string().default(""),
|
|
67
|
+
state: z.enum(["draft", "active", "stable"]).default("draft"),
|
|
68
|
+
/** ESR closure evidence — stable requires all three gates. */
|
|
69
|
+
artifact: z.string().nullable().default(null),
|
|
70
|
+
evaluation: z.string().nullable().default(null),
|
|
71
|
+
/** memory_ref ids recorded at closure time. */
|
|
72
|
+
memoryRefs: z.array(z.string()).default([]),
|
|
73
|
+
sessionId: z.string().min(1),
|
|
74
|
+
createdAt: z.number().int().nonnegative(),
|
|
75
|
+
updatedAt: z.number().int().nonnegative(),
|
|
76
|
+
/** When the state last transitioned (maintained by the tools layer). */
|
|
77
|
+
stateChangedAt: z.number().int().nonnegative().default(0),
|
|
78
|
+
/** Set by GC when a stable task passed its retention window. Archived tasks
|
|
79
|
+
* leave the [ESR] surface but stay retrievable (id + name kept). */
|
|
80
|
+
archivedAt: z.number().int().nullable().default(null),
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
const LinkSchema = z.object({
|
|
84
|
+
id: z.string().min(1),
|
|
85
|
+
workspace: z.string().min(1),
|
|
86
|
+
/** Opaque entity ids (pi-esr style). */
|
|
87
|
+
source: z.string().min(1),
|
|
88
|
+
relation: z.string().min(1),
|
|
89
|
+
target: z.string().min(1),
|
|
90
|
+
confidence: z.number().min(0).max(1).default(1),
|
|
91
|
+
sessionId: z.string().min(1),
|
|
92
|
+
createdAt: z.number().int().nonnegative(),
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
const EntitySchema = z.object({
|
|
96
|
+
/** Stable node id (table key); esr_node derives it from the name by default. */
|
|
97
|
+
id: z.string().min(1),
|
|
98
|
+
workspace: z.string().min(1),
|
|
99
|
+
/** Display label (the thing's name). */
|
|
100
|
+
name: z.string().min(1),
|
|
101
|
+
/** Free-form description, e.g. what the node is / why it matters. */
|
|
102
|
+
description: z.string().default(""),
|
|
103
|
+
/** Free-form kind tag: package / service / doc / repo / person / bug ... */
|
|
104
|
+
kind: z.string().default(""),
|
|
105
|
+
sessionId: z.string().min(1),
|
|
106
|
+
createdAt: z.number().int().nonnegative(),
|
|
107
|
+
updatedAt: z.number().int().nonnegative(),
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Per (workspace, day) rollup of agent behaviour: which engram_ and esr_
|
|
112
|
+
* tools the model actually called, failures, and recall mechanics (queries
|
|
113
|
+
* with hits, total hits returned, recall->detail drill-throughs). Feeds
|
|
114
|
+
* the real-data observability panel (`/api/dsh-engram/stats`).
|
|
115
|
+
*/
|
|
116
|
+
const UsageSchema = z.object({
|
|
117
|
+
id: z.string().min(1), // `${workspace}::${day}`
|
|
118
|
+
workspace: z.string().min(1),
|
|
119
|
+
day: z.string().min(1), // YYYY-MM-DD (local)
|
|
120
|
+
counts: z.record(z.string(), z.number().int().nonnegative()).default({}),
|
|
121
|
+
failures: z.number().int().nonnegative().default(0),
|
|
122
|
+
recall: z.record(z.string(), z.number().int().nonnegative()).default({}),
|
|
123
|
+
updatedAt: z.number().int().nonnegative(),
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
const GlobalSchema = z.object({
|
|
127
|
+
initialized: z.boolean(),
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
/** Identity validation (mirrors shipped defineDomain; fails loud at load). */
|
|
131
|
+
export const engramDomainSpec = defineDomain({
|
|
132
|
+
name: "dsh_engram",
|
|
133
|
+
version: DOMAIN_VERSION,
|
|
134
|
+
global: { schema: GlobalSchema, initial: { initialized: true } },
|
|
135
|
+
tables: {
|
|
136
|
+
memories: domainTable(MemorySchema),
|
|
137
|
+
tasks: domainTable(TaskSchema),
|
|
138
|
+
links: domainTable(LinkSchema),
|
|
139
|
+
entities: domainTable(EntitySchema),
|
|
140
|
+
usage: domainTable(UsageSchema),
|
|
141
|
+
},
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
export class EngramError extends Error {
|
|
145
|
+
constructor(code, message) {
|
|
146
|
+
super(message);
|
|
147
|
+
this.code = code;
|
|
148
|
+
this.name = "EngramError";
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** True when `error` carries a harness-style machine code equal to `code`. */
|
|
153
|
+
export function hasCode(error, code) {
|
|
154
|
+
return error instanceof Error && "code" in error && error.code === code;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Open the engram domain over a storage-domain facility. Returns a typed handle
|
|
159
|
+
* with synchronous reads and async writes (both are durable on the backend).
|
|
160
|
+
*/
|
|
161
|
+
export async function openEngramDomain(facility) {
|
|
162
|
+
const opened = await facility.open(engramDomainSpec);
|
|
163
|
+
const memories = opened.table("memories");
|
|
164
|
+
const tasks = opened.table("tasks");
|
|
165
|
+
const links = opened.table("links");
|
|
166
|
+
const entities = opened.table("entities");
|
|
167
|
+
const usage = opened.table("usage");
|
|
168
|
+
|
|
169
|
+
const allMemories = (workspace) =>
|
|
170
|
+
[...memories.entries()]
|
|
171
|
+
.map(([, m]) => m)
|
|
172
|
+
.filter((m) => m.workspace === workspace && m.status === "active");
|
|
173
|
+
|
|
174
|
+
const allEntities = (workspace) =>
|
|
175
|
+
[...entities.entries()]
|
|
176
|
+
.map(([, e]) => e)
|
|
177
|
+
.filter((e) => e.workspace === workspace);
|
|
178
|
+
|
|
179
|
+
const activeTasks = (workspace) =>
|
|
180
|
+
[...tasks.entries()]
|
|
181
|
+
.map(([, t]) => t)
|
|
182
|
+
.filter((t) => t.workspace === workspace && t.state !== "stable" && (t.archivedAt === null || t.archivedAt === void 0));
|
|
183
|
+
|
|
184
|
+
const notArchivedTask = (t) => t.archivedAt === null || t.archivedAt === void 0;
|
|
185
|
+
|
|
186
|
+
/** Skip (but do not mutate during sync reads) memories past their TTL. */
|
|
187
|
+
const notExpiredNow = (m) => m.expiresAt === null || m.expiresAt > now();
|
|
188
|
+
|
|
189
|
+
return {
|
|
190
|
+
workspaceEntries(workspace) {
|
|
191
|
+
return {
|
|
192
|
+
memories: allMemories(workspace).length,
|
|
193
|
+
tasks: activeTasks(workspace).length,
|
|
194
|
+
links: [...links.entries()].filter(([, l]) => l.workspace === workspace).length,
|
|
195
|
+
nodes: allEntities(workspace).length,
|
|
196
|
+
};
|
|
197
|
+
},
|
|
198
|
+
// ── memories ──────────────────────────────────────────────────────
|
|
199
|
+
getMemory(workspace, id) {
|
|
200
|
+
const m = memories.get(id);
|
|
201
|
+
if (m === void 0 || m.workspace !== workspace || m.status !== "active" || !notExpiredNow(m)) return void 0;
|
|
202
|
+
return m;
|
|
203
|
+
},
|
|
204
|
+
listMemories(workspace, limit = 50) {
|
|
205
|
+
return allMemories(workspace)
|
|
206
|
+
.filter(notExpiredNow)
|
|
207
|
+
.sort(byRecency)
|
|
208
|
+
.slice(0, Math.max(1, Math.min(200, limit)));
|
|
209
|
+
},
|
|
210
|
+
/**
|
|
211
|
+
* GUI-oriented memory search spanning one workspace or (when `workspace`
|
|
212
|
+
* is falsy) every workspace. Optional kind/status filters; `q` reuses the
|
|
213
|
+
* deterministic recall scorer.
|
|
214
|
+
*/
|
|
215
|
+
searchMemories({ workspace, q, kind, status, limit = 100 } = {}) {
|
|
216
|
+
const bounded = Math.max(1, Math.min(500, Math.trunc(limit) || 100));
|
|
217
|
+
let pool = [...memories.entries()].map(([, m]) => m);
|
|
218
|
+
if (workspace) pool = pool.filter((m) => m.workspace === workspace);
|
|
219
|
+
// Explicitly browsing archived entries must see TTL-expired ones too
|
|
220
|
+
// (GC archives expired memories; the archive view is their pointer).
|
|
221
|
+
if (status !== "archived") pool = pool.filter(notExpiredNow);
|
|
222
|
+
if (kind) pool = pool.filter((m) => m.kind === kind);
|
|
223
|
+
if (status) pool = pool.filter((m) => m.status === status);
|
|
224
|
+
if (q !== void 0 && String(q).trim().length > 0) {
|
|
225
|
+
const trimmed = String(q).trim();
|
|
226
|
+
const tokens = tokenizeLocal(trimmed);
|
|
227
|
+
const phraseBoost = /\s/.test(trimmed);
|
|
228
|
+
const folded = trimmed.toLowerCase();
|
|
229
|
+
return pool
|
|
230
|
+
.map((m) => ({ m, score: scoreRecordLocal(m, tokens, folded, phraseBoost) }))
|
|
231
|
+
.filter((e) => e.score > 0)
|
|
232
|
+
.sort((a, b) => b.score - a.score || byRecency(a.m, b.m))
|
|
233
|
+
.slice(0, bounded)
|
|
234
|
+
.map((e) => e.m);
|
|
235
|
+
}
|
|
236
|
+
return pool.sort(byRecency).slice(0, bounded);
|
|
237
|
+
},
|
|
238
|
+
/** Cross-workspace counts and kind histogram for the GUI overview. */
|
|
239
|
+
summarize() {
|
|
240
|
+
const workspaces = {};
|
|
241
|
+
const kinds = {};
|
|
242
|
+
let memoryCount = 0;
|
|
243
|
+
let taskCount = 0;
|
|
244
|
+
let linkCount = 0;
|
|
245
|
+
let nodeCount = 0;
|
|
246
|
+
const zeroCounts = () => ({ memories: 0, tasks: 0, links: 0, nodes: 0 });
|
|
247
|
+
for (const [, m] of memories.entries()) {
|
|
248
|
+
const ws = m.workspace;
|
|
249
|
+
workspaces[ws] ??= zeroCounts();
|
|
250
|
+
if (m.status === "active") {
|
|
251
|
+
workspaces[ws].memories += 1;
|
|
252
|
+
memoryCount += 1;
|
|
253
|
+
kinds[m.kind] = (kinds[m.kind] ?? 0) + 1;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
for (const [, t] of tasks.entries()) {
|
|
257
|
+
workspaces[t.workspace] ??= zeroCounts();
|
|
258
|
+
if (t.state !== "stable" && notArchivedTask(t)) {
|
|
259
|
+
workspaces[t.workspace].tasks += 1;
|
|
260
|
+
taskCount += 1;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
for (const [, l] of links.entries()) {
|
|
264
|
+
workspaces[l.workspace] ??= zeroCounts();
|
|
265
|
+
workspaces[l.workspace].links += 1;
|
|
266
|
+
linkCount += 1;
|
|
267
|
+
}
|
|
268
|
+
for (const [, e] of entities.entries()) {
|
|
269
|
+
workspaces[e.workspace] ??= zeroCounts();
|
|
270
|
+
workspaces[e.workspace].nodes += 1;
|
|
271
|
+
nodeCount += 1;
|
|
272
|
+
}
|
|
273
|
+
return { workspaces, kinds, totals: { memories: memoryCount, tasks: taskCount, links: linkCount, nodes: nodeCount } };
|
|
274
|
+
},
|
|
275
|
+
/** Hard delete (GUI only; provenance stays in the session log). */
|
|
276
|
+
async deleteMemory(workspace, id) {
|
|
277
|
+
const m = memories.get(id);
|
|
278
|
+
if (m === void 0 || m.workspace !== workspace) return false;
|
|
279
|
+
await memories.delete(id);
|
|
280
|
+
return true;
|
|
281
|
+
},
|
|
282
|
+
/**
|
|
283
|
+
* Store one memory. Exact-duplicate text in the same workspace refreshes
|
|
284
|
+
* `updatedAt` (and returns the existing id) instead of piling up.
|
|
285
|
+
*/
|
|
286
|
+
async storeMemory(entry, config) {
|
|
287
|
+
const text = String(entry.text).trim();
|
|
288
|
+
if (text.length === 0) throw new EngramError("ENGRAM_INVALID_ARGS", "memory text must not be empty");
|
|
289
|
+
if ([...text].length > config.maxMemoryChars) {
|
|
290
|
+
throw new EngramError("ENGRAM_CAP_EXCEEDED", `memory text is ${[...text].length} characters, over the cap of ${config.maxMemoryChars}`);
|
|
291
|
+
}
|
|
292
|
+
const dup = allMemories(entry.workspace).find((m) => hashText(m.text) === hashText(text) && m.kind === (entry.kind ?? "fact"));
|
|
293
|
+
if (dup !== void 0) {
|
|
294
|
+
const next = { ...dup, updatedAt: now() };
|
|
295
|
+
await memories.put(dup.id, next);
|
|
296
|
+
return { stored: next, duplicated: true, id: dup.id };
|
|
297
|
+
}
|
|
298
|
+
if (allMemories(entry.workspace).length >= config.maxMemoriesPerWorkspace) {
|
|
299
|
+
throw new EngramError("ENGRAM_CAP_EXCEEDED", `workspace already holds ${config.maxMemoriesPerWorkspace} active memories (the cap)`);
|
|
300
|
+
}
|
|
301
|
+
const expiresAt =
|
|
302
|
+
entry.expiresAt !== void 0 && entry.expiresAt !== null
|
|
303
|
+
? entry.expiresAt
|
|
304
|
+
: config.expireDays > 0
|
|
305
|
+
? now() + config.expireDays * 24 * 60 * 60 * 1000
|
|
306
|
+
: null;
|
|
307
|
+
const stored = {
|
|
308
|
+
id: uuid(),
|
|
309
|
+
workspace: entry.workspace,
|
|
310
|
+
kind: entry.kind ?? "fact",
|
|
311
|
+
text,
|
|
312
|
+
tags: entry.tags ?? [],
|
|
313
|
+
entity: entry.entity ?? null,
|
|
314
|
+
sessionId: entry.sessionId,
|
|
315
|
+
seq: entry.seq ?? 0,
|
|
316
|
+
createdAt: now(),
|
|
317
|
+
updatedAt: now(),
|
|
318
|
+
hits: 0,
|
|
319
|
+
signal: typeof entry.signal === "number" ? Math.max(0, Math.min(1, entry.signal)) : 0.5,
|
|
320
|
+
status: "active",
|
|
321
|
+
expiresAt,
|
|
322
|
+
};
|
|
323
|
+
await memories.put(stored.id, stored);
|
|
324
|
+
return { stored, duplicated: false, id: stored.id };
|
|
325
|
+
},
|
|
326
|
+
async touchMemory(workspace, id) {
|
|
327
|
+
const m = memories.get(id);
|
|
328
|
+
if (m === void 0 || m.workspace !== workspace || !notExpiredNow(m)) return void 0;
|
|
329
|
+
const next = { ...m, hits: (m.hits ?? 0) + 1, updatedAt: m.updatedAt };
|
|
330
|
+
await memories.put(id, next);
|
|
331
|
+
return next;
|
|
332
|
+
},
|
|
333
|
+
/** Deterministic recall: tag-exact > text-substring > fuzzy, recency ties. */
|
|
334
|
+
recall(workspace, query, limit = 20) {
|
|
335
|
+
const active = allMemories(workspace).filter(notExpiredNow);
|
|
336
|
+
const bounded = Math.max(1, Math.min(50, Math.trunc(limit) || 20));
|
|
337
|
+
if (query !== void 0 && String(query).trim().length > 0) {
|
|
338
|
+
const trimmed = String(query).trim();
|
|
339
|
+
const tokens = tokenizeLocal(trimmed);
|
|
340
|
+
const phraseBoost = /\s/.test(trimmed);
|
|
341
|
+
const folded = trimmed.toLowerCase();
|
|
342
|
+
return active
|
|
343
|
+
.map((m) => ({ m, score: scoreRecordLocal(m, tokens, folded, phraseBoost) }))
|
|
344
|
+
.filter((e) => e.score > 0)
|
|
345
|
+
.sort((a, b) => b.score - a.score || byRecency(a.m, b.m))
|
|
346
|
+
.slice(0, bounded)
|
|
347
|
+
.map((e) => e.m);
|
|
348
|
+
}
|
|
349
|
+
return active.sort(byRecency).slice(0, bounded);
|
|
350
|
+
},
|
|
351
|
+
timeline(workspace, entity, limit = 30) {
|
|
352
|
+
const active = allMemories(workspace).filter(notExpiredNow);
|
|
353
|
+
const bounded = Math.max(1, Math.min(100, Math.trunc(limit) || 30));
|
|
354
|
+
if (entity === void 0 || String(entity).trim().length === 0) {
|
|
355
|
+
return active.sort((a, b) => (a.createdAt ?? 0) - (b.createdAt ?? 0)).slice(0, bounded);
|
|
356
|
+
}
|
|
357
|
+
const key = String(entity).trim();
|
|
358
|
+
return active
|
|
359
|
+
.filter((m) => m.entity !== null && (m.entity === key || String(m.entity).includes(key) || key.includes(String(m.entity))))
|
|
360
|
+
.sort((a, b) => (a.createdAt ?? 0) - (b.createdAt ?? 0))
|
|
361
|
+
.slice(0, bounded);
|
|
362
|
+
},
|
|
363
|
+
async archiveMemory(workspace, id) {
|
|
364
|
+
const m = memories.get(id);
|
|
365
|
+
if (m === void 0 || m.workspace !== workspace || m.status === "archived") return false;
|
|
366
|
+
await memories.put(id, { ...m, status: "archived", updatedAt: now() });
|
|
367
|
+
return true;
|
|
368
|
+
},
|
|
369
|
+
/** Lazy TTL sweep (write path only; async). Returns archived count. */
|
|
370
|
+
async expireSweep(workspace) {
|
|
371
|
+
let n = 0;
|
|
372
|
+
for (const [, m] of memories.entries()) {
|
|
373
|
+
if (m.workspace === workspace && m.status === "active" && m.expiresAt !== null && m.expiresAt <= now()) {
|
|
374
|
+
await memories.put(m.id, { ...m, status: "archived", updatedAt: now() });
|
|
375
|
+
n += 1;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
return n;
|
|
379
|
+
},
|
|
380
|
+
// ── tasks ─────────────────────────────────────────────────────────
|
|
381
|
+
getTask(workspace, id) {
|
|
382
|
+
const t = tasks.get(id);
|
|
383
|
+
return t === void 0 || t.workspace !== workspace ? void 0 : t;
|
|
384
|
+
},
|
|
385
|
+
listTasks(workspace, { includeStable = false } = {}) {
|
|
386
|
+
return [...tasks.entries()]
|
|
387
|
+
.map(([, t]) => t)
|
|
388
|
+
.filter((t) => t.workspace === workspace && notArchivedTask(t) && (includeStable || t.state !== "stable"))
|
|
389
|
+
.sort(byRecency);
|
|
390
|
+
},
|
|
391
|
+
async putTask(task) {
|
|
392
|
+
if ([...task.name].length > 200) throw new EngramError("ENGRAM_CAP_EXCEEDED", "task name is over 200 characters");
|
|
393
|
+
if (task.id !== task.id) throw new EngramError("ENGRAM_INVALID_ARGS", "task id mismatch");
|
|
394
|
+
await tasks.put(task.id, task);
|
|
395
|
+
return task;
|
|
396
|
+
},
|
|
397
|
+
/** Cap is enforced by the tools layer (needs config); expose the count here. */
|
|
398
|
+
activeTaskCount(workspace) {
|
|
399
|
+
return activeTasks(workspace).length;
|
|
400
|
+
},
|
|
401
|
+
async archiveTask(workspace, id) {
|
|
402
|
+
const t = tasks.get(id);
|
|
403
|
+
if (t === void 0 || t.workspace !== workspace || t.archivedAt !== null) return false;
|
|
404
|
+
await tasks.put(id, { ...t, archivedAt: now() });
|
|
405
|
+
return true;
|
|
406
|
+
},
|
|
407
|
+
/**
|
|
408
|
+
* pi-esr GC, adapted to the memory store. Mechanical, deterministic,
|
|
409
|
+
* working-set-protected, archive-only (never hard-deletes — everything
|
|
410
|
+
* archived stays re-fetchable, honouring pointer-salience):
|
|
411
|
+
* - TTL-expired memories beyond their `expiresAt` are archived;
|
|
412
|
+
* - over-cap workspaces evict the lowest-value non-protected memories
|
|
413
|
+
* (hits asc, signal asc, age asc) until at the cap;
|
|
414
|
+
* - stable tasks past `config.gcStableRetentionDays` become `archived`;
|
|
415
|
+
* - links whose BOTH endpoints are gone (no active memory entity, no
|
|
416
|
+
* surviving task id) are removed (dangling graph edges).
|
|
417
|
+
* The working set is never touched: memories referenced by an ACTIVE task
|
|
418
|
+
* (`memoryRefs`), memories with kind `task`, task-anchored entities, and
|
|
419
|
+
* already-indexed hits (`hits >= promoteHits`) are all protected.
|
|
420
|
+
* @param workspace - workspace key, or falsy to sweep every open workspace.
|
|
421
|
+
* @param config - effective plugin config (gcStableRetentionDays, promoteHits, maxMemoriesPerWorkspace).
|
|
422
|
+
* @param opts - `dryRun: true` computes the full report without writing.
|
|
423
|
+
* @returns a report of what would be / was archived and why.
|
|
424
|
+
*/
|
|
425
|
+
async gc(workspace, config, { dryRun = false } = {}) {
|
|
426
|
+
const conf = config ?? {};
|
|
427
|
+
const retentionMs = (conf.gcStableRetentionDays ?? 120) * 24 * 60 * 60 * 1000;
|
|
428
|
+
const cap = conf.maxMemoriesPerWorkspace ?? 2000;
|
|
429
|
+
const promoteHits = conf.promoteHits ?? 3;
|
|
430
|
+
const t0 = now();
|
|
431
|
+
|
|
432
|
+
const workspaces = workspace
|
|
433
|
+
? [workspace]
|
|
434
|
+
: [...new Set([...memories.entries(), ...tasks.entries(), ...links.entries()].map(([, v]) => v.workspace))];
|
|
435
|
+
|
|
436
|
+
const report = {
|
|
437
|
+
dryRun,
|
|
438
|
+
workspaces,
|
|
439
|
+
protectedMemories: 0,
|
|
440
|
+
archivedMemories: [],
|
|
441
|
+
archivedTasks: [],
|
|
442
|
+
removedLinks: [],
|
|
443
|
+
};
|
|
444
|
+
|
|
445
|
+
for (const ws of workspaces) {
|
|
446
|
+
const allMem = [...memories.entries()].map(([, m]) => m).filter((m) => m.workspace === ws);
|
|
447
|
+
const activeMem = allMem.filter((m) => m.status === "active");
|
|
448
|
+
const wsTasks = [...tasks.entries()].map(([, t]) => t).filter((t) => t.workspace === ws);
|
|
449
|
+
const wsLinks = [...links.entries()].map(([, l]) => l).filter((l) => l.workspace === ws);
|
|
450
|
+
|
|
451
|
+
// ── working set (pi-esr: working-set-protected) ─────────────────
|
|
452
|
+
const activeTaskIds = new Set(wsTasks.filter((t) => t.state === "active" && notArchivedTask(t)).map((t) => t.id));
|
|
453
|
+
const activeRefs = new Set(wsTasks.filter((t) => t.state === "active" && notArchivedTask(t)).flatMap((t) => t.memoryRefs ?? []));
|
|
454
|
+
const isProtected = (m) =>
|
|
455
|
+
m.kind === "task" ||
|
|
456
|
+
(m.hits ?? 0) >= promoteHits ||
|
|
457
|
+
activeRefs.has(m.id) ||
|
|
458
|
+
(m.entity !== null && activeTaskIds.has(m.entity));
|
|
459
|
+
|
|
460
|
+
// ── stale / over-cap memory archiving ───────────────────────────
|
|
461
|
+
const expired = [];
|
|
462
|
+
const candidates = [];
|
|
463
|
+
for (const m of activeMem) {
|
|
464
|
+
if (isProtected(m)) {
|
|
465
|
+
report.protectedMemories += 1;
|
|
466
|
+
continue;
|
|
467
|
+
}
|
|
468
|
+
if (m.expiresAt !== null && m.expiresAt <= t0) expired.push(m);
|
|
469
|
+
else candidates.push(m);
|
|
470
|
+
}
|
|
471
|
+
for (const m of expired) {
|
|
472
|
+
if (!dryRun) await memories.put(m.id, { ...m, status: "archived", updatedAt: t0 });
|
|
473
|
+
report.archivedMemories.push({ id: m.id, kind: m.kind, text: truncateText(m.text, 60), reason: "expired", workspace: ws });
|
|
474
|
+
}
|
|
475
|
+
const survivable = activeMem.length - report.archivedMemories.filter((e) => e.workspace === ws).length;
|
|
476
|
+
if (survivable > cap) {
|
|
477
|
+
const over = survivable - cap;
|
|
478
|
+
const evict = candidates
|
|
479
|
+
.slice()
|
|
480
|
+
.sort((a, b) => (a.hits ?? 0) - (b.hits ?? 0) || (a.signal ?? 0) - (b.signal ?? 0) || (a.createdAt ?? 0) - (b.createdAt ?? 0))
|
|
481
|
+
.slice(0, over);
|
|
482
|
+
for (const m of evict) {
|
|
483
|
+
if (!dryRun) await memories.put(m.id, { ...m, status: "archived", updatedAt: t0 });
|
|
484
|
+
report.archivedMemories.push({ id: m.id, kind: m.kind, text: truncateText(m.text, 60), reason: "cap", workspace: ws });
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
// ── stable-task retention (stable-task-evictable, archive-only) ─
|
|
489
|
+
const retentionCutoff = t0 - retentionMs;
|
|
490
|
+
for (const t of wsTasks) {
|
|
491
|
+
if (t.state === "stable" && notArchivedTask(t) && t.updatedAt <= retentionCutoff) {
|
|
492
|
+
if (!dryRun) await tasks.put(t.id, { ...t, archivedAt: t0 });
|
|
493
|
+
report.archivedTasks.push({ id: t.id, name: truncateText(t.name, 60), reason: "stable-retention", workspace: ws });
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
// ── dangling links (both endpoints gone) ────────────────────────
|
|
498
|
+
const liveEntities = new Set();
|
|
499
|
+
for (const m of activeMem) if (m.entity !== null) liveEntities.add(m.entity);
|
|
500
|
+
for (const t of wsTasks) if (notArchivedTask(t)) liveEntities.add(t.id);
|
|
501
|
+
for (const e of allEntities(ws)) liveEntities.add(e.id);
|
|
502
|
+
for (const l of wsLinks) {
|
|
503
|
+
if (!liveEntities.has(l.source) && !liveEntities.has(l.target)) {
|
|
504
|
+
if (!dryRun) await links.delete(l.id);
|
|
505
|
+
report.removedLinks.push({ id: l.id, source: l.source, relation: l.relation, target: l.target, reason: "dangling", workspace: ws });
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
return report;
|
|
511
|
+
},
|
|
512
|
+
// ── links ─────────────────────────────────────────────────────────
|
|
513
|
+
addLink(link) {
|
|
514
|
+
return links.put(link.id, link);
|
|
515
|
+
},
|
|
516
|
+
linksFor(workspace, entity) {
|
|
517
|
+
return [...links.entries()]
|
|
518
|
+
.map(([, l]) => l)
|
|
519
|
+
.filter((l) => l.workspace === workspace && (l.source === entity || l.target === entity))
|
|
520
|
+
.sort((a, b) => (a.createdAt ?? 0) - (b.createdAt ?? 0));
|
|
521
|
+
},
|
|
522
|
+
allLinks(workspace) {
|
|
523
|
+
return [...links.entries()].map(([, l]) => l).filter((l) => l.workspace === workspace);
|
|
524
|
+
},
|
|
525
|
+
// ── entities (esr_node) ───────────────────────────────────────────
|
|
526
|
+
getEntity(workspace, id) {
|
|
527
|
+
const e = entities.get(id);
|
|
528
|
+
if (e === void 0 || e.workspace !== workspace) return void 0;
|
|
529
|
+
return e;
|
|
530
|
+
},
|
|
531
|
+
listEntities(workspace) {
|
|
532
|
+
return allEntities(workspace).sort((a, b) => (a.createdAt ?? 0) - (b.createdAt ?? 0));
|
|
533
|
+
},
|
|
534
|
+
putEntity(entity) {
|
|
535
|
+
return entities.put(entity.id, entity);
|
|
536
|
+
},
|
|
537
|
+
async removeEntity(workspace, id) {
|
|
538
|
+
const e = entities.get(id);
|
|
539
|
+
if (e === void 0 || e.workspace !== workspace) return false;
|
|
540
|
+
await entities.delete(id);
|
|
541
|
+
return true;
|
|
542
|
+
},
|
|
543
|
+
// ── usage / observability ────────────────────────────────────────
|
|
544
|
+
/** Merge one tool-call event into the (workspace, day) rollup row. */
|
|
545
|
+
async bumpUsage(workspace, day, { counts = {}, failures = 0, recall = {} }) {
|
|
546
|
+
const id = `${workspace}::${day}`;
|
|
547
|
+
const cur = usage.get(id);
|
|
548
|
+
const row = cur ?? { id, workspace, day, counts: {}, failures: 0, recall: {}, updatedAt: 0 };
|
|
549
|
+
for (const [k, v] of Object.entries(counts)) row.counts[k] = (row.counts[k] ?? 0) + v;
|
|
550
|
+
row.failures += failures;
|
|
551
|
+
for (const [k, v] of Object.entries(recall)) row.recall[k] = (row.recall[k] ?? 0) + v;
|
|
552
|
+
row.updatedAt = Date.now();
|
|
553
|
+
await usage.put(id, row);
|
|
554
|
+
return row;
|
|
555
|
+
},
|
|
556
|
+
/** Usage rollup rows, oldest first (all workspaces when workspace is undefined). */
|
|
557
|
+
usageRows(workspace) {
|
|
558
|
+
const all = [...usage.entries()].map(([, r]) => r);
|
|
559
|
+
return (workspace ? all.filter((r) => r.workspace === workspace) : all).sort((a, b) => a.day.localeCompare(b.day));
|
|
560
|
+
},
|
|
561
|
+
close() {
|
|
562
|
+
return opened.close();
|
|
563
|
+
},
|
|
564
|
+
};
|
|
565
|
+
}
|