dsh-engram 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +283 -27
- package/README.zh.md +184 -16
- package/cordis.patch.yml +6 -0
- package/docs/CONTEXT-GC-GUIDE.zh.md +192 -0
- package/docs/PROPOSAL-agentmemory.zh.md +108 -0
- package/docs/PROPOSAL-beads.zh.md +81 -0
- package/docs/PROPOSAL-contextgc.zh.md +188 -0
- package/docs/PROPOSAL-esrtrigger.zh.md +199 -0
- package/docs/PROPOSAL-hindsight.zh.md +156 -0
- package/lib/api.js +76 -2
- package/lib/capture.js +132 -4
- package/lib/client.js +45 -2
- package/lib/compaction.js +2 -18
- package/lib/context-gc.js +234 -2
- package/lib/goal-capture.js +89 -0
- package/lib/index-block.js +96 -1
- package/lib/index.js +233 -5
- package/lib/redact.js +48 -0
- package/lib/session-cjk.js +159 -0
- package/lib/settings.js +13 -0
- package/lib/status.js +57 -0
- package/lib/store.js +411 -6
- package/lib/todo-sink.js +146 -0
- package/lib/tools.js +98 -40
- package/lib/trigger.js +10 -8
- package/lib/usage.js +147 -5
- package/lib/web-provision.js +430 -0
- package/package.json +13 -2
- package/scripts/dsh-engram.mjs +268 -0
- package/scripts/engram-web-compaction.mjs +7 -0
- package/scripts/prepublish-check.mjs +106 -0
- package/scripts/setup-links.mjs +241 -0
package/lib/store.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* dsh-engram: storage layer over DSH's own `ctx.storageDomain`.
|
|
3
3
|
*
|
|
4
|
-
* One domain `dsh_engram` (JSON unit `~/.dsh/storages/dsh_engram.json`) with
|
|
4
|
+
* One domain `dsh_engram` (JSON unit `~/.dsh/storages/dsh_engram.json`) with four
|
|
5
5
|
* tables:
|
|
6
6
|
* - `memories` — the memory pool (kinds: decision/error/procedure/fact/
|
|
7
7
|
* insight/handoff/task), with provenance, signal strength, hit counts,
|
|
@@ -22,6 +22,7 @@ import path from "node:path";
|
|
|
22
22
|
import { defineDomain, domainTable } from "@deepseek-ai/dsh-storage-domain";
|
|
23
23
|
import { hashText, byRecency, uuid, now, truncate as truncateText, bm25Rank, tokenize as tokenizeLocal, dedupeBySession } from "./util.js";
|
|
24
24
|
import { deriveObservations } from "./obs.js";
|
|
25
|
+
import { redactText } from "./redact.js";
|
|
25
26
|
|
|
26
27
|
export const DOMAIN_VERSION = 1;
|
|
27
28
|
|
|
@@ -64,6 +65,14 @@ const MemorySchema = z.object({
|
|
|
64
65
|
status: z.enum(["active", "archived"]).default("active"),
|
|
65
66
|
/** Absolute TTL expiry (epoch ms), null = no expiry. */
|
|
66
67
|
expiresAt: z.number().int().nullable().default(null),
|
|
68
|
+
/** Memory id this row supersedes (newer statement replaces older truth). */
|
|
69
|
+
supersedes: z.string().nullable().default(null),
|
|
70
|
+
/** Memory id this row directly contradicts (conflicting claim, keep both). */
|
|
71
|
+
contradicts: z.string().nullable().default(null),
|
|
72
|
+
/** Optional provenance source class: manual | session | report | upstream | handoff. */
|
|
73
|
+
source: z.string().nullable().default(null),
|
|
74
|
+
/** Optional trigger/condition under which this memory should be recalled. */
|
|
75
|
+
conditions: z.string().nullable().default(null),
|
|
67
76
|
});
|
|
68
77
|
|
|
69
78
|
const TaskSchema = z.object({
|
|
@@ -201,6 +210,72 @@ function repeatOverlap(previous, incoming) {
|
|
|
201
210
|
return repeatOverlapRatio(previous, incoming) >= REPEAT_OVERLAP;
|
|
202
211
|
}
|
|
203
212
|
|
|
213
|
+
/**
|
|
214
|
+
* Validate a memory-to-memory supersedes/contradicts target: must be a real
|
|
215
|
+
* memory in the same workspace (fail loud on typos / cross-workspace refs).
|
|
216
|
+
*/
|
|
217
|
+
function validateRelationTarget(workspace, target, kind, memories) {
|
|
218
|
+
if (target === null || target === void 0 || target === "") return null;
|
|
219
|
+
const mem = memories.find((m) => m.id === target);
|
|
220
|
+
if (mem === void 0 || mem.workspace !== workspace) {
|
|
221
|
+
throw new EngramError("ENGRAM_INVALID_ARGS", `${kind} target memory "${target}" does not exist in this workspace`);
|
|
222
|
+
}
|
|
223
|
+
return target;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Build supersede/contradiction lookup maps from a memory pool (deterministic). */
|
|
227
|
+
function memoryRelationMaps(pool) {
|
|
228
|
+
const supersededBy = new Map(); // target id -> newer row
|
|
229
|
+
const contradictedBy = new Map(); // target id -> rows[]
|
|
230
|
+
for (const m of pool) {
|
|
231
|
+
if (m.supersedes !== null && m.supersedes !== void 0) supersededBy.set(m.supersedes, m);
|
|
232
|
+
if (m.contradicts !== null && m.contradicts !== void 0) {
|
|
233
|
+
const list = contradictedBy.get(m.contradicts) ?? [];
|
|
234
|
+
list.push(m);
|
|
235
|
+
contradictedBy.set(m.contradicts, list);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
return { supersededBy, contradictedBy };
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Conservative replacement/negation cues that justify an auto-supersede edge.
|
|
243
|
+
* Kept explicit and small on purpose: a plain fact update must NOT silently
|
|
244
|
+
* stale-mark its sibling — only "replaced / switched / no longer / 改用 / 不再"
|
|
245
|
+
* style statements do.
|
|
246
|
+
*/
|
|
247
|
+
const SUPERSEDE_CUES = [
|
|
248
|
+
"supersedes", "replaces", "replaced", "deprecated", "no longer", "instead",
|
|
249
|
+
"switched", "migrated", "dropped", "removed",
|
|
250
|
+
"改用", "换成", "切换", "改为", "取代", "替代", "迁移", "废除", "弃用", "不再",
|
|
251
|
+
];
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* Deterministic auto-supersede candidate (config.autoSupersede, default OFF).
|
|
255
|
+
* Fires only when ALL hold:
|
|
256
|
+
* - the new entry has a non-empty entity anchor,
|
|
257
|
+
* - its text carries a replacement/negation cue (SUPERSEDE_CUES),
|
|
258
|
+
* - an older ACTIVE memory shares that entity AND kind AND ≥1 token with it.
|
|
259
|
+
* The most recent such memory is superseded; already-superseded rows and
|
|
260
|
+
* cross-kind/entity rows are never picked. Zero LLM, conservative by design.
|
|
261
|
+
*/
|
|
262
|
+
function autoSupersedeCandidate(all, entry, text) {
|
|
263
|
+
const kind = entry.kind ?? "fact";
|
|
264
|
+
if (typeof entry.entity !== "string" || entry.entity.length === 0) return null;
|
|
265
|
+
const lowered = text.toLowerCase();
|
|
266
|
+
if (!SUPERSEDE_CUES.some((cue) => lowered.includes(cue))) return null;
|
|
267
|
+
const newTokens = new Set(tokenizeLocal(text));
|
|
268
|
+
let best = null;
|
|
269
|
+
for (const m of all) {
|
|
270
|
+
if (m.kind !== kind || m.entity !== entry.entity || m.status !== "active") continue;
|
|
271
|
+
if (m.supersedes !== null) continue;
|
|
272
|
+
const overlap = tokenizeLocal(m.text).some((t) => newTokens.has(t));
|
|
273
|
+
if (!overlap) continue;
|
|
274
|
+
if (best === null || (m.updatedAt ?? m.createdAt ?? 0) > (best.updatedAt ?? best.createdAt ?? 0)) best = m;
|
|
275
|
+
}
|
|
276
|
+
return best;
|
|
277
|
+
}
|
|
278
|
+
|
|
204
279
|
/** Identity validation (mirrors shipped defineDomain; fails loud at load). */
|
|
205
280
|
export const engramDomainSpec = defineDomain({
|
|
206
281
|
name: "dsh_engram",
|
|
@@ -290,7 +365,7 @@ export async function openEngramDomain(facility) {
|
|
|
290
365
|
* is falsy) every workspace. Optional kind/status filters; `q` reuses the
|
|
291
366
|
* deterministic recall scorer.
|
|
292
367
|
*/
|
|
293
|
-
searchMemories({ workspace, q, kind, status, limit = 100 } = {}) {
|
|
368
|
+
searchMemories({ workspace, q, kind, tag, status, limit = 100 } = {}) {
|
|
294
369
|
const bounded = Math.max(1, Math.min(500, Math.trunc(limit) || 100));
|
|
295
370
|
let pool = [...memories.entries()].map(([, m]) => m);
|
|
296
371
|
if (workspace) pool = pool.filter((m) => m.workspace === workspace);
|
|
@@ -298,6 +373,7 @@ export async function openEngramDomain(facility) {
|
|
|
298
373
|
// (GC archives expired memories; the archive view is their pointer).
|
|
299
374
|
if (status !== "archived") pool = pool.filter(notExpiredNow);
|
|
300
375
|
if (kind) pool = pool.filter((m) => m.kind === kind);
|
|
376
|
+
if (tag) pool = pool.filter((m) => (m.tags ?? []).includes(tag));
|
|
301
377
|
if (status) pool = pool.filter((m) => m.status === status);
|
|
302
378
|
if (q !== void 0 && String(q).trim().length > 0) {
|
|
303
379
|
return bm25Rank(pool, String(q).trim(), bounded);
|
|
@@ -362,12 +438,19 @@ export async function openEngramDomain(facility) {
|
|
|
362
438
|
* near-repeat of a past ERROR re-warms that entry (failure revival).
|
|
363
439
|
*/
|
|
364
440
|
async storeMemory(entry, config) {
|
|
365
|
-
|
|
441
|
+
// Redact secrets before anything else: before the char cap, before the
|
|
442
|
+
// dedup hash, and before the revive overlap ratio — so sensitive text
|
|
443
|
+
// never lands in storage and two identical secrets collapse the same
|
|
444
|
+
// way as two identical plain texts.
|
|
445
|
+
const text = redactText(String(entry.text).trim());
|
|
366
446
|
if (text.length === 0) throw new EngramError("ENGRAM_INVALID_ARGS", "memory text must not be empty");
|
|
367
447
|
if ([...text].length > config.maxMemoryChars) {
|
|
368
448
|
throw new EngramError("ENGRAM_CAP_EXCEEDED", `memory text is ${[...text].length} characters, over the cap of ${config.maxMemoryChars}`);
|
|
369
449
|
}
|
|
370
|
-
const
|
|
450
|
+
const all = allMemories(entry.workspace);
|
|
451
|
+
let supersedes = validateRelationTarget(entry.workspace, entry.supersedes ?? null, "supersedes", all);
|
|
452
|
+
const contradicts = validateRelationTarget(entry.workspace, entry.contradicts ?? null, "contradicts", all);
|
|
453
|
+
const dup = all.find((m) => hashText(m.text) === hashText(text) && m.kind === (entry.kind ?? "fact"));
|
|
371
454
|
if (dup !== void 0) {
|
|
372
455
|
// Repeat of the exact same failure is itself a signal: refresh recency
|
|
373
456
|
// AND climb the hit counter (errors only) so the [ENGRAM] index can
|
|
@@ -396,6 +479,13 @@ export async function openEngramDomain(facility) {
|
|
|
396
479
|
if (allMemories(entry.workspace).length >= config.maxMemoriesPerWorkspace) {
|
|
397
480
|
throw new EngramError("ENGRAM_CAP_EXCEEDED", `workspace already holds ${config.maxMemoriesPerWorkspace} active memories (the cap)`);
|
|
398
481
|
}
|
|
482
|
+
// Auto-supersede (config.autoSupersede, OFF by default): only when the
|
|
483
|
+
// writer did not name a target AND the hired heuristic fires (see
|
|
484
|
+
// autoSupersedeCandidate) — explicit supersedes always wins.
|
|
485
|
+
if (supersedes === null && config?.autoSupersede === true) {
|
|
486
|
+
const auto = autoSupersedeCandidate(all, entry, text);
|
|
487
|
+
if (auto !== null) supersedes = auto.id;
|
|
488
|
+
}
|
|
399
489
|
const expiresAt =
|
|
400
490
|
entry.expiresAt !== void 0 && entry.expiresAt !== null
|
|
401
491
|
? entry.expiresAt
|
|
@@ -418,6 +508,10 @@ export async function openEngramDomain(facility) {
|
|
|
418
508
|
filePath: entry.filePath ?? null,
|
|
419
509
|
signal: typeof entry.signal === "number" ? Math.max(0, Math.min(1, entry.signal)) : 0.5,
|
|
420
510
|
status: "active",
|
|
511
|
+
supersedes,
|
|
512
|
+
contradicts,
|
|
513
|
+
source: entry.source ?? null,
|
|
514
|
+
conditions: entry.conditions ?? null,
|
|
421
515
|
expiresAt,
|
|
422
516
|
};
|
|
423
517
|
await memories.put(stored.id, stored);
|
|
@@ -444,12 +538,81 @@ export async function openEngramDomain(facility) {
|
|
|
444
538
|
const active = allMemories(workspace).filter(notExpiredNow);
|
|
445
539
|
const bounded = Math.max(1, Math.min(50, Math.trunc(limit) || 20));
|
|
446
540
|
if (query !== void 0 && String(query).trim().length > 0) {
|
|
541
|
+
const { supersededBy } = memoryRelationMaps(active);
|
|
447
542
|
const ranked = bm25Rank(active, String(query).trim(), bounded);
|
|
448
|
-
|
|
449
|
-
|
|
543
|
+
// Superseded = stale truth: an active memory marked itself as replacing
|
|
544
|
+
// that row. Keep it re-fetchable, but never let it win the ranking —
|
|
545
|
+
// demote it to the tail so the current statement surfaces first.
|
|
546
|
+
const main = ranked.filter((m) => !supersededBy.has(m.id));
|
|
547
|
+
const stale = ranked.filter((m) => supersededBy.has(m.id));
|
|
548
|
+
const merged = main.concat(stale).slice(0, bounded);
|
|
549
|
+
if (maxPerSession === void 0 || maxPerSession === null) return merged;
|
|
550
|
+
return dedupeBySession(merged, maxPerSession); // 候选① per-session cap
|
|
450
551
|
}
|
|
451
552
|
return active.sort(byRecency).slice(0, bounded);
|
|
452
553
|
},
|
|
554
|
+
/**
|
|
555
|
+
* Optional cross-workspace recall (issue #5, recallScope: "global").
|
|
556
|
+
* Ranks across EVERY workspace's active pool with the same deterministic
|
|
557
|
+
* BM25 scorer, then applies a small origin penalty so an out-of-workspace
|
|
558
|
+
* hit needs meaningfully stronger lexical relevance to displace an
|
|
559
|
+
* in-workspace one. Results are annotated with `_origin` (the workspace key)
|
|
560
|
+
* so the tools layer can surface a [W:<workspace>] marker — the model then
|
|
561
|
+
* knows the context a cross-workspace entry came from. Default strictly
|
|
562
|
+
* preserves per-workspace isolation; this is only invoked when the caller
|
|
563
|
+
* opts in (config recallScope="global" or an explicit scope="global" call).
|
|
564
|
+
*/
|
|
565
|
+
recallGlobal(query, limit = 20, { maxPerSession, excludeWorkspace } = {}) {
|
|
566
|
+
const bounded = Math.max(1, Math.min(50, Math.trunc(limit) || 20));
|
|
567
|
+
let pool = [...memories.entries()].map(([, m]) => m).filter((m) => m.status === "active").filter(notExpiredNow);
|
|
568
|
+
if (excludeWorkspace !== void 0 && excludeWorkspace !== null) {
|
|
569
|
+
pool = pool.filter((m) => m.workspace !== excludeWorkspace);
|
|
570
|
+
}
|
|
571
|
+
if (pool.length === 0) return [];
|
|
572
|
+
// Tag every row with its origin workspace for the display layer.
|
|
573
|
+
const tagged = pool.map((m) => ({ ...m, _origin: m.workspace }));
|
|
574
|
+
const trimmed = String(query ?? "").trim();
|
|
575
|
+
const half = Math.max(1, bounded * 3);
|
|
576
|
+
let ranked;
|
|
577
|
+
if (trimmed.length === 0) {
|
|
578
|
+
ranked = tagged.sort(byRecency).slice(0, bounded);
|
|
579
|
+
} else {
|
|
580
|
+
ranked = bm25Rank(tagged, trimmed, half);
|
|
581
|
+
}
|
|
582
|
+
// Origin penalty: an out-of-workspace hit is discounted by 20% (a
|
|
583
|
+
// same-workspace entry with comparable relevance wins ties). bm25Rank
|
|
584
|
+
// drops its internal score, but the returned order is monotonic in it;
|
|
585
|
+
// we only re-stabilize near-equal neighbours so cross-workspace hits
|
|
586
|
+
// never displace an equally-relevant in-workspace one.
|
|
587
|
+
if (excludeWorkspace !== void 0 && excludeWorkspace !== null && ranked.length > 1) {
|
|
588
|
+
ranked = ranked.slice().sort((a, b) => {
|
|
589
|
+
const crossA = (a._origin ?? a.workspace) !== excludeWorkspace;
|
|
590
|
+
const crossB = (b._origin ?? b.workspace) !== excludeWorkspace;
|
|
591
|
+
if (crossA === crossB) return 0;
|
|
592
|
+
return crossA ? 1 : -1; // same-workspace first within ties
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
const out = [];
|
|
596
|
+
const seen = new Set();
|
|
597
|
+
for (const m of ranked) {
|
|
598
|
+
if (seen.has(m.id)) continue;
|
|
599
|
+
seen.add(m.id);
|
|
600
|
+
out.push(m);
|
|
601
|
+
if (out.length >= bounded) break;
|
|
602
|
+
}
|
|
603
|
+
if (maxPerSession !== void 0 && maxPerSession !== null) return dedupeBySession(out, maxPerSession);
|
|
604
|
+
return out;
|
|
605
|
+
},
|
|
606
|
+
/**
|
|
607
|
+
* Memory-to-memory relation lookup maps for a workspace (deterministic):
|
|
608
|
+
* supersededBy: id -> { newer } (a row marked m.supersedes === id)
|
|
609
|
+
* contradictedBy: id -> [ rows ] (rows marked m.contradicts === id)
|
|
610
|
+
* Built purely from the active pool; used by recall rendering and the
|
|
611
|
+
* [ENGRAM] index to surface "stale truth" honestly without deleting.
|
|
612
|
+
*/
|
|
613
|
+
memoryRelations(workspace) {
|
|
614
|
+
return memoryRelationMaps(allMemories(workspace).filter(notExpiredNow));
|
|
615
|
+
},
|
|
453
616
|
timeline(workspace, entity, limit = 30) {
|
|
454
617
|
const active = allMemories(workspace).filter(notExpiredNow);
|
|
455
618
|
const bounded = Math.max(1, Math.min(100, Math.trunc(limit) || 30));
|
|
@@ -468,6 +631,19 @@ export async function openEngramDomain(facility) {
|
|
|
468
631
|
await memories.put(id, { ...m, status: "archived", updatedAt: now() });
|
|
469
632
|
return true;
|
|
470
633
|
},
|
|
634
|
+
/**
|
|
635
|
+
* Append a tag to an ACTIVE memory (idempotent; does NOT touch updatedAt,
|
|
636
|
+
* so a resolution annotation never re-ranks an error into recall).
|
|
637
|
+
* Used by the failure→fix closure to mark resolved errors. Zero LLM.
|
|
638
|
+
*/
|
|
639
|
+
async tagMemory(workspace, id, tag) {
|
|
640
|
+
const m = memories.get(id);
|
|
641
|
+
if (m === void 0 || m.workspace !== workspace || m.status !== "active") return false;
|
|
642
|
+
if (typeof tag !== "string" || tag.length === 0) return true;
|
|
643
|
+
const tags = (m.tags ?? []).includes(tag) ? m.tags : [...(m.tags ?? []), tag];
|
|
644
|
+
await memories.put(id, { ...m, tags });
|
|
645
|
+
return true;
|
|
646
|
+
},
|
|
471
647
|
/** Lazy TTL sweep (write path only; async). Returns archived count. */
|
|
472
648
|
async expireSweep(workspace) {
|
|
473
649
|
let n = 0;
|
|
@@ -853,6 +1029,235 @@ export async function openEngramDomain(facility) {
|
|
|
853
1029
|
await entities.delete(id);
|
|
854
1030
|
return true;
|
|
855
1031
|
},
|
|
1032
|
+
// ── export / import (data contract, format "dsh-engram/export" v1) ─
|
|
1033
|
+
/**
|
|
1034
|
+
* Dump every row of the four memory tables — including archived memories /
|
|
1035
|
+
* tasks — so a backup or migration never loses provenance. `workspace`
|
|
1036
|
+
* restricts to one workspace; falsy dumps every workspace. The dump is the
|
|
1037
|
+
* plain contract a consumer (or a future dsh release) can round-trip back
|
|
1038
|
+
* through `importAll` — it carries `meta.format` + `meta.version` so
|
|
1039
|
+
* forwards-incompatible formats can be rejected by the importer.
|
|
1040
|
+
*/
|
|
1041
|
+
exportAll(workspace) {
|
|
1042
|
+
const pick = (table) => [...table.entries()].map(([, v]) => v).filter((v) => v.workspace === workspace);
|
|
1043
|
+
const all = workspace
|
|
1044
|
+
? { memories: pick(memories), tasks: pick(tasks), links: pick(links), entities: pick(entities) }
|
|
1045
|
+
: {
|
|
1046
|
+
memories: [...memories.entries()].map(([, v]) => v),
|
|
1047
|
+
tasks: [...tasks.entries()].map(([, v]) => v),
|
|
1048
|
+
links: [...links.entries()].map(([, v]) => v),
|
|
1049
|
+
entities: [...entities.entries()].map(([, v]) => v),
|
|
1050
|
+
};
|
|
1051
|
+
return {
|
|
1052
|
+
meta: { format: "dsh-engram/export", version: 1, exportedAt: now(), workspace: workspace ?? null },
|
|
1053
|
+
...all,
|
|
1054
|
+
};
|
|
1055
|
+
},
|
|
1056
|
+
/**
|
|
1057
|
+
* Restore a dump produced by `exportAll` (same format/version), honoring
|
|
1058
|
+
* the same write-path invariants as `storeMemory`: memory text is run
|
|
1059
|
+
* through the deterministic secret redactor before it can land on disk.
|
|
1060
|
+
*
|
|
1061
|
+
* Modes:
|
|
1062
|
+
* - `merge` (default) — idempotent: writes only rows whose id is absent,
|
|
1063
|
+
* kept per-row (each row keeps its own `workspace`).
|
|
1064
|
+
* - `replace` — wipes the given `workspace` from all four tables first,
|
|
1065
|
+
* then writes only rows belonging to it (a full restore; the caller
|
|
1066
|
+
* owns the destructive step).
|
|
1067
|
+
*
|
|
1068
|
+
* Bad rows never abort the whole import: structural/validation failures
|
|
1069
|
+
* (missing id/workspace, empty text, text over the char cap, id already
|
|
1070
|
+
* present in merge, workspace memory cap in merge) are skipped and listed
|
|
1071
|
+
* in the report. `dryRun: true` computes the identical plan without
|
|
1072
|
+
* writing a single row.
|
|
1073
|
+
*
|
|
1074
|
+
* @param payload - the dump (`{ meta?, memories?, tasks?, links?, entities? }`).
|
|
1075
|
+
* @param opts - `{ mode, dryRun, workspace, config }` (workspace mandatory for replace).
|
|
1076
|
+
* @returns { mode, dryRun, written: {table: n}, skipped: [{table,id,reason}] }
|
|
1077
|
+
*/
|
|
1078
|
+
async importAll(payload, { mode = "merge", dryRun = false, workspace, config } = {}) {
|
|
1079
|
+
if (payload === null || typeof payload !== "object" || Array.isArray(payload)) {
|
|
1080
|
+
throw new EngramError("ENGRAM_INVALID_ARGS", "import payload must be an object (a dsh-engram/export dump)");
|
|
1081
|
+
}
|
|
1082
|
+
if (mode !== "merge" && mode !== "replace") {
|
|
1083
|
+
throw new EngramError("ENGRAM_INVALID_ARGS", "import mode must be 'merge' or 'replace'");
|
|
1084
|
+
}
|
|
1085
|
+
if (mode === "replace" && (typeof workspace !== "string" || workspace.length === 0)) {
|
|
1086
|
+
throw new EngramError("ENGRAM_INVALID_ARGS", "replace mode requires the workspace to restore into");
|
|
1087
|
+
}
|
|
1088
|
+
const maxMemoryChars = config?.maxMemoryChars ?? 1600;
|
|
1089
|
+
const maxPerWs = config?.maxMemoriesPerWorkspace ?? 2000;
|
|
1090
|
+
const tables = {
|
|
1091
|
+
memories: Array.isArray(payload.memories) ? payload.memories : [],
|
|
1092
|
+
tasks: Array.isArray(payload.tasks) ? payload.tasks : [],
|
|
1093
|
+
links: Array.isArray(payload.links) ? payload.links : [],
|
|
1094
|
+
entities: Array.isArray(payload.entities) ? payload.entities : [],
|
|
1095
|
+
};
|
|
1096
|
+
const written = { memories: 0, tasks: 0, links: 0, entities: 0 };
|
|
1097
|
+
const skipped = [];
|
|
1098
|
+
const rowTable = (name) => ({ memories, tasks, links, entities })[name];
|
|
1099
|
+
const rowsOf = (table, ws) => [...table.entries()].map(([, v]) => v).filter((v) => v.workspace === ws);
|
|
1100
|
+
|
|
1101
|
+
// Destructive mode: drop every row of the target workspace up front
|
|
1102
|
+
// (planned under dryRun, executed otherwise).
|
|
1103
|
+
if (mode === "replace") {
|
|
1104
|
+
for (const [name, table] of Object.entries({ memories, tasks, links, entities })) {
|
|
1105
|
+
for (const row of rowsOf(table, workspace)) {
|
|
1106
|
+
if (!dryRun) await table.delete(row.id);
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
// Per-workspace running count of ACTIVE memories — the invariant the
|
|
1112
|
+
// workspace cap protects (index-block size); archived rows never count,
|
|
1113
|
+
// and the cap is per workspace, not global.
|
|
1114
|
+
const activeByWs = new Map();
|
|
1115
|
+
for (const [, m] of memories.entries()) {
|
|
1116
|
+
if (m.status === "active") activeByWs.set(m.workspace, (activeByWs.get(m.workspace) ?? 0) + 1);
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
const putRow = async (name, key, row) => {
|
|
1120
|
+
if (!dryRun) await rowTable(name).put(key, row);
|
|
1121
|
+
};
|
|
1122
|
+
|
|
1123
|
+
// Schema defaults, mirroring storeMemory / the zod schemas: an imported
|
|
1124
|
+
// row must behave exactly like one that went through the live write path
|
|
1125
|
+
// (listMemories/recall only see status==="active", tasks need a state,
|
|
1126
|
+
// etc.) — so absent optional fields get their canonical default, never
|
|
1127
|
+
// `undefined`.
|
|
1128
|
+
const normalizeMemory = (row, key, ws, text) => ({
|
|
1129
|
+
kind: row.kind ?? "fact",
|
|
1130
|
+
tags: Array.isArray(row.tags) ? row.tags.slice() : [],
|
|
1131
|
+
entity: row.entity ?? null,
|
|
1132
|
+
seq: Number.isFinite(row.seq) ? row.seq : 0,
|
|
1133
|
+
...row,
|
|
1134
|
+
id: key,
|
|
1135
|
+
workspace: ws,
|
|
1136
|
+
text,
|
|
1137
|
+
status: row.status ?? "active",
|
|
1138
|
+
hits: Number.isFinite(row.hits) ? row.hits : 0,
|
|
1139
|
+
lastAccessAt: row.lastAccessAt ?? null,
|
|
1140
|
+
filePath: row.filePath ?? null,
|
|
1141
|
+
signal: typeof row.signal === "number" ? row.signal : 0.5,
|
|
1142
|
+
supersedes: row.supersedes ?? null,
|
|
1143
|
+
contradicts: row.contradicts ?? null,
|
|
1144
|
+
source: row.source ?? null,
|
|
1145
|
+
conditions: row.conditions ?? null,
|
|
1146
|
+
expiresAt: row.expiresAt ?? null,
|
|
1147
|
+
sessionId: row.sessionId ?? "import",
|
|
1148
|
+
createdAt: Number.isFinite(row.createdAt) ? row.createdAt : now(),
|
|
1149
|
+
updatedAt: Number.isFinite(row.updatedAt) ? row.updatedAt : now(),
|
|
1150
|
+
});
|
|
1151
|
+
const normalizeTask = (row, key, ws) => ({
|
|
1152
|
+
...row,
|
|
1153
|
+
id: key,
|
|
1154
|
+
workspace: ws,
|
|
1155
|
+
state: row.state ?? "draft",
|
|
1156
|
+
artifact: row.artifact ?? null,
|
|
1157
|
+
evaluation: row.evaluation ?? null,
|
|
1158
|
+
memoryRefs: Array.isArray(row.memoryRefs) ? row.memoryRefs.slice() : [],
|
|
1159
|
+
deps: Array.isArray(row.deps) ? row.deps.slice() : [],
|
|
1160
|
+
assignee: row.assignee ?? null,
|
|
1161
|
+
claimedAt: row.claimedAt ?? null,
|
|
1162
|
+
stateChangedAt: Number.isFinite(row.stateChangedAt) ? row.stateChangedAt : 0,
|
|
1163
|
+
archivedAt: row.archivedAt ?? null,
|
|
1164
|
+
sessionId: row.sessionId ?? "import",
|
|
1165
|
+
createdAt: Number.isFinite(row.createdAt) ? row.createdAt : now(),
|
|
1166
|
+
updatedAt: Number.isFinite(row.updatedAt) ? row.updatedAt : now(),
|
|
1167
|
+
});
|
|
1168
|
+
const normalizeLink = (row, key, ws) => ({
|
|
1169
|
+
...row,
|
|
1170
|
+
id: key,
|
|
1171
|
+
workspace: ws,
|
|
1172
|
+
relation: row.relation ?? "relates-to",
|
|
1173
|
+
confidence: typeof row.confidence === "number" ? row.confidence : 1,
|
|
1174
|
+
sessionId: row.sessionId ?? "import",
|
|
1175
|
+
createdAt: Number.isFinite(row.createdAt) ? row.createdAt : now(),
|
|
1176
|
+
});
|
|
1177
|
+
const normalizeEntity = (row, key, ws) => ({
|
|
1178
|
+
...row,
|
|
1179
|
+
id: key,
|
|
1180
|
+
workspace: ws,
|
|
1181
|
+
description: row.description ?? "",
|
|
1182
|
+
kind: row.kind ?? "",
|
|
1183
|
+
sessionId: row.sessionId ?? "import",
|
|
1184
|
+
createdAt: Number.isFinite(row.createdAt) ? row.createdAt : now(),
|
|
1185
|
+
updatedAt: Number.isFinite(row.updatedAt) ? row.updatedAt : now(),
|
|
1186
|
+
});
|
|
1187
|
+
|
|
1188
|
+
for (const row of tables.memories) {
|
|
1189
|
+
const key = typeof row?.id === "string" && row.id.length > 0 ? row.id : "";
|
|
1190
|
+
const ws = typeof row?.workspace === "string" && row.workspace.length > 0 ? row.workspace : workspace;
|
|
1191
|
+
if (key === "") {
|
|
1192
|
+
skipped.push({ table: "memories", id: null, reason: "missing id" });
|
|
1193
|
+
continue;
|
|
1194
|
+
}
|
|
1195
|
+
if (mode === "merge" && memories.get(key) !== void 0) {
|
|
1196
|
+
skipped.push({ table: "memories", id: key, reason: "exists" });
|
|
1197
|
+
continue;
|
|
1198
|
+
}
|
|
1199
|
+
if (typeof ws !== "string" || ws.length === 0) {
|
|
1200
|
+
skipped.push({ table: "memories", id: key, reason: "missing workspace" });
|
|
1201
|
+
continue;
|
|
1202
|
+
}
|
|
1203
|
+
if (mode === "replace" && ws !== workspace) {
|
|
1204
|
+
skipped.push({ table: "memories", id: key, reason: "outside restored workspace" });
|
|
1205
|
+
continue;
|
|
1206
|
+
}
|
|
1207
|
+
const text = redactText(String(row.text ?? "")).trim();
|
|
1208
|
+
if (text.length === 0) {
|
|
1209
|
+
skipped.push({ table: "memories", id: key, reason: "empty text" });
|
|
1210
|
+
continue;
|
|
1211
|
+
}
|
|
1212
|
+
if ([...text].length > maxMemoryChars) {
|
|
1213
|
+
skipped.push({ table: "memories", id: key, reason: "over maxMemoryChars" });
|
|
1214
|
+
continue;
|
|
1215
|
+
}
|
|
1216
|
+
if (mode === "merge" && (row.status ?? "active") === "active" && (activeByWs.get(ws) ?? 0) >= maxPerWs) {
|
|
1217
|
+
skipped.push({ table: "memories", id: key, reason: "workspace memory cap" });
|
|
1218
|
+
continue;
|
|
1219
|
+
}
|
|
1220
|
+
if ((row.status ?? "active") === "active") activeByWs.set(ws, (activeByWs.get(ws) ?? 0) + 1);
|
|
1221
|
+
try {
|
|
1222
|
+
await putRow("memories", key, normalizeMemory(row, key, ws, text));
|
|
1223
|
+
written.memories += 1;
|
|
1224
|
+
} catch (error) {
|
|
1225
|
+
skipped.push({ table: "memories", id: key, reason: `invalid row: ${String(error?.message ?? error)}` });
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
for (const [name, rows] of Object.entries({ tasks: tables.tasks, links: tables.links, entities: tables.entities })) {
|
|
1230
|
+
for (const row of rows) {
|
|
1231
|
+
const key = typeof row?.id === "string" && row.id.length > 0 ? row.id : "";
|
|
1232
|
+
if (key === "") {
|
|
1233
|
+
skipped.push({ table: name, id: null, reason: "missing id" });
|
|
1234
|
+
continue;
|
|
1235
|
+
}
|
|
1236
|
+
if (mode === "merge" && rowTable(name).get(key) !== void 0) {
|
|
1237
|
+
skipped.push({ table: name, id: key, reason: "exists" });
|
|
1238
|
+
continue;
|
|
1239
|
+
}
|
|
1240
|
+
const ws = typeof row?.workspace === "string" && row.workspace.length > 0 ? row.workspace : workspace;
|
|
1241
|
+
if (typeof ws !== "string" || ws.length === 0) {
|
|
1242
|
+
skipped.push({ table: name, id: key, reason: "missing workspace" });
|
|
1243
|
+
continue;
|
|
1244
|
+
}
|
|
1245
|
+
if (mode === "replace" && ws !== workspace) {
|
|
1246
|
+
skipped.push({ table: name, id: key, reason: "outside restored workspace" });
|
|
1247
|
+
continue;
|
|
1248
|
+
}
|
|
1249
|
+
const normalize = { tasks: normalizeTask, links: normalizeLink, entities: normalizeEntity }[name];
|
|
1250
|
+
try {
|
|
1251
|
+
await putRow(name, key, normalize(row, key, ws));
|
|
1252
|
+
written[name] += 1;
|
|
1253
|
+
} catch (error) {
|
|
1254
|
+
skipped.push({ table: name, id: key, reason: `invalid row: ${String(error?.message ?? error)}` });
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
return { mode, dryRun, written, skipped };
|
|
1260
|
+
},
|
|
856
1261
|
// ── usage / observability ────────────────────────────────────────
|
|
857
1262
|
/** Merge one tool-call event into the (workspace, day) rollup row. */
|
|
858
1263
|
async bumpUsage(workspace, day, { counts = {}, failures = 0, recall = {} }) {
|
package/lib/todo-sink.js
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-engram: session-end todo auto-sink (audit follow-up).
|
|
3
|
+
*
|
|
4
|
+
* The todo plane (`todo_write`) is per-session working memory: last-write-wins,
|
|
5
|
+
* dies with the session. The promote hint (P0-A) and the GUI 「沉淀到 ESR」 button
|
|
6
|
+
* cover the INTENTIONAL path, but both only fire/run while the session is alive
|
|
7
|
+
* and the funnel is narrow — a session that ends with pending todos silently
|
|
8
|
+
* evaporates its plan (the observed "todo 一直建、esr 长期空" leak).
|
|
9
|
+
*
|
|
10
|
+
* This module closes that gap at the ONE safe boundary: `session/disposed`
|
|
11
|
+
* (the session's teardown edge). The trade-off from the design audit is kept
|
|
12
|
+
* intact:
|
|
13
|
+
* - nothing is auto-written DURING the session — todo stays cheap/ephemeral
|
|
14
|
+
* and the [ESR] prefix stays stable;
|
|
15
|
+
* - the sink never pollutes the active list — landed tasks are `draft`
|
|
16
|
+
* (excluded from [ESR] active rows by buildEsrSnapshot), deduped against
|
|
17
|
+
* existing non-stable tasks by name, and capped by `maxTasksPerWorkspace`;
|
|
18
|
+
* - a draft still needs the deliberate `esr_claim` / `esr_task` step to
|
|
19
|
+
* become active, so the "committed work" obligation is never created
|
|
20
|
+
* implicitly — the session plan becomes lossless, not obligated.
|
|
21
|
+
*
|
|
22
|
+
* All functions are pure / contained and unit-testable without the harness.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { pendingTodos } from "./trigger.js";
|
|
26
|
+
import { shortId, uuid } from "./util.js";
|
|
27
|
+
|
|
28
|
+
/** Description stamped on every auto-sunk draft (mirrors the GUI button's note). */
|
|
29
|
+
export const SINK_DESCRIPTION = "源自会话计划(会话结束自动沉淀)";
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Extract the FINAL pending todo list from a session's event log: the last
|
|
33
|
+
* `todo/write` event (whole-list snapshot) filtered to unfinished items, in
|
|
34
|
+
* order. Returns [] when the session never wrote todos or everything is done.
|
|
35
|
+
* Deterministic — this is a pure function of the session log, so it survives
|
|
36
|
+
* process restarts (no reliance on the process-local recorder snapshot).
|
|
37
|
+
*/
|
|
38
|
+
export function pendingTodosFromSession(session) {
|
|
39
|
+
if (session === null || typeof session !== "object") return [];
|
|
40
|
+
const events = Array.isArray(session.events) ? session.events : [];
|
|
41
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
42
|
+
const event = events[i];
|
|
43
|
+
if (event === null || typeof event !== "object" || event.type !== "todo/write") continue;
|
|
44
|
+
// Event data is the same { todos: TodoItem[] } shape todo_write args wear.
|
|
45
|
+
return pendingTodos(event.data);
|
|
46
|
+
}
|
|
47
|
+
return [];
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Decide which pending todos should sink into ESR, and in what order. Pure:
|
|
52
|
+
* - names already held by a non-stable task are skipped (dedupe — a task the
|
|
53
|
+
* promote hint / GUI button already created is not re-created);
|
|
54
|
+
* - within-list duplicates sink only once;
|
|
55
|
+
* - the workspace cap (`maxTasksPerWorkspace`, counting drafts) is honoured:
|
|
56
|
+
* once `activeCount + created` reaches `maxTasks`, the rest are reported as
|
|
57
|
+
* `atCap` instead of created.
|
|
58
|
+
*
|
|
59
|
+
* @param {string[]} todos pending todo names, in plan order
|
|
60
|
+
* @param {Array<{name?: string}>} existingTasks non-stable tasks of the workspace
|
|
61
|
+
* @param {{ maxTasks?: number, activeCount?: number }} opts
|
|
62
|
+
* @returns {{ toCreate: string[], existed: string[], atCap: string[] }}
|
|
63
|
+
*/
|
|
64
|
+
export function planTodoSink(todos, existingTasks, opts = {}) {
|
|
65
|
+
const maxTasks = Number.isFinite(opts.maxTasks) && opts.maxTasks > 0 ? opts.maxTasks : 10;
|
|
66
|
+
const activeCount = opts.activeCount !== void 0
|
|
67
|
+
? opts.activeCount
|
|
68
|
+
: (Array.isArray(existingTasks) ? existingTasks.length : 0);
|
|
69
|
+
const existing = new Set(
|
|
70
|
+
(Array.isArray(existingTasks) ? existingTasks : [])
|
|
71
|
+
.map((t) => String(t?.name ?? "").trim().toLowerCase())
|
|
72
|
+
.filter(Boolean),
|
|
73
|
+
);
|
|
74
|
+
const toCreate = [];
|
|
75
|
+
const existed = [];
|
|
76
|
+
const atCap = [];
|
|
77
|
+
const seen = new Set();
|
|
78
|
+
for (const todo of todos ?? []) {
|
|
79
|
+
const name = String(todo ?? "").trim().slice(0, 200);
|
|
80
|
+
if (name.length === 0) continue;
|
|
81
|
+
const key = name.toLowerCase();
|
|
82
|
+
if (seen.has(key)) continue; // within-list duplicate → single sink
|
|
83
|
+
seen.add(key);
|
|
84
|
+
if (existing.has(key)) {
|
|
85
|
+
existed.push(name);
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
if (activeCount + toCreate.length >= maxTasks) {
|
|
89
|
+
atCap.push(name);
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
toCreate.push(name);
|
|
93
|
+
}
|
|
94
|
+
return { toCreate, existed, atCap };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Sink a finished session's pending todos into ESR as DRAFT tasks. Best-effort,
|
|
99
|
+
* never throws into the caller — failures surface via the returned counts plus
|
|
100
|
+
* whatever the caller logs. Returns { created, existed, atCap } name lists.
|
|
101
|
+
*
|
|
102
|
+
* @param {object} domain the opened engram store domain (putTask/listTasks/activeTaskCount)
|
|
103
|
+
* @param {string} workspace normalized workspace key
|
|
104
|
+
* @param {string[]} todos pending todo names
|
|
105
|
+
* @param {{ sessionId?: string, maxTasks?: number, description?: string, now?: number }} opts
|
|
106
|
+
*/
|
|
107
|
+
export async function sinkPendingTodos(domain, workspace, todos, opts = {}) {
|
|
108
|
+
const record = { created: [], existed: [], atCap: [] };
|
|
109
|
+
if (!Array.isArray(todos) || todos.length === 0) return record;
|
|
110
|
+
if (domain === null || typeof domain !== "object" || typeof domain.putTask !== "function") return record;
|
|
111
|
+
const sessionId = opts.sessionId ?? "auto-sink";
|
|
112
|
+
const nowMs = typeof opts.now === "function" ? opts.now() : Number.isFinite(opts.now) ? opts.now : Date.now();
|
|
113
|
+
const existingTasks = typeof domain.listTasks === "function"
|
|
114
|
+
? domain.listTasks(workspace, { includeStable: false })
|
|
115
|
+
: [];
|
|
116
|
+
const activeCount = typeof domain.activeTaskCount === "function"
|
|
117
|
+
? domain.activeTaskCount(workspace)
|
|
118
|
+
: existingTasks.length;
|
|
119
|
+
const { toCreate, existed, atCap } = planTodoSink(todos, existingTasks, {
|
|
120
|
+
maxTasks: opts.maxTasks,
|
|
121
|
+
activeCount,
|
|
122
|
+
});
|
|
123
|
+
record.existed = existed;
|
|
124
|
+
record.atCap = atCap;
|
|
125
|
+
for (const name of toCreate) {
|
|
126
|
+
await domain.putTask({
|
|
127
|
+
id: `tsk_${shortId(uuid())}`,
|
|
128
|
+
workspace,
|
|
129
|
+
name,
|
|
130
|
+
description: opts.description ?? SINK_DESCRIPTION,
|
|
131
|
+
state: "draft",
|
|
132
|
+
artifact: null,
|
|
133
|
+
evaluation: null,
|
|
134
|
+
memoryRefs: [],
|
|
135
|
+
sessionId,
|
|
136
|
+
createdAt: nowMs,
|
|
137
|
+
updatedAt: nowMs,
|
|
138
|
+
stateChangedAt: 0,
|
|
139
|
+
deps: [],
|
|
140
|
+
assignee: null,
|
|
141
|
+
claimedAt: null,
|
|
142
|
+
});
|
|
143
|
+
record.created.push(name);
|
|
144
|
+
}
|
|
145
|
+
return record;
|
|
146
|
+
}
|