dsh-engram 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +283 -27
- package/README.zh.md +184 -16
- package/cordis.patch.yml +6 -0
- package/docs/CONTEXT-GC-GUIDE.zh.md +192 -0
- package/docs/PROPOSAL-agentmemory.zh.md +108 -0
- package/docs/PROPOSAL-beads.zh.md +81 -0
- package/docs/PROPOSAL-contextgc.zh.md +188 -0
- package/docs/PROPOSAL-esrtrigger.zh.md +199 -0
- package/docs/PROPOSAL-hindsight.zh.md +156 -0
- package/lib/api.js +76 -2
- package/lib/capture.js +132 -4
- package/lib/client.js +45 -2
- package/lib/compaction.js +2 -18
- package/lib/context-gc.js +234 -2
- package/lib/goal-capture.js +89 -0
- package/lib/index-block.js +96 -1
- package/lib/index.js +233 -5
- package/lib/redact.js +48 -0
- package/lib/session-cjk.js +159 -0
- package/lib/settings.js +13 -0
- package/lib/status.js +57 -0
- package/lib/store.js +411 -6
- package/lib/todo-sink.js +146 -0
- package/lib/tools.js +98 -40
- package/lib/trigger.js +10 -8
- package/lib/usage.js +147 -5
- package/lib/web-provision.js +430 -0
- package/package.json +13 -2
- package/scripts/dsh-engram.mjs +268 -0
- package/scripts/engram-web-compaction.mjs +7 -0
- package/scripts/prepublish-check.mjs +106 -0
- package/scripts/setup-links.mjs +241 -0
package/lib/index.js
CHANGED
|
@@ -6,8 +6,10 @@
|
|
|
6
6
|
* - a compact, per-session-frozen [ENGRAM] symbolic index injected through a
|
|
7
7
|
* systemPrompt section (order 40) — prefix-stable for KV cache reuse
|
|
8
8
|
* - an [ESR] task/closure block (order 41) listing evidence gaps
|
|
9
|
-
* -
|
|
10
|
-
* esr_close / esr_link
|
|
9
|
+
* - fourteen tight tools: memory plane (engram_store / engram_recall /
|
|
10
|
+
* engram_detail) + ESR protocol (esr_task / esr_close / esr_link / esr_dep /
|
|
11
|
+
* esr_claim / esr_unclaim / esr_ready / esr_status / esr_node / esr_gc /
|
|
12
|
+
* esr_model)
|
|
11
13
|
* - storage on ctx.storageDomain (JSON unit dsh_engram), reads synchronous
|
|
12
14
|
*
|
|
13
15
|
* All model-visible writes are immediate (no approval gate — zero friction,
|
|
@@ -20,11 +22,15 @@ import { resolve as resolvePath } from "node:path";
|
|
|
20
22
|
import { openEngramDomain } from "./store.js";
|
|
21
23
|
import { registerTools } from "./tools.js";
|
|
22
24
|
import { makeCaptureHandler } from "./capture.js";
|
|
23
|
-
import {
|
|
25
|
+
import { installGoalCapture } from "./goal-capture.js";
|
|
26
|
+
import { renderIndex, renderStartRecall, firstUserMessageText, renderEsr, esrMethodology, buildEsrSnapshot, buildFrozenEsrBlock } from "./index-block.js";
|
|
24
27
|
import { makeTriggerRecorder } from "./trigger.js";
|
|
28
|
+
import { pendingTodosFromSession, sinkPendingTodos } from "./todo-sink.js";
|
|
25
29
|
import { installEngramSettings } from "./settings.js";
|
|
26
30
|
import { makeEngramRoutes } from "./api.js";
|
|
27
|
-
import {
|
|
31
|
+
import { collectPointerContext, mountCompactionEngine } from "./context-gc.js";
|
|
32
|
+
import { provisionWebCompaction } from "./web-provision.js";
|
|
33
|
+
import { writeContextGcStatus } from "./status.js";
|
|
28
34
|
import { workspaceKey as wk } from "./util.js";
|
|
29
35
|
|
|
30
36
|
/** Stable cordis plugin name. */
|
|
@@ -42,9 +48,25 @@ const DEFAULTS = {
|
|
|
42
48
|
autoCaptureGlobalCap: 500,
|
|
43
49
|
// recall
|
|
44
50
|
sessionSearch: true,
|
|
51
|
+
// opt-in cross-workspace recall ("workspace" = current default, strict
|
|
52
|
+
// per-workspace isolation; "global" = when the local pool has weak/no hits,
|
|
53
|
+
// also rank entries from other workspaces' pools with a recency/origin-aware
|
|
54
|
+
// penalty and surface a [W:<workspace>] origin marker). Token-discipline
|
|
55
|
+
// default is preserved; this is strictly an enable knob.
|
|
56
|
+
recallScope: "workspace",
|
|
45
57
|
// session-diversified recall: max memories per session before cross-session
|
|
46
58
|
// evidence takes over (agentmemory 候选①; "few and precise")
|
|
47
59
|
maxRecallPerSession: 3,
|
|
60
|
+
// Host-side session-start auto-recall (方案 B): at the FIRST assembly of a
|
|
61
|
+
// new session, key a bounded BM25 recall off the session's first user message
|
|
62
|
+
// and inject the top hits as a [RECALL] block — relevant memory reaches the
|
|
63
|
+
// model even when it never calls engram_recall on its own (the observed gap:
|
|
64
|
+
// pull-only recall left everything to model initiative and almost never
|
|
65
|
+
// fired). Pure rule, zero LLM, per-session frozen with the [ENGRAM] block.
|
|
66
|
+
// "few and precise" — agentmemory 候选①'s lesson from the coding leaderboard.
|
|
67
|
+
autoRecallOnStart: true,
|
|
68
|
+
autoRecallLimit: 3,
|
|
69
|
+
autoRecallMaxChars: 700,
|
|
48
70
|
// [ENGRAM] index budget (token discipline)
|
|
49
71
|
indexMaxLines: 12,
|
|
50
72
|
indexMaxChars: 700,
|
|
@@ -88,9 +110,30 @@ const DEFAULTS = {
|
|
|
88
110
|
minErrorHits: 2,
|
|
89
111
|
// P3: an active task with no update for this many days is surfaced as stale.
|
|
90
112
|
staleTaskDays: 14,
|
|
113
|
+
// Session-end todo continuity (audit follow-up): when a session is disposed
|
|
114
|
+
// while its plan still has pending todos, auto-sink them into ESR as DRAFT
|
|
115
|
+
// tasks (deduped by name, capped by maxTasksPerWorkspace). During the session
|
|
116
|
+
// nothing is auto-written — todo stays cheap/ephemeral and the [ESR] prefix
|
|
117
|
+
// stays stable; the sink only lands drafts, which still need the deliberate
|
|
118
|
+
// esr_claim / esr_task step to become active.
|
|
119
|
+
autoSinkTodosOnEnd: true,
|
|
120
|
+
// Web-plane auto-wiring for Context GC: web sessions resolve `compaction`
|
|
121
|
+
// inside the agent preset's own realm, which a profile patch cannot reach,
|
|
122
|
+
// so the plugin edits every stock-layout preset's `compaction` group once —
|
|
123
|
+
// the default preset plus the whole roster (shipped + user roots) — swapping
|
|
124
|
+
// the `compaction-basic` row for `dsh-engram/compaction` (backup + strict
|
|
125
|
+
// validation + idempotent; a preset with a custom compaction layout — or
|
|
126
|
+
// none, like `minimal` — is never touched). Off = every preset keeps DSH's
|
|
127
|
+
// default LLM summarizer; the host plane (headless/TUI/base) is unaffected.
|
|
128
|
+
autoWebCompaction: true,
|
|
129
|
+
// Memory-to-memory semantics (zero LLM): when ON, an entity-anchored write
|
|
130
|
+
// that carries a replacement/negation cue ("改用 / no longer / switched…")
|
|
131
|
+
// auto-marks the matching older same-entity+kind memory as superseded.
|
|
132
|
+
// OFF by default — explicit `supersedes` is always the primary path.
|
|
133
|
+
autoSupersede: false,
|
|
91
134
|
};
|
|
92
135
|
|
|
93
|
-
function resolveConfig(config) {
|
|
136
|
+
export function resolveConfig(config) {
|
|
94
137
|
const src = config && typeof config === "object" ? config : {};
|
|
95
138
|
const out = { ...DEFAULTS };
|
|
96
139
|
for (const key of Object.keys(DEFAULTS)) {
|
|
@@ -167,6 +210,12 @@ export function apply(ctx, config) {
|
|
|
167
210
|
captureStats: { total: 0, git: 0, file: 0, error: 0 },
|
|
168
211
|
/** Last GC sweep totals (GUI overview observability). */
|
|
169
212
|
gcStats: { lastRun: 0, archivedMemories: 0, archivedTasks: 0, removedLinks: 0 },
|
|
213
|
+
/** Live Context GC status (API overview + durable status file). */
|
|
214
|
+
compactionStatus: {
|
|
215
|
+
host: "pending",
|
|
216
|
+
web: { action: "pending", presets: [] },
|
|
217
|
+
config: { autoWebCompaction: resolved.autoWebCompaction, gcReplacesCompaction: resolved.gcReplacesCompaction, gcNarrative: resolved.gcNarrative },
|
|
218
|
+
},
|
|
170
219
|
/** Recompute the exact [ENGRAM] block for a workspace (GUI overview cost). */
|
|
171
220
|
renderIndexBlock(workspace) {
|
|
172
221
|
if (opened === void 0) return "";
|
|
@@ -249,12 +298,32 @@ export function apply(ctx, config) {
|
|
|
249
298
|
return "";
|
|
250
299
|
}
|
|
251
300
|
let block = "";
|
|
301
|
+
let pendingRecall = false;
|
|
252
302
|
try {
|
|
253
303
|
const workspace = resolvePath(cwd);
|
|
254
304
|
block = renderIndex(opened, workspace, cwd, resolved);
|
|
305
|
+
// Host-side session-start auto-recall (方案 B): append the [RECALL]
|
|
306
|
+
// block keyed on the session's FIRST user message. The cache only freezes
|
|
307
|
+
// once there is a query to key on — if the user text isn't in the events
|
|
308
|
+
// yet (exotic pre-message assemblies), return the index alone and let the
|
|
309
|
+
// next assembly retry, so a new conversation never permanently misses its
|
|
310
|
+
// recalled memory.
|
|
311
|
+
if (resolved.autoRecallOnStart !== false) {
|
|
312
|
+
const query = firstUserMessageText(session);
|
|
313
|
+
if (query.length === 0) {
|
|
314
|
+
pendingRecall = true;
|
|
315
|
+
} else {
|
|
316
|
+
const recall = renderStartRecall(opened, workspace, resolved, { query, allowRecencyFallback: false });
|
|
317
|
+
block = recall.length > 0 ? `${block}\n${recall}`.trim() : block;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
255
320
|
} catch (error) {
|
|
256
321
|
log?.warn?.(`engram index render failed: ${String(error)}`);
|
|
257
322
|
}
|
|
323
|
+
// Never cache a render that skipped auto-recall for lack of an opening
|
|
324
|
+
// message — the next assembly will have it and can still materialize the
|
|
325
|
+
// [RECALL] block.
|
|
326
|
+
if (pendingRecall) return block;
|
|
258
327
|
sessionCache.set(session, block);
|
|
259
328
|
return block;
|
|
260
329
|
};
|
|
@@ -313,6 +382,117 @@ export function apply(ctx, config) {
|
|
|
313
382
|
// tools
|
|
314
383
|
disposers.push(registerTools(ctx, service));
|
|
315
384
|
|
|
385
|
+
// Context GC (host plane) — the auto-GC that REPLACES DSH's built-in
|
|
386
|
+
// lossy LLM-summary compaction. Mounts the `compaction` service on THIS
|
|
387
|
+
// fiber (cordis removes it again when the fiber tears down, so reloading /
|
|
388
|
+
// unloading engram rolls back automatically). The profile patch disables
|
|
389
|
+
// the base `compaction-basic` row, making engram the sole provider:
|
|
390
|
+
// `gcReplacesCompaction:true` (default) → ContextGcEngine; `false` → a bare
|
|
391
|
+
// BasicCompactionEngine so the `compaction` service never disappears while
|
|
392
|
+
// dsh-engram is installed. Web profiles mount per-session compaction inside
|
|
393
|
+
// the agent preset's own realm (see README); this host-plane provider
|
|
394
|
+
// covers headless/TUI/base profiles and any host-plane consumer.
|
|
395
|
+
const hostCompactionPromise = mountCompactionEngine(ctx, resolved, {
|
|
396
|
+
readWorkspace: async (workspace, report) => {
|
|
397
|
+
if (!workspace) return null;
|
|
398
|
+
try {
|
|
399
|
+
const domain = await service.getDomain();
|
|
400
|
+
return collectPointerContext(domain, wk(workspace), report);
|
|
401
|
+
} catch {
|
|
402
|
+
return null;
|
|
403
|
+
}
|
|
404
|
+
},
|
|
405
|
+
});
|
|
406
|
+
disposers.push(() => {
|
|
407
|
+
// The engine's service registration is fiber-scoped and auto-removed on
|
|
408
|
+
// teardown; this only settles the async mount and surfaces late errors.
|
|
409
|
+
void hostCompactionPromise.catch(() => {});
|
|
410
|
+
});
|
|
411
|
+
|
|
412
|
+
// Context GC status: record the settled host/web facts and flush a durable
|
|
413
|
+
// snapshot to $DSH_HOME/engram/context-gc.status.json once BOTH planes have
|
|
414
|
+
// settled (the web inject may never fire on a profile without agentPresets,
|
|
415
|
+
// so a bounded timer marks the web plane done for host-only profiles).
|
|
416
|
+
let hostSettled = false;
|
|
417
|
+
let webSettled = resolved.autoWebCompaction === false;
|
|
418
|
+
let statusTimer = void 0;
|
|
419
|
+
const flushStatus = () => {
|
|
420
|
+
if (!hostSettled || !webSettled) return;
|
|
421
|
+
if (statusTimer !== void 0) { clearTimeout(statusTimer); statusTimer = void 0; }
|
|
422
|
+
void writeContextGcStatus({
|
|
423
|
+
planes: resolved.autoWebCompaction === false ? ["host"] : ["host", "web"],
|
|
424
|
+
host: service.compactionStatus.host,
|
|
425
|
+
web: service.compactionStatus.web,
|
|
426
|
+
config: service.compactionStatus.config,
|
|
427
|
+
configSource: {
|
|
428
|
+
autoWebCompaction: resolved.autoWebCompaction,
|
|
429
|
+
gcReplacesCompaction: resolved.gcReplacesCompaction,
|
|
430
|
+
gcNarrative: resolved.gcNarrative,
|
|
431
|
+
},
|
|
432
|
+
}).catch(() => {});
|
|
433
|
+
};
|
|
434
|
+
void hostCompactionPromise
|
|
435
|
+
.then((Engine) => {
|
|
436
|
+
hostSettled = true;
|
|
437
|
+
service.compactionStatus.host = Engine
|
|
438
|
+
? (resolved.gcReplacesCompaction ? "context-gc" : "default")
|
|
439
|
+
: "unavailable";
|
|
440
|
+
flushStatus();
|
|
441
|
+
})
|
|
442
|
+
.catch(() => {
|
|
443
|
+
hostSettled = true;
|
|
444
|
+
service.compactionStatus.host = "unavailable";
|
|
445
|
+
flushStatus();
|
|
446
|
+
});
|
|
447
|
+
|
|
448
|
+
// Web-plane auto-wiring (Context GC): web sessions resolve `compaction`
|
|
449
|
+
// inside the agent preset's own realm, which a profile patch cannot reach.
|
|
450
|
+
// When the `agentPresets` service is present, swap the stock
|
|
451
|
+
// `compaction-basic` row for `dsh-engram/compaction` in every stock-layout
|
|
452
|
+
// preset — the default one plus the whole roster (shipped + user roots) —
|
|
453
|
+
// so a session composed from any preset gets Context GC. Surgical,
|
|
454
|
+
// idempotent, backed up, validated; a preset with a custom compaction
|
|
455
|
+
// layout (or none) is never touched. The row's engine config is derived
|
|
456
|
+
// from the settings knobs (`gcReplacesCompaction` / `gcNarrative`), so
|
|
457
|
+
// changing them re-provisions the web rows on the next boot. Flip
|
|
458
|
+
// `autoWebCompaction:false` to stop auto-wiring on later boots (it does
|
|
459
|
+
// not un-wire already-wired presets — use `npm run web-compaction:revert`
|
|
460
|
+
// for that). Best-effort: a failure only warns.
|
|
461
|
+
if (resolved.autoWebCompaction !== false) {
|
|
462
|
+
ctx.inject(["agentPresets"], (webCtx) => {
|
|
463
|
+
const agentPresets = webCtx.get("agentPresets");
|
|
464
|
+
provisionWebCompaction(agentPresets, {
|
|
465
|
+
rowConfig: {
|
|
466
|
+
gcReplacesCompaction: resolved.gcReplacesCompaction,
|
|
467
|
+
gcNarrative: resolved.gcNarrative,
|
|
468
|
+
},
|
|
469
|
+
log: (line) => webCtx.logger.info(line),
|
|
470
|
+
})
|
|
471
|
+
.then((report) => {
|
|
472
|
+
webSettled = true;
|
|
473
|
+
service.compactionStatus.web = { action: report.action, presets: report.presets };
|
|
474
|
+
flushStatus();
|
|
475
|
+
})
|
|
476
|
+
.catch((error) => {
|
|
477
|
+
webSettled = true;
|
|
478
|
+
service.compactionStatus.web = { action: "failed", presets: [], error: String(error?.message ?? error) };
|
|
479
|
+
webCtx.logger.warn(`engram web-provision failed: ${String(error?.message ?? error)}`);
|
|
480
|
+
flushStatus();
|
|
481
|
+
});
|
|
482
|
+
});
|
|
483
|
+
// Host-only profiles never provide `agentPresets`; settle the web plane
|
|
484
|
+
// after a bounded wait so the status flush still happens there.
|
|
485
|
+
statusTimer = setTimeout(() => {
|
|
486
|
+
if (!webSettled) {
|
|
487
|
+
webSettled = true;
|
|
488
|
+
flushStatus();
|
|
489
|
+
}
|
|
490
|
+
}, 15_000);
|
|
491
|
+
setTimeout(() => { statusTimer = void 0; }, 16_000);
|
|
492
|
+
} else {
|
|
493
|
+
flushStatus();
|
|
494
|
+
}
|
|
495
|
+
|
|
316
496
|
// systemPrompt sections (frozen per session)
|
|
317
497
|
const systemPrompt = ctx.get("systemPrompt");
|
|
318
498
|
if (systemPrompt !== void 0) {
|
|
@@ -359,6 +539,38 @@ export function apply(ctx, config) {
|
|
|
359
539
|
});
|
|
360
540
|
disposers.push(disposeTrigger);
|
|
361
541
|
|
|
542
|
+
// session-end todo auto-sink: a session whose plan still has pending todos
|
|
543
|
+
// lands them as ESR drafts at its teardown edge, so the plan survives the
|
|
544
|
+
// session (flip `autoSinkTodosOnEnd` off to opt out). Live-toggleable — the
|
|
545
|
+
// flag is read per event. Fully contained: a sink failure only warns.
|
|
546
|
+
const disposeSink = ctx.on("session/disposed", (session) => {
|
|
547
|
+
if (resolved.autoSinkTodosOnEnd === false) return;
|
|
548
|
+
const cwd = session?.header?.cwd;
|
|
549
|
+
const sessionId = session?.id;
|
|
550
|
+
if (typeof cwd !== "string" || cwd.length === 0) return;
|
|
551
|
+
if (typeof sessionId !== "string" || sessionId.length === 0) return;
|
|
552
|
+
const todos = pendingTodosFromSession(session);
|
|
553
|
+
if (todos.length === 0) return;
|
|
554
|
+
const workspace = wk(cwd);
|
|
555
|
+
service
|
|
556
|
+
.getDomain()
|
|
557
|
+
.then((domain) =>
|
|
558
|
+
sinkPendingTodos(domain, workspace, todos, {
|
|
559
|
+
sessionId,
|
|
560
|
+
maxTasks: resolved.maxTasksPerWorkspace,
|
|
561
|
+
}),
|
|
562
|
+
)
|
|
563
|
+
.then((report) => {
|
|
564
|
+
if (report.created.length > 0) {
|
|
565
|
+
log?.info?.(`engram todo auto-sink: ${report.created.length} pending todo(s) → ${report.created.length} ESR draft(s) in ${workspace}`);
|
|
566
|
+
} else if (report.existed.length > 0 || report.atCap.length > 0) {
|
|
567
|
+
log?.info?.(`engram todo auto-sink: ${report.existed.length} already in ESR, ${report.atCap.length} skipped (cap) in ${workspace}`);
|
|
568
|
+
}
|
|
569
|
+
})
|
|
570
|
+
.catch((error) => log?.warn?.(`engram todo auto-sink failed: ${String(error)}`));
|
|
571
|
+
});
|
|
572
|
+
disposers.push(disposeSink);
|
|
573
|
+
|
|
362
574
|
// zero-LLM auto-capture
|
|
363
575
|
if (resolved.autoCapture) {
|
|
364
576
|
// The capture handler needs a storeMemory face, but the domain opens
|
|
@@ -374,12 +586,28 @@ export function apply(ctx, config) {
|
|
|
374
586
|
}
|
|
375
587
|
return domain.storeMemory(...args);
|
|
376
588
|
},
|
|
589
|
+
// failure→fix closure reads (sync; [] until the domain is open)
|
|
590
|
+
listMemories(workspace, limit) {
|
|
591
|
+
const domain = service.openedDomain();
|
|
592
|
+
return domain === void 0 ? [] : domain.listMemories(workspace, limit);
|
|
593
|
+
},
|
|
594
|
+
// failure→fix closure tags prior error rows as resolved (async)
|
|
595
|
+
tagMemory(workspace, id, tag) {
|
|
596
|
+
const domain = service.openedDomain();
|
|
597
|
+
return domain === void 0 ? Promise.resolve(false) : domain.tagMemory(workspace, id, tag);
|
|
598
|
+
},
|
|
377
599
|
};
|
|
378
600
|
const capture = makeCaptureHandler(captureStore, resolved, log, service.captureStats);
|
|
379
601
|
const disposeCapture = ctx.on("tools/result", (exec, result) => {
|
|
380
602
|
capture(exec, result);
|
|
381
603
|
});
|
|
382
604
|
disposers.push(disposeCapture);
|
|
605
|
+
|
|
606
|
+
// DSH goal-domain integration: a completed/blocked goal auto-sediments
|
|
607
|
+
// as a memory (`goal/change` session event → handoff/error row), so the
|
|
608
|
+
// outcome survives the session without any LLM call.
|
|
609
|
+
const disposeGoal = installGoalCapture(ctx, captureStore, log);
|
|
610
|
+
disposers.push(disposeGoal);
|
|
383
611
|
}
|
|
384
612
|
|
|
385
613
|
// memory GC scheduling: one sweep after the store is reachable, then on
|
package/lib/redact.js
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-engram: deterministic secret redaction for the memory write path.
|
|
3
|
+
*
|
|
4
|
+
* Auto-captured tool results, git commit subjects and error messages can
|
|
5
|
+
* carry real credentials (API keys, tokens, private keys) into long-term
|
|
6
|
+
* memory — and later straight back into a prompt via recall/[ENGRAM]. This
|
|
7
|
+
* module strips them BEFORE persistence so nothing sensitive is ever stored.
|
|
8
|
+
*
|
|
9
|
+
* Rule-based and deterministic (same input ⇒ same output, zero deps, zero
|
|
10
|
+
* LLM). Replacement markers are derived from the secret kind, so exact
|
|
11
|
+
* duplicate detection and error-revival hashing keep behaving consistently
|
|
12
|
+
* across runs. Applied once at the single store write choke point
|
|
13
|
+
* (`storeMemory`), which covers engram_store, auto-capture and any future
|
|
14
|
+
* write path.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const RULES = [
|
|
18
|
+
// Private key blocks (longest first so later rules cannot see them).
|
|
19
|
+
[/-----BEGIN [A-Z0-9 ]+PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]+PRIVATE KEY-----/g, "<REDACTED:private-key>"],
|
|
20
|
+
// JWT (header.payload.signature).
|
|
21
|
+
[/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, "<REDACTED:jwt>"],
|
|
22
|
+
// Stripe live/test keys.
|
|
23
|
+
[/\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}\b/g, "<REDACTED:stripe>"],
|
|
24
|
+
// OpenAI/Anthropic/OpenRouter style sk-… keys.
|
|
25
|
+
[/\bsk-(?:ant-)?[A-Za-z0-9_-]{16,}\b/g, "<REDACTED:sk>"],
|
|
26
|
+
// GitHub fine-grained / classic tokens.
|
|
27
|
+
[/\bgh[pousr]_[A-Za-z0-9]{20,}\b/g, "<REDACTED:github>"],
|
|
28
|
+
// AWS access key IDs.
|
|
29
|
+
[/\b(?:AKIA|ASIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA)[0-9A-Z]{16}\b/g, "<REDACTED:aws>"],
|
|
30
|
+
// Slack workspace tokens.
|
|
31
|
+
[/\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g, "<REDACTED:slack>"],
|
|
32
|
+
// Basic auth user:pass inside URLs.
|
|
33
|
+
[/\/\/[^:\/\s]+:[^@\s\/]+@/g, "//<REDACTED>@"],
|
|
34
|
+
// Authorization: Bearer <long token>.
|
|
35
|
+
[/\bBearer\s+[A-Za-z0-9._~+\/-]{16,}\b/gi, "<REDACTED:bearer>"],
|
|
36
|
+
// Generic key=value / key: value assignments for obvious secret keys.
|
|
37
|
+
[/\b(?:api[_-]?key|apikey|access[_-]?token|auth[_-]?token|client[_-]?secret|secret|passwd|password|persist[_-]?token|refresh[_-]?token|token)\b\s*[:=]\s*['"]?[A-Za-z0-9_\-.\/+=]{6,}['"]?/gi, "<REDACTED:key>"],
|
|
38
|
+
];
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Redact known secret shapes in `text`. Deterministic: identical input yields
|
|
42
|
+
* identical output. Non-secret prose is untouched.
|
|
43
|
+
*/
|
|
44
|
+
export function redactText(text) {
|
|
45
|
+
let out = String(text ?? "");
|
|
46
|
+
for (const [re, repl] of RULES) out = out.replace(re, repl);
|
|
47
|
+
return out;
|
|
48
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-engram: bounded CJK substring search over the workspace's own session
|
|
3
|
+
* corpus.
|
|
4
|
+
*
|
|
5
|
+
* WHY: the cross-session FTS fallback (`ctx.sessionQuery`) is tokenizer-blind
|
|
6
|
+
* to Chinese. The harness's session-query SQLite builds its FTS5 index with
|
|
7
|
+
* `tokenize='unicode61'` (packages/session-query/session-query-sqlite), which
|
|
8
|
+
* keeps a contiguous CJK run as ONE token — a mid-run substring like 「中间表示」
|
|
9
|
+
* never matches a session containing 「…的中间表示生成」 (verified against the
|
|
10
|
+
* harness source and empirically). The host search API additionally
|
|
11
|
+
* double-quotes every query (`quoteFtsData`), so FTS5 prefix/OR tricks like
|
|
12
|
+
* `编译*` are unavailable through it either.
|
|
13
|
+
*
|
|
14
|
+
* This module is the in-constraint fix: pure mechanical, zero LLM, bounded
|
|
15
|
+
* (newest N session files, LRU text cache, contiguous substring match), and
|
|
16
|
+
* consulted ONLY for CJK queries after the official FTS has returned zero.
|
|
17
|
+
* Same privacy scope as the FTS fallback — a single workspace === one cwd
|
|
18
|
+
* bucket under `~/.dsh/sessions`, so we never look outside the caller's
|
|
19
|
+
* workspace. Deterministic ordering: match count desc, then mtime desc.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import fs from "node:fs";
|
|
23
|
+
import path from "node:path";
|
|
24
|
+
|
|
25
|
+
import { decodeZstd, sessionsRoot } from "./usage.js";
|
|
26
|
+
import { redactText } from "./redact.js";
|
|
27
|
+
|
|
28
|
+
/** CJK ideographs (incl. Ext-A and compatibility ideographs). */
|
|
29
|
+
const CJK_RE = /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/;
|
|
30
|
+
|
|
31
|
+
/** Newest session files scanned per query (keeps the hot path bounded). */
|
|
32
|
+
const MAX_FILES = 4;
|
|
33
|
+
/** Max decompressed files kept in the LRU cache. */
|
|
34
|
+
const MAX_CACHE_FILES = 6;
|
|
35
|
+
/** Max cached decompressed bytes (~24 MB). */
|
|
36
|
+
const MAX_CACHE_BYTES = 24 * 1024 * 1024;
|
|
37
|
+
/** Snippet length around the first match. */
|
|
38
|
+
const SNIPPET = 160;
|
|
39
|
+
|
|
40
|
+
/** Does the query contain CJK ideographs? */
|
|
41
|
+
export function hasCJK(text) {
|
|
42
|
+
return CJK_RE.test(String(text));
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Map a session cwd to its bucket directory name under `~/.dsh/sessions`
|
|
47
|
+
* (`--` + workspace-key with separators flattened + `--`), mirroring the
|
|
48
|
+
* workspaceKey() normalization (win32 lowercasing included).
|
|
49
|
+
*/
|
|
50
|
+
export function bucketFromCwd(cwd) {
|
|
51
|
+
let norm = path.resolve(String(cwd)).replace(/[\\/]+$/, "");
|
|
52
|
+
if (process.platform === "win32") norm = norm.toLowerCase();
|
|
53
|
+
return `--${norm.replace(/^\/*/, "").replace(/[\\/]/g, "-")}--`;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// LRU-ish text cache keyed by file path; re-set moves the entry last.
|
|
57
|
+
const textCache = new Map(); // file -> { mtimeMs, size, text }
|
|
58
|
+
let cacheBytes = 0;
|
|
59
|
+
|
|
60
|
+
function decodedText(file, st) {
|
|
61
|
+
const hit = textCache.get(file);
|
|
62
|
+
if (hit !== void 0 && hit.mtimeMs === st.mtimeMs && hit.size === st.size) return hit.text;
|
|
63
|
+
const text = decodeZstd(file);
|
|
64
|
+
while (textCache.size >= MAX_CACHE_FILES || cacheBytes + text.length > MAX_CACHE_BYTES) {
|
|
65
|
+
const oldest = textCache.keys().next().value;
|
|
66
|
+
if (oldest === void 0) break;
|
|
67
|
+
const evicted = textCache.get(oldest);
|
|
68
|
+
textCache.delete(oldest);
|
|
69
|
+
cacheBytes -= evicted.text.length;
|
|
70
|
+
}
|
|
71
|
+
textCache.set(file, { mtimeMs: st.mtimeMs, size: st.size, text });
|
|
72
|
+
cacheBytes += text.length;
|
|
73
|
+
return text;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Count contiguous occurrences of `needle` in `text` (ASCII case-folded). */
|
|
77
|
+
export function countMatches(text, needle) {
|
|
78
|
+
let t = String(text);
|
|
79
|
+
let n = String(needle);
|
|
80
|
+
if (t.length === 0 || n.length === 0) return 0;
|
|
81
|
+
// Case-folding only matters for the non-CJK part of a mixed query.
|
|
82
|
+
if (/[a-zA-Z]/.test(n)) {
|
|
83
|
+
t = t.toLowerCase();
|
|
84
|
+
n = n.toLowerCase();
|
|
85
|
+
}
|
|
86
|
+
let count = 0;
|
|
87
|
+
let idx = 0;
|
|
88
|
+
while ((idx = t.indexOf(n, idx)) !== -1) {
|
|
89
|
+
count += 1;
|
|
90
|
+
idx += n.length;
|
|
91
|
+
}
|
|
92
|
+
return count;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function snippetAround(text, needle) {
|
|
96
|
+
const t = String(text);
|
|
97
|
+
const n = String(needle);
|
|
98
|
+
let hay = t;
|
|
99
|
+
let at = hay.indexOf(n);
|
|
100
|
+
if (at === -1 && /[a-zA-Z]/.test(n)) {
|
|
101
|
+
hay = t.toLowerCase();
|
|
102
|
+
at = hay.toLowerCase().indexOf(n.toLowerCase());
|
|
103
|
+
}
|
|
104
|
+
const start = Math.max(0, at - 60);
|
|
105
|
+
const end = Math.min(t.length, at + n.length + 100);
|
|
106
|
+
// PRIVACY: these snippets come from RAW session logs (never redacted at the
|
|
107
|
+
// source), so they run through the same deterministic secret redactor as
|
|
108
|
+
// the memory write path before any text reaches the model or the GUI.
|
|
109
|
+
const raw = redactText(t.slice(start, end).replace(/\s*\n+\s*/g, " ").replace(/\s+/g, " ").trim());
|
|
110
|
+
const prefix = start > 0 ? "…" : "";
|
|
111
|
+
const suffix = end < t.length ? "…" : "";
|
|
112
|
+
return prefix + raw.slice(0, SNIPPET) + suffix;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Bounded CJK substring search over the workspace's own recent sessions.
|
|
117
|
+
* @returns [{ sessionId, snippet, hits }] ordered by hits desc, then recency.
|
|
118
|
+
* Empty when the query has no CJK or no session file matched.
|
|
119
|
+
*/
|
|
120
|
+
export function searchSessionsCJK({
|
|
121
|
+
cwd,
|
|
122
|
+
query,
|
|
123
|
+
limit = 3,
|
|
124
|
+
maxFiles = MAX_FILES,
|
|
125
|
+
root = sessionsRoot(),
|
|
126
|
+
} = {}) {
|
|
127
|
+
const q = String(query ?? "").trim();
|
|
128
|
+
if (q.length === 0 || !hasCJK(q)) return [];
|
|
129
|
+
const bucketDir = path.join(root, bucketFromCwd(cwd));
|
|
130
|
+
let sids = [];
|
|
131
|
+
try {
|
|
132
|
+
sids = fs.readdirSync(bucketDir);
|
|
133
|
+
} catch {
|
|
134
|
+
return [];
|
|
135
|
+
}
|
|
136
|
+
const files = [];
|
|
137
|
+
for (const sid of sids) {
|
|
138
|
+
const file = path.join(bucketDir, sid, "session.jsonl.zstd");
|
|
139
|
+
let st;
|
|
140
|
+
try {
|
|
141
|
+
st = fs.statSync(file);
|
|
142
|
+
} catch {
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
files.push({ file, sid, mtimeMs: st.mtimeMs, size: st.size });
|
|
146
|
+
}
|
|
147
|
+
files.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
148
|
+
|
|
149
|
+
const matches = [];
|
|
150
|
+
for (const f of files.slice(0, maxFiles)) {
|
|
151
|
+
const text = decodedText(f.file, f);
|
|
152
|
+
const hits = countMatches(text, q);
|
|
153
|
+
if (hits > 0) {
|
|
154
|
+
matches.push({ sessionId: f.sid, snippet: snippetAround(text, q), hits, mtimeMs: f.mtimeMs });
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
matches.sort((a, b) => b.hits - a.hits || b.mtimeMs - a.mtimeMs);
|
|
158
|
+
return matches.slice(0, limit).map(({ sessionId, snippet, hits }) => ({ sessionId, snippet, hits }));
|
|
159
|
+
}
|
package/lib/settings.js
CHANGED
|
@@ -22,6 +22,9 @@ export const SETTINGS_NS = settingsNamespace(SETTINGS_NS_NAME);
|
|
|
22
22
|
export const SETTINGS_KEYS = [
|
|
23
23
|
"autoCapture",
|
|
24
24
|
"sessionSearch",
|
|
25
|
+
"recallScope",
|
|
26
|
+
"autoRecallOnStart",
|
|
27
|
+
"autoRecallLimit",
|
|
25
28
|
"maxRecallPerSession",
|
|
26
29
|
"autoCapturePerSession",
|
|
27
30
|
"indexMaxLines",
|
|
@@ -34,6 +37,10 @@ export const SETTINGS_KEYS = [
|
|
|
34
37
|
"gcStableRetentionDays",
|
|
35
38
|
"gcReplacesCompaction",
|
|
36
39
|
"gcNarrative",
|
|
40
|
+
"verifyArtifact",
|
|
41
|
+
"autoSupersede",
|
|
42
|
+
"autoSinkTodosOnEnd",
|
|
43
|
+
"autoWebCompaction",
|
|
37
44
|
"trustedHosts",
|
|
38
45
|
];
|
|
39
46
|
|
|
@@ -51,6 +58,9 @@ export function makeSettingsSchema(defaults) {
|
|
|
51
58
|
return z.object({
|
|
52
59
|
autoCapture: z.boolean().default(defaults.autoCapture),
|
|
53
60
|
sessionSearch: z.boolean().default(defaults.sessionSearch),
|
|
61
|
+
recallScope: z.union(["workspace", "global"]).default(defaults.recallScope ?? "workspace").description("召回范围:workspace=严格按当前工作区隔离(默认、token 纪律);global=本工作区命中弱/空时,从其它工作区池按 recency/origin 加权召回并标 [W:<workspace>] 来源。开启会打破跨工作区隔离,请按需选用"),
|
|
62
|
+
autoRecallOnStart: z.boolean().default(defaults.autoRecallOnStart).description("新会话开始时,宿主按首条消息对工作区做确定性召回,把命中记忆直接注入 [RECALL] 块(无需模型自己想起调 engram_recall;关掉=回到纯拉取)"),
|
|
63
|
+
autoRecallLimit: z.natural().min(1).max(8).default(defaults.autoRecallLimit).description("[RECALL] 块最多注入的记忆条数(agentmemory 候选①:少而准)"),
|
|
54
64
|
maxRecallPerSession: z.natural().min(1).max(10).default(defaults.maxRecallPerSession),
|
|
55
65
|
autoCapturePerSession: z.natural().max(1000).default(defaults.autoCapturePerSession),
|
|
56
66
|
indexMaxLines: z.natural().max(50).default(defaults.indexMaxLines),
|
|
@@ -64,6 +74,9 @@ export function makeSettingsSchema(defaults) {
|
|
|
64
74
|
gcReplacesCompaction: z.boolean().default(defaults.gcReplacesCompaction),
|
|
65
75
|
gcNarrative: z.boolean().default(defaults.gcNarrative),
|
|
66
76
|
verifyArtifact: z.boolean().default(defaults.verifyArtifact).description("ESR 闭环时校验 artifact 路径真实存在"),
|
|
77
|
+
autoSupersede: z.boolean().default(defaults.autoSupersede).description("实体锚定 + 替换式更新(改用/不再…)自动把旧记忆标为 superseded(默认关)"),
|
|
78
|
+
autoSinkTodosOnEnd: z.boolean().default(defaults.autoSinkTodosOnEnd).description("会话结束时把仍 pending 的 todo 自动沉淀为 ESR draft 任务(去重、受容量上限约束;关掉=会话结束即弃)"),
|
|
79
|
+
autoWebCompaction: z.boolean().default(defaults.autoWebCompaction).description("web 面启动时把全部 stock 布局的 agent 预设(shipped + 用户根)的 compaction 组自动换成 dsh-engram/compaction(带备份、幂等、不碰自定义/无 compaction 预设;关掉=不再自动接管,已接管的预设保持 Context GC、需 npm run web-compaction:revert 还原,host 面不受影响)"),
|
|
67
80
|
// Security knob, not rendered in the card: hostnames allowed past the
|
|
68
81
|
// loopback fence for /api/dsh-engram (operator-authorized tunnel hosts).
|
|
69
82
|
trustedHosts: z.array(z.string()).default([]),
|
package/lib/status.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable Context GC status artifact.
|
|
3
|
+
*
|
|
4
|
+
* The plugin writes a tiny JSON file under the DSH home at boot once the host
|
|
5
|
+
* compaction engine and the web-plane wiring have both settled. It is the
|
|
6
|
+
* offline, inspectable truth for "is Context GC actually active and where?" —
|
|
7
|
+
* readable by the shipped `dsh-engram` CLI and by support/diagnostics without
|
|
8
|
+
* booting a session or parsing startup logs.
|
|
9
|
+
* @module dsh-engram/status
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
13
|
+
import { dirname, join } from "node:path";
|
|
14
|
+
import { homedir } from "node:os";
|
|
15
|
+
|
|
16
|
+
/** The DSH home directory (env override, else `~/.dsh`). */
|
|
17
|
+
export function engramHomeDir() {
|
|
18
|
+
return process.env.DSH_HOME !== undefined && process.env.DSH_HOME.length > 0
|
|
19
|
+
? process.env.DSH_HOME
|
|
20
|
+
: join(homedir(), ".dsh");
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Absolute path of the Context GC status file. */
|
|
24
|
+
export function contextGcStatusPath() {
|
|
25
|
+
return join(engramHomeDir(), "engram", "context-gc.status.json");
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Atomically write the status snapshot (tmp + rename). Never throws for the
|
|
30
|
+
* caller's convenience; a failure is logged by the caller if it cares.
|
|
31
|
+
* @param {object} payload
|
|
32
|
+
* @returns {Promise<void>}
|
|
33
|
+
*/
|
|
34
|
+
export async function writeContextGcStatus(payload) {
|
|
35
|
+
const path = contextGcStatusPath();
|
|
36
|
+
await mkdir(dirname(path), { recursive: true });
|
|
37
|
+
const tmp = `${path}.tmp`;
|
|
38
|
+
await writeFile(
|
|
39
|
+
tmp,
|
|
40
|
+
`${JSON.stringify({ version: 1, ...payload, writtenAt: Date.now() }, null, 2)}\n`,
|
|
41
|
+
"utf8",
|
|
42
|
+
);
|
|
43
|
+
await rename(tmp, path);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Read the last status snapshot. Returns `undefined` when none exists yet
|
|
48
|
+
* (safe to treat as "not yet initialized").
|
|
49
|
+
* @returns {Promise<Record<string, any> | undefined>}
|
|
50
|
+
*/
|
|
51
|
+
export async function readContextGcStatus() {
|
|
52
|
+
try {
|
|
53
|
+
return JSON.parse(await readFile(contextGcStatusPath(), "utf8"));
|
|
54
|
+
} catch {
|
|
55
|
+
return undefined;
|
|
56
|
+
}
|
|
57
|
+
}
|