dsh-engram 0.7.2 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +104 -7
- package/README.zh.md +13 -3
- package/docs/CONTEXT-GC-GUIDE.zh.md +15 -1
- package/docs/G6-quality-ab.zh.md +540 -0
- package/docs/GATES.zh.md +41 -0
- package/docs/PROPOSAL-evolution.zh.md +12 -0
- package/docs/RELEASE-NOTES-v0.7.10.md +49 -0
- package/docs/RELEASE-NOTES-v0.7.11.md +31 -0
- package/docs/RELEASE-NOTES-v0.7.12.md +22 -0
- package/docs/RELEASE-NOTES-v0.7.13.md +41 -0
- package/docs/RELEASE-NOTES-v0.7.14.md +37 -0
- package/docs/RELEASE-NOTES-v0.7.15.md +36 -0
- package/docs/RELEASE-NOTES-v0.7.16.md +42 -0
- package/docs/RELEASE-NOTES-v0.7.3.md +30 -0
- package/docs/RELEASE-NOTES-v0.7.4.md +71 -0
- package/docs/RELEASE-NOTES-v0.7.5.md +134 -0
- package/docs/RELEASE-NOTES-v0.7.6.md +74 -0
- package/docs/RELEASE-NOTES-v0.7.7.md +60 -0
- package/docs/RELEASE-NOTES-v0.7.8.md +56 -0
- package/docs/RELEASE-NOTES-v0.7.9.md +46 -0
- package/docs/RELEASE-NOTES-v0.8.0.md +59 -0
- package/docs/RELEASE-NOTES-v0.8.1.md +70 -0
- package/docs/RELEASE-NOTES-v0.8.2.md +62 -0
- package/docs/RELEASE-NOTES-v0.8.3.md +65 -0
- package/docs/RELEASE-NOTES-v0.8.4.md +56 -0
- package/docs/RELEASE-NOTES-v0.8.5.md +60 -0
- package/docs/RELEASE-NOTES-v0.8.6.md +71 -0
- package/eval/gate-audit.mjs +538 -0
- package/lib/api.js +70 -0
- package/lib/approvals.js +84 -0
- package/lib/boundary-prune.js +71 -5
- package/lib/capsule.js +134 -0
- package/lib/capture.js +85 -6
- package/lib/client.js +1813 -1745
- package/lib/compaction.js +4 -11
- package/lib/context-gc.js +63 -2
- package/lib/episode.js +10 -5
- package/lib/index-block.js +89 -8
- package/lib/index.js +108 -21
- package/lib/ledger.js +200 -10
- package/lib/pack-telemetry.js +10 -3
- package/lib/preference.js +2 -1
- package/lib/prune.js +89 -0
- package/lib/session-events.js +168 -0
- package/lib/skill-export.js +289 -0
- package/lib/status-insights.js +5 -0
- package/lib/status.js +138 -0
- package/lib/store.js +34 -0
- package/lib/todo-sink.js +3 -2
- package/lib/tools.js +2 -1
- package/lib/usage.js +32 -3
- package/lib/web-provision.js +59 -14
- package/package.json +15 -8
- package/scripts/dsh-engram.mjs +229 -4
package/lib/compaction.js
CHANGED
|
@@ -62,17 +62,10 @@ export function apply(ctx, config = {}) {
|
|
|
62
62
|
if (!domain) return null;
|
|
63
63
|
return collectPointerContext(domain, wk(workspace), report);
|
|
64
64
|
},
|
|
65
|
-
//
|
|
66
|
-
//
|
|
67
|
-
//
|
|
68
|
-
//
|
|
69
|
-
// one durable ledger instead of vanishing with the realm.
|
|
70
|
-
onCompacted: ({ session, workspace, charsSaved }) => {
|
|
71
|
-
try {
|
|
72
|
-
const hub = ctx.get("engramLedger");
|
|
73
|
-
hub?.noteCompaction?.(session?.id, workspace, { charsSaved });
|
|
74
|
-
} catch { /* contained: accounting never breaks compaction */ }
|
|
75
|
-
},
|
|
65
|
+
// No ledger bridge here: crediting inside `summarize()` claims a
|
|
66
|
+
// saving before the harness decides whether it commits the
|
|
67
|
+
// checkpoint (a rejected/overridden summary saves nothing). The host
|
|
68
|
+
// plugin credits from the committed `compaction/summary` event.
|
|
76
69
|
}
|
|
77
70
|
: {};
|
|
78
71
|
|
package/lib/context-gc.js
CHANGED
|
@@ -33,6 +33,7 @@
|
|
|
33
33
|
|
|
34
34
|
import { basename } from "node:path";
|
|
35
35
|
import { slugId } from "./util.js";
|
|
36
|
+
import { recordGcAttempt } from "./status.js";
|
|
36
37
|
|
|
37
38
|
/** Memory ids are uuids; the [ENGRAM] index shows `#` + first 8 alnum chars. */
|
|
38
39
|
const MEMORY_ID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi;
|
|
@@ -608,6 +609,43 @@ export async function summarizeUnprovenanced(messages, { enabled = true, maxChar
|
|
|
608
609
|
* Build the Context GC replacement summary (pointer block + narrative). This
|
|
609
610
|
* is the text that lands inside DSH's `<compacted-summary>` checkpoint node.
|
|
610
611
|
*/
|
|
612
|
+
/** The line every Context GC checkpoint carries — proof of authorship when a
|
|
613
|
+
* committed summary is read back from the session log. */
|
|
614
|
+
export const CONTEXT_GC_HEADING = "## Context GC — evicted detail is RE-FETCHABLE (do not re-derive)";
|
|
615
|
+
export const CONTEXT_GC_FOOTER = "_Context GC: detail above is re-fetchable via the listed tools. Prefer re-fetching over re-deriving._";
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* Turn a committed `compaction/summary` event into a Context Ledger credit, or
|
|
619
|
+
* `null` when engram did not author it.
|
|
620
|
+
*
|
|
621
|
+
* The credit has to come from the COMMITTED record, not from inside
|
|
622
|
+
* `summarize()`: the harness may reject a checkpoint (framed ≥ shadowed) or fall
|
|
623
|
+
* back to its own summarizer, and a saving that never reached the session is not
|
|
624
|
+
* a saving. `shadowedTokenCount` is the span that stopped replaying, the summary
|
|
625
|
+
* text is what replaced it.
|
|
626
|
+
*/
|
|
627
|
+
export function committedGcSaving(event, charsPerToken = 4) {
|
|
628
|
+
if (event?.type !== "compaction/summary") return null;
|
|
629
|
+
// DSH publishes committed events as { type, seq, time, data: { summary,
|
|
630
|
+
// shadowedTokenCount, … } }. Reading the payload off the event root made this
|
|
631
|
+
// matcher return null for EVERY real checkpoint — 82 GC-authored summaries on
|
|
632
|
+
// this machine, none credited — while the unit test kept passing because it
|
|
633
|
+
// built a flat synthetic event. Accept both shapes; prefer the real one.
|
|
634
|
+
const payload = event?.data !== null && typeof event?.data === "object" ? event.data : event;
|
|
635
|
+
const text = Array.isArray(payload?.summary)
|
|
636
|
+
? payload.summary.map((part) => (typeof part?.text === "string" ? part.text : "")).join("")
|
|
637
|
+
: "";
|
|
638
|
+
// Either end identifies the checkpoint: the heading survives a trim (it is
|
|
639
|
+
// the head), the footer is the cheaper full-text check. A checkpoint trimmed
|
|
640
|
+
// to fit keeps the heading but may lose the footer, and it still committed.
|
|
641
|
+
const authored = text.includes(CONTEXT_GC_FOOTER) || [...text].slice(0, 200).join("").includes(CONTEXT_GC_HEADING);
|
|
642
|
+
if (!authored) return null;
|
|
643
|
+
const shadowed = Number(payload?.shadowedTokenCount ?? event?.shadowedTokenCount);
|
|
644
|
+
if (!Number.isFinite(shadowed) || shadowed <= 0) return null;
|
|
645
|
+
const charsSaved = shadowed * charsPerToken - [...text].length;
|
|
646
|
+
return charsSaved > 0 ? { charsSaved, shadowedTokens: shadowed, summaryChars: [...text].length } : null;
|
|
647
|
+
}
|
|
648
|
+
|
|
611
649
|
export function buildContextGcSummary(report, context, { turns, narrative }) {
|
|
612
650
|
const label = context?.label ?? "this workspace";
|
|
613
651
|
const pointer = buildPointerSummary(report, context, { turns, label });
|
|
@@ -617,7 +655,7 @@ export function buildContextGcSummary(report, context, { turns, narrative }) {
|
|
|
617
655
|
narrative.text,
|
|
618
656
|
"",
|
|
619
657
|
"---",
|
|
620
|
-
|
|
658
|
+
CONTEXT_GC_FOOTER,
|
|
621
659
|
];
|
|
622
660
|
return parts.join("\n");
|
|
623
661
|
}
|
|
@@ -732,15 +770,37 @@ export async function loadContextGcEngine(ctx, config = {}, deps = {}) {
|
|
|
732
770
|
constructor() {
|
|
733
771
|
super(ctx, engineConfig);
|
|
734
772
|
this.gcDeps = deps;
|
|
773
|
+
/** Last credit computed by `_contextGc`, surfaced in the attempt log. */
|
|
774
|
+
this.gcLastCredit = void 0;
|
|
735
775
|
}
|
|
736
776
|
|
|
737
777
|
/** The one seam we take over: produce the checkpoint summary. */
|
|
738
778
|
async summarize(input, agent, signal) {
|
|
779
|
+
const startedAt = Date.now();
|
|
780
|
+
const workspace = agent?.session?.header?.cwd;
|
|
739
781
|
try {
|
|
740
|
-
|
|
782
|
+
const result = await this._contextGc(input, agent, signal);
|
|
783
|
+
// Evidence, not vibes: the ledger's credit side only moves when THIS
|
|
784
|
+
// path runs, so every attempt is recorded where it can be inspected
|
|
785
|
+
// offline (a silent fallback used to look identical to a working GC).
|
|
786
|
+
// The head + length identify the checkpoint: if the committed
|
|
787
|
+
// `compaction/summary` in the session log does not start like this, the
|
|
788
|
+
// harness used someone else's summary and nothing was saved.
|
|
789
|
+
const text = Array.isArray(result?.summary) ? result.summary.map((b) => (typeof b?.text === "string" ? b.text : "")).join("") : "";
|
|
790
|
+
await recordGcAttempt({
|
|
791
|
+
plane: "compaction",
|
|
792
|
+
ok: true,
|
|
793
|
+
ms: Date.now() - startedAt,
|
|
794
|
+
workspace,
|
|
795
|
+
charsSaved: this.gcLastCredit?.charsSaved ?? 0,
|
|
796
|
+
summaryChars: [...text].length,
|
|
797
|
+
head: [...text].slice(0, 60).join(""),
|
|
798
|
+
});
|
|
799
|
+
return result;
|
|
741
800
|
} catch (error) {
|
|
742
801
|
const logger = this.ctx?.logger;
|
|
743
802
|
const message = error instanceof Error ? error.message : String(error);
|
|
803
|
+
await recordGcAttempt({ plane: "compaction", ok: false, ms: Date.now() - startedAt, workspace, error: message });
|
|
744
804
|
if (typeof logger?.warn === "function") logger.warn(`engram context-gc failed (${message}); falling back to default compaction`);
|
|
745
805
|
if (typeof this.gcDeps.defaultSummarize === "function") {
|
|
746
806
|
return this.gcDeps.defaultSummarize(input, agent, signal);
|
|
@@ -804,6 +864,7 @@ export async function loadContextGcEngine(ctx, config = {}, deps = {}) {
|
|
|
804
864
|
const summaryChars = [...summaryText].length;
|
|
805
865
|
const charsSaved = spanChars - summaryChars;
|
|
806
866
|
if (charsSaved > 0) {
|
|
867
|
+
this.gcLastCredit = { charsSaved, spanChars, summaryChars };
|
|
807
868
|
this.gcDeps.onCompacted({ session: agent?.session, workspace: ws, charsSaved, spanChars, summaryChars });
|
|
808
869
|
}
|
|
809
870
|
}
|
package/lib/episode.js
CHANGED
|
@@ -6,7 +6,9 @@
|
|
|
6
6
|
* later succeeded), the written commit subjects, and what is still open.
|
|
7
7
|
*
|
|
8
8
|
* Design constraints:
|
|
9
|
-
* - zero LLM, pure function over
|
|
9
|
+
* - zero LLM, pure function over the session event log (read through
|
|
10
|
+
* `readSessionEvents`, so it survives restarts and DSH ≥0.2.x, where the
|
|
11
|
+
* old `session.events` getter no longer exists);
|
|
10
12
|
* - never enters the [ENGRAM] index (long background context, not a symbol);
|
|
11
13
|
* - bounded by construction (≤4 files / ≤4 commands / ≤2 decisions / ≤3 open);
|
|
12
14
|
* - stored only when there is substance (a bare `ls` session produces none);
|
|
@@ -17,6 +19,7 @@
|
|
|
17
19
|
|
|
18
20
|
import { cmdTag } from "./capture.js";
|
|
19
21
|
import { pendingTodosFromSession } from "./todo-sink.js";
|
|
22
|
+
import { readSessionEvents } from "./session-events.js";
|
|
20
23
|
|
|
21
24
|
const EDIT_TOOLS = new Set(["write", "edit", "str_replace_editor"]);
|
|
22
25
|
const RUN_TOOLS = new Set(["bash", "shell", "exec"]);
|
|
@@ -59,13 +62,15 @@ function commitSubject(command) {
|
|
|
59
62
|
/**
|
|
60
63
|
* Build the episode record from a disposed session.
|
|
61
64
|
*
|
|
62
|
-
* @param {object} session session
|
|
65
|
+
* @param {object} session host session (`.header.cwd` + a readable event log)
|
|
63
66
|
* @param {object} [opts]
|
|
67
|
+
* @param {number} [opts.now] clock injection for the header stamp
|
|
68
|
+
* @param {object} [opts.sessionQuery] `ctx.sessionQuery` — how ≥0.2.x logs are read
|
|
64
69
|
* @returns {{text: string, tags: string[], signal: number, summary: object}|null}
|
|
65
70
|
*/
|
|
66
|
-
export function buildEpisode(session, { now = Date.now() } = {}) {
|
|
71
|
+
export function buildEpisode(session, { now = Date.now(), sessionQuery = void 0 } = {}) {
|
|
67
72
|
if (session === null || typeof session !== "object") return null;
|
|
68
|
-
const events =
|
|
73
|
+
const events = readSessionEvents(session, sessionQuery);
|
|
69
74
|
if (events.length === 0) return null;
|
|
70
75
|
const cwd = session?.header?.cwd;
|
|
71
76
|
|
|
@@ -119,7 +124,7 @@ export function buildEpisode(session, { now = Date.now() } = {}) {
|
|
|
119
124
|
}
|
|
120
125
|
}
|
|
121
126
|
|
|
122
|
-
const open = pendingTodosFromSession(session).slice(0, 3);
|
|
127
|
+
const open = pendingTodosFromSession(session, sessionQuery).slice(0, 3);
|
|
123
128
|
const hasSubstance = files.size > 0 || decisions.length > 0 || failedRuns > 0 || open.length > 0;
|
|
124
129
|
if (!hasSubstance) return null;
|
|
125
130
|
|
package/lib/index-block.js
CHANGED
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
import { label, shortId, fmtDate, escapeLt, truncate, byRecency } from "./util.js";
|
|
13
13
|
import { buildEntityPage, renderEntityCard } from "./entity-page.js";
|
|
14
14
|
import { memoryIsStale } from "./anchor.js";
|
|
15
|
+
import { readSessionEvents } from "./session-events.js";
|
|
16
|
+
import fs from "node:fs";
|
|
15
17
|
|
|
16
18
|
/** Deterministic index line prefix per memory kind. */
|
|
17
19
|
export function prefixFor(kind) {
|
|
@@ -41,9 +43,77 @@ function isGitPlumbingEcho(m) {
|
|
|
41
43
|
return /(^|\s)(cd\s|\&\&|;|\|)/.test(text);
|
|
42
44
|
}
|
|
43
45
|
|
|
44
|
-
/**
|
|
46
|
+
/**
|
|
47
|
+
* Has the file this memory is anchored to changed since the memory was written?
|
|
48
|
+
*
|
|
49
|
+
* The reverse G6 experiment (docs/G6-quality-ab.zh.md §9) is why this exists:
|
|
50
|
+
* with a stale convention in memory and a contradicting docstring in the code,
|
|
51
|
+
* the agent followed memory on 1 of 6 tasks because nothing marked the memory as
|
|
52
|
+
* old. A memory that names a file can carry that check for free — one `stat`,
|
|
53
|
+
* and the injected line can say "the file moved on".
|
|
54
|
+
*
|
|
55
|
+
* Returns "" when there is nothing to say (no anchor, file unchanged, or the
|
|
56
|
+
* stat fails); a short marker otherwise. Never throws: injection must not break
|
|
57
|
+
* because a path is unreadable.
|
|
58
|
+
*/
|
|
59
|
+
export function fileDrift(m, nowMs = Date.now(), stat = void 0) {
|
|
60
|
+
const file = typeof m?.filePath === "string" ? m.filePath : "";
|
|
61
|
+
if (file.length === 0) return "";
|
|
62
|
+
const written = m?.createdAt ?? m?.updatedAt;
|
|
63
|
+
if (written === null || written === void 0) return "";
|
|
64
|
+
const writtenMs = typeof written === "number" ? written : Date.parse(String(written));
|
|
65
|
+
if (!Number.isFinite(writtenMs) || writtenMs > nowMs) return "";
|
|
66
|
+
const doStat = stat ?? ((target) => fs.statSync(target));
|
|
67
|
+
let statInfo;
|
|
68
|
+
try {
|
|
69
|
+
statInfo = doStat(file);
|
|
70
|
+
} catch {
|
|
71
|
+
return "⚠file-missing";
|
|
72
|
+
}
|
|
73
|
+
const mtimeMs = statInfo?.mtimeMs ?? 0;
|
|
74
|
+
if (!Number.isFinite(mtimeMs) || mtimeMs <= writtenMs) return "";
|
|
75
|
+
// Only a coarse signal: "the anchored file was edited after this was written".
|
|
76
|
+
// The model can then check the file instead of trusting the note.
|
|
77
|
+
return "⚠file-changed";
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* v0.8-A promotion rules for procedures.
|
|
82
|
+
*
|
|
83
|
+
* A procedure earns an index line only when it is still true:
|
|
84
|
+
* - verified at least `procedureMinEvidence` (3) times,
|
|
85
|
+
* - verified within `procedureMaxAgeDays` (14),
|
|
86
|
+
* - and not failed since its last verification.
|
|
87
|
+
* A manual procedure with no evidence fields keeps the older hit-based path
|
|
88
|
+
* (`hits >= promoteHits`), so hand-written procedures are not silently demoted.
|
|
89
|
+
*/
|
|
90
|
+
export function procedureInvalidated(m) {
|
|
91
|
+
if ((m?.kind ?? "") !== "procedure") return false;
|
|
92
|
+
if (m?.lastFailedAt === null || m?.lastFailedAt === void 0) return false;
|
|
93
|
+
return m.lastVerifiedAt === null || m.lastVerifiedAt === void 0 || m.lastFailedAt >= m.lastVerifiedAt;
|
|
94
|
+
}
|
|
95
|
+
|
|
45
96
|
export function isProvenProcedure(m, config) {
|
|
46
|
-
|
|
97
|
+
if ((m?.kind ?? "") !== "procedure") return false;
|
|
98
|
+
if (procedureInvalidated(m)) return false;
|
|
99
|
+
const nowMs = config?.nowMs ?? Date.now();
|
|
100
|
+
const minEvidence = config?.procedureMinEvidence ?? 3;
|
|
101
|
+
const maxAgeMs = (config?.procedureMaxAgeDays ?? 14) * 24 * 60 * 60 * 1000;
|
|
102
|
+
if ((m.evidenceCount ?? 0) > 0) {
|
|
103
|
+
const verified = m.lastVerifiedAt ?? 0;
|
|
104
|
+
return (m.evidenceCount ?? 0) >= minEvidence && verified > 0 && nowMs - verified <= maxAgeMs;
|
|
105
|
+
}
|
|
106
|
+
return (m?.hits ?? 0) >= (config?.promoteHits ?? 3);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* An auto-captured `fixed:` row is a hypothesis until it has evidence — it
|
|
111
|
+
* must not earn an index line on the strength of one success.
|
|
112
|
+
*/
|
|
113
|
+
export function isUnprovenAutoProcedure(m, config) {
|
|
114
|
+
if ((m?.kind ?? "") !== "procedure") return false;
|
|
115
|
+
if (!Array.isArray(m.tags) || !m.tags.includes("auto-captured")) return false;
|
|
116
|
+
return !isProvenProcedure(m, config);
|
|
47
117
|
}
|
|
48
118
|
|
|
49
119
|
/**
|
|
@@ -66,6 +136,10 @@ export function selectIndexMemories(store, workspace, config) {
|
|
|
66
136
|
// A memory whose file source changed/vanished is no longer current truth
|
|
67
137
|
// for an auto-injected index line (still re-fetchable via engram_detail).
|
|
68
138
|
.filter((m) => !memoryIsStale(m))
|
|
139
|
+
// v0.8-A: a just-invalidated procedure is worse than none — the next agent
|
|
140
|
+
// would copy it verbatim. Unproven auto-captures stay out until they earn it.
|
|
141
|
+
.filter((m) => !procedureInvalidated(m))
|
|
142
|
+
.filter((m) => !isUnprovenAutoProcedure(m, config))
|
|
69
143
|
.filter(
|
|
70
144
|
(m) =>
|
|
71
145
|
(isGitPlumbingEcho(m) && (m.hits ?? 0) < config.promoteHits
|
|
@@ -98,7 +172,8 @@ function memoryLine(m, config) {
|
|
|
98
172
|
const mark = isProvenProcedure(m, config) ? `${px}✓` : px;
|
|
99
173
|
const date = fmtDate(m.createdAt ?? m.updatedAt);
|
|
100
174
|
const text = truncate(String(m.text).replace(/\s*\n+\s*/g, " ").trim(), INDEX_LINE_CHARS);
|
|
101
|
-
|
|
175
|
+
const drift = fileDrift(m, config?.nowMs ?? Date.now(), config?.stat);
|
|
176
|
+
return `[${mark}] ${date} ${escapeLt(text)}${drift === "" ? "" : ` ${drift}`} #${shortId(m.id)}`;
|
|
102
177
|
}
|
|
103
178
|
|
|
104
179
|
function taskStateLine(t) {
|
|
@@ -190,13 +265,18 @@ function enforceCharCap(lines, maxChars) {
|
|
|
190
265
|
}
|
|
191
266
|
|
|
192
267
|
/** Text of the session's first user message (the opening instruction of the
|
|
193
|
-
* conversation). Walk
|
|
268
|
+
* conversation). Walk the session log for the first `user/message` event and
|
|
194
269
|
* join its `data.content` text blocks; fall back to the first text-bearing
|
|
195
270
|
* event ("goal"/reminder start) guardedly. Returns "" when there is none.
|
|
196
|
-
* Pure, deterministic — the auto-recall (B) keys its query off this.
|
|
197
|
-
|
|
271
|
+
* Pure, deterministic — the auto-recall (B) keys its query off this.
|
|
272
|
+
*
|
|
273
|
+
* The log is read through `readSessionEvents` because DSH 0.2.x dropped the
|
|
274
|
+
* `session.events` getter: reading it directly returned `[]` on every real
|
|
275
|
+
* session, which silently killed the whole [RECALL] push leg (and, via
|
|
276
|
+
* `pendingRecall`, kept the index block from ever being cached). */
|
|
277
|
+
export function firstUserMessageText(session, sessionQuery = void 0) {
|
|
198
278
|
if (session === null || typeof session !== "object") return "";
|
|
199
|
-
const events =
|
|
279
|
+
const events = readSessionEvents(session, sessionQuery);
|
|
200
280
|
if (events.length === 0) return "";
|
|
201
281
|
const textOf = (data) => {
|
|
202
282
|
if (data === null || typeof data !== "object") return "";
|
|
@@ -268,7 +348,8 @@ export function renderStartRecall(store, workspace, config = {}, opts = {}) {
|
|
|
268
348
|
const truncated = [...full].length > 140;
|
|
269
349
|
const text = truncate(full, 140);
|
|
270
350
|
if (truncated && truncatedId === null) truncatedId = shortId(m.id);
|
|
271
|
-
|
|
351
|
+
const drift = fileDrift(m, config?.nowMs ?? Date.now(), config?.stat);
|
|
352
|
+
lines.push(`- [${mark}] ${date} ${escapeLt(text)}${drift === "" ? "" : ` ${drift}`} #${shortId(m.id)}${truncated ? " …" : ""}${(m.hits ?? 0) > 0 ? ` ×${m.hits}` : ""}`);
|
|
272
353
|
}
|
|
273
354
|
const drill = truncatedId !== null
|
|
274
355
|
? `context: use above · #${truncatedId} truncated — engram_detail ${truncatedId} (full) · engram_recall <query>`
|
package/lib/index.js
CHANGED
|
@@ -27,18 +27,19 @@ import { installPreferenceCapture } from "./preference.js";
|
|
|
27
27
|
import { boundaryPruneAvailability, installBoundaryPrune } from "./boundary-prune.js";
|
|
28
28
|
import { resultPackDir, archivePrunedResults, resultRecallTool, sweepExpired } from "./result-pack.js";
|
|
29
29
|
import { recordBoundaryPruneFire, recordRecallInjection, readPackTelemetry, engramHome } from "./pack-telemetry.js";
|
|
30
|
-
import { createLedger, estimatePrefixChars } from "./ledger.js";
|
|
30
|
+
import { CHARS_PER_TOKEN, createLedger, estimatePrefixChars } from "./ledger.js";
|
|
31
31
|
import { createJit } from "./jit.js";
|
|
32
32
|
import { buildEpisode } from "./episode.js";
|
|
33
33
|
import { collectStaleAnchors } from "./anchor.js";
|
|
34
34
|
import { renderIndex, renderStartRecall, renderResumeBrief, firstUserMessageText, renderEsr, esrMethodology, buildEsrSnapshot, buildFrozenEsrBlock } from "./index-block.js";
|
|
35
|
+
import { describeSessionSurface, sessionEventsSource, sessionSeq } from "./session-events.js";
|
|
35
36
|
import { makeTriggerRecorder } from "./trigger.js";
|
|
36
37
|
import { pendingTodosFromSession, sinkPendingTodos } from "./todo-sink.js";
|
|
37
38
|
import { installEngramSettings, makeEngineConfigSchema, SETTINGS_KEYS } from "./settings.js";
|
|
38
39
|
import { makeEngramRoutes } from "./api.js";
|
|
39
|
-
import { collectPointerContext, mountCompactionEngine } from "./context-gc.js";
|
|
40
|
+
import { collectPointerContext, committedGcSaving, mountCompactionEngine } from "./context-gc.js";
|
|
40
41
|
import { provisionWebCompaction } from "./web-provision.js";
|
|
41
|
-
import { writeContextGcStatus } from "./status.js";
|
|
42
|
+
import { writeContextGcStatus, recordPruneAttempt } from "./status.js";
|
|
42
43
|
import { workspaceKey as wk } from "./util.js";
|
|
43
44
|
|
|
44
45
|
/** Stable cordis plugin name. */
|
|
@@ -80,11 +81,23 @@ const DEFAULTS = {
|
|
|
80
81
|
// memories that outrank the [ENGRAM] index.
|
|
81
82
|
autoCapturePreferences: true,
|
|
82
83
|
preferenceCapturePerSession: 8,
|
|
84
|
+
// v0.8-C (advisory): a subagent report above this many chars is distilled
|
|
85
|
+
// into a durable `handoff` capsule. The parent still receives the full text —
|
|
86
|
+
// DSH's tools/result is observer-only (no mutation channel) — so this buys
|
|
87
|
+
// durability and a measurable saving, not a smaller prompt. See lib/capsule.js.
|
|
88
|
+
capsuleMinSourceChars: 1500,
|
|
89
|
+
capsuleMaxChars: 1800,
|
|
83
90
|
// [ENGRAM] index budget (token discipline)
|
|
84
91
|
indexMaxLines: 12,
|
|
85
92
|
indexMaxChars: 700,
|
|
86
93
|
minIndexSignal: 0.4,
|
|
87
94
|
promoteHits: 3,
|
|
95
|
+
// v0.8-A evidence-based procedure promotion: a `fixed:` procedure reaches the
|
|
96
|
+
// [ENGRAM] index only after this many verified successes, and only while the
|
|
97
|
+
// last verification is younger than procedureMaxAgeDays. A later failure
|
|
98
|
+
// invalidates it immediately (see store.recordProcedureOutcome).
|
|
99
|
+
procedureMinEvidence: 3,
|
|
100
|
+
procedureMaxAgeDays: 14,
|
|
88
101
|
// retention / caps
|
|
89
102
|
expireDays: 180,
|
|
90
103
|
maxMemoriesPerWorkspace: 2000,
|
|
@@ -185,9 +198,13 @@ const DEFAULTS = {
|
|
|
185
198
|
// (deterministic surface-replace, zero LLM). L0 simulation (77 sessions):
|
|
186
199
|
// todo-progress boundaries cover 79% of replay exposure; the repayment
|
|
187
200
|
// gate below lifted cache-aware net savings 12–32% on every threshold.
|
|
188
|
-
//
|
|
189
|
-
//
|
|
190
|
-
|
|
201
|
+
//
|
|
202
|
+
// Default OFF since v0.7.11: 15 h of live accounting showed the host prunes
|
|
203
|
+
// over-threshold results by itself (64,200 chars across two sessions with
|
|
204
|
+
// zero engram involvement), while the boundary source never fired in an
|
|
205
|
+
// ordinary 220-request session — it only pays for itself in planning
|
|
206
|
+
// sessions that actually use todos/goals. `true` is the opt-in.
|
|
207
|
+
boundaryPrune: false,
|
|
191
208
|
// Adaptive off-switch (v0.6-B): the Context Ledger vetoes further fires in a
|
|
192
209
|
// workspace whose own accounting is net negative after real fires. Evidence
|
|
193
210
|
// can only turn pruning OFF — never on. `false` = repayment gate only.
|
|
@@ -421,6 +438,8 @@ export function apply(ctx, config) {
|
|
|
421
438
|
|
|
422
439
|
const service = {
|
|
423
440
|
config: resolved,
|
|
441
|
+
/** Where local (non-store) engram state lives — approvals, telemetry, ledger. */
|
|
442
|
+
engramHome: () => engramHome(),
|
|
424
443
|
getDomain,
|
|
425
444
|
openedDomain: () => opened,
|
|
426
445
|
recorder,
|
|
@@ -522,6 +541,8 @@ export function apply(ctx, config) {
|
|
|
522
541
|
// [ESR] block below follows the same frozen-snapshot rule but additionally
|
|
523
542
|
// accumulates this-session actionables (see esrBlockFor).
|
|
524
543
|
const sessionCache = new WeakMap();
|
|
544
|
+
/** One-time-per-process notice: the host's session log is unreadable. */
|
|
545
|
+
let warnedUnreadableLog = false;
|
|
525
546
|
const blockFor = (agent) => {
|
|
526
547
|
if (agent === void 0) return "";
|
|
527
548
|
const session = agent.session;
|
|
@@ -560,9 +581,20 @@ export function apply(ctx, config) {
|
|
|
560
581
|
// next assembly retry, so a new conversation never permanently misses its
|
|
561
582
|
// recalled memory.
|
|
562
583
|
if (resolved.autoRecallOnStart !== false) {
|
|
563
|
-
const query = firstUserMessageText(session);
|
|
584
|
+
const query = firstUserMessageText(session, sessionQuery);
|
|
564
585
|
if (query.length === 0) {
|
|
565
|
-
|
|
586
|
+
// Retry on the next assembly ONLY when the log is readable and the
|
|
587
|
+
// opening message simply has not been appended yet. An unreadable
|
|
588
|
+
// log (host API drift — see lib/session-events.js) will never gain
|
|
589
|
+
// one, and retrying forever would also keep the [ENGRAM] block
|
|
590
|
+
// uncached, churning the prefix every turn.
|
|
591
|
+
pendingRecall = sessionEventsSource(session, sessionQuery) !== "none";
|
|
592
|
+
if (!pendingRecall && !warnedUnreadableLog) {
|
|
593
|
+
warnedUnreadableLog = true;
|
|
594
|
+
log?.warn?.(
|
|
595
|
+
`engram: this host exposes no readable session event log (no session.events, no ctx.sessionQuery, no snapshotEvents, no eventAt) — [RECALL] / episode / todo-sink stay dormant. Reporting once per session. Session surface: ${describeSessionSurface(session).join(", ") || "(empty)"}`,
|
|
596
|
+
);
|
|
597
|
+
}
|
|
566
598
|
} else {
|
|
567
599
|
const recall = renderStartRecall(opened, workspace, resolved, { query, allowRecencyFallback: false });
|
|
568
600
|
if (recall.length > 0) {
|
|
@@ -654,6 +686,9 @@ export function apply(ctx, config) {
|
|
|
654
686
|
|
|
655
687
|
ctx.effect(() => {
|
|
656
688
|
const disposers = [];
|
|
689
|
+
// sessionId → { seqs, at }: the prunes this plugin caused, so the
|
|
690
|
+
// compaction/prune observer can tell engram's savings from the host's.
|
|
691
|
+
const ownPruneSeqs = new Map();
|
|
657
692
|
|
|
658
693
|
// Settings card edits (volatile Config fields) hot-commit into the refs
|
|
659
694
|
// in place; re-read them so `resolved` reflects new values for new
|
|
@@ -689,16 +724,10 @@ export function apply(ctx, config) {
|
|
|
689
724
|
return null;
|
|
690
725
|
}
|
|
691
726
|
},
|
|
692
|
-
//
|
|
693
|
-
//
|
|
694
|
-
//
|
|
695
|
-
|
|
696
|
-
try {
|
|
697
|
-
ledger?.noteCompaction(session?.id, workspace ?? wk(session?.header?.cwd), { charsSaved });
|
|
698
|
-
} catch (error) {
|
|
699
|
-
log?.warn?.(`engram ledger compaction credit failed: ${String(error)}`);
|
|
700
|
-
}
|
|
701
|
-
},
|
|
727
|
+
// Credit is committed-record-driven (see the `compaction/summary`
|
|
728
|
+
// listener below), not summarize()-driven: a checkpoint the harness
|
|
729
|
+
// rejects or overrides saves nothing, and crediting it would make the
|
|
730
|
+
// ledger claim savings that never landed.
|
|
702
731
|
});
|
|
703
732
|
disposers.push(() => {
|
|
704
733
|
// The engine's service registration is fiber-scoped and auto-removed on
|
|
@@ -721,6 +750,7 @@ export function apply(ctx, config) {
|
|
|
721
750
|
if (!hostSettled || !webSettled) return;
|
|
722
751
|
if (statusTimer !== void 0) { clearTimeout(statusTimer); statusTimer = void 0; }
|
|
723
752
|
void writeContextGcStatus({
|
|
753
|
+
pid: process.pid,
|
|
724
754
|
planes: resolved.autoWebCompaction === false ? ["host"] : ["host", "web"],
|
|
725
755
|
host: service.compactionStatus.host,
|
|
726
756
|
web: service.compactionStatus.web,
|
|
@@ -872,6 +902,45 @@ export function apply(ctx, config) {
|
|
|
872
902
|
ledger.dispose(session?.id);
|
|
873
903
|
});
|
|
874
904
|
disposers.push(disposeLedgerSession);
|
|
905
|
+
// Compaction credit, committed-record-driven: only a summary that carries
|
|
906
|
+
// the Context GC footer AND actually landed in the session counts. The
|
|
907
|
+
// engine's own measurement (`summarize`) is not evidence — the harness can
|
|
908
|
+
// reject or override it, and then nothing was saved.
|
|
909
|
+
const disposeLedgerCompaction = ctx.on("session/event", (session, event) => {
|
|
910
|
+
const credit = committedGcSaving(event);
|
|
911
|
+
if (credit === null) return;
|
|
912
|
+
ledger.noteCompaction(session?.id, wk(session?.header?.cwd), { charsSaved: credit.charsSaved });
|
|
913
|
+
});
|
|
914
|
+
disposers.push(disposeLedgerCompaction);
|
|
915
|
+
// Prune accounting: DSH prunes over-threshold tool results by itself
|
|
916
|
+
// (mid-step, no boundary involved). Those savings are real but they are
|
|
917
|
+
// NOT engram's, so they land in `hostPrunedChars` — a visible leg that
|
|
918
|
+
// never feeds NetTokens. Prunes we caused are already credited by the
|
|
919
|
+
// boundary hook (matched by seq here and skipped).
|
|
920
|
+
const disposeLedgerPrune = ctx.on("session/event", (session, event) => {
|
|
921
|
+
if (event?.type !== "compaction/prune") return;
|
|
922
|
+
const sid = session?.id;
|
|
923
|
+
const workspace = wk(session?.header?.cwd);
|
|
924
|
+
const tokens = Number(event?.data?.shadowedTokenCount);
|
|
925
|
+
if (!Number.isFinite(tokens) || tokens <= 0) return;
|
|
926
|
+
const shadowed = Array.isArray(event?.data?.shadowedSeqs) ? event.data.shadowedSeqs : [];
|
|
927
|
+
const own = typeof sid === "string" ? ownPruneSeqs.get(sid) : void 0;
|
|
928
|
+
let source = "host";
|
|
929
|
+
if (own !== void 0) {
|
|
930
|
+
const ours = shadowed.some((seq) => own.seqs.has(seq));
|
|
931
|
+
// A host prune that merely follows ours within the same step still
|
|
932
|
+
// counts as host work; only overlapping seqs mean "we did this".
|
|
933
|
+
if (ours) source = "engram";
|
|
934
|
+
if (Date.now() - own.at > 120_000) ownPruneSeqs.delete(sid);
|
|
935
|
+
}
|
|
936
|
+
if (source === "engram") return;
|
|
937
|
+
ledger.notePrune(sid, workspace, {
|
|
938
|
+
charsRemoved: tokens * CHARS_PER_TOKEN,
|
|
939
|
+
rewriteChars: estimatePrefixChars(session) + tokens * CHARS_PER_TOKEN,
|
|
940
|
+
source: "host",
|
|
941
|
+
});
|
|
942
|
+
});
|
|
943
|
+
disposers.push(disposeLedgerPrune);
|
|
875
944
|
}
|
|
876
945
|
|
|
877
946
|
// JIT observers: tool results (error recurrence / file focus) and goal
|
|
@@ -901,7 +970,7 @@ export function apply(ctx, config) {
|
|
|
901
970
|
const sessionId = session?.id;
|
|
902
971
|
if (typeof cwd !== "string" || cwd.length === 0) return;
|
|
903
972
|
if (typeof sessionId !== "string" || sessionId.length === 0) return;
|
|
904
|
-
const todos = pendingTodosFromSession(session);
|
|
973
|
+
const todos = pendingTodosFromSession(session, sessionQuery);
|
|
905
974
|
if (todos.length === 0) return;
|
|
906
975
|
const workspace = wk(cwd);
|
|
907
976
|
service
|
|
@@ -932,7 +1001,7 @@ export function apply(ctx, config) {
|
|
|
932
1001
|
const sessionId = session?.id;
|
|
933
1002
|
if (typeof cwd !== "string" || cwd.length === 0) return;
|
|
934
1003
|
if (typeof sessionId !== "string" || sessionId.length === 0) return;
|
|
935
|
-
const episode = buildEpisode(session);
|
|
1004
|
+
const episode = buildEpisode(session, { sessionQuery });
|
|
936
1005
|
if (episode === null) return;
|
|
937
1006
|
const workspace = wk(cwd);
|
|
938
1007
|
const maxEpisodes = Number.isFinite(resolved.episodeMaxPerWorkspace)
|
|
@@ -950,7 +1019,7 @@ export function apply(ctx, config) {
|
|
|
950
1019
|
signal: episode.signal,
|
|
951
1020
|
filePath: null,
|
|
952
1021
|
sessionId,
|
|
953
|
-
seq:
|
|
1022
|
+
seq: sessionSeq(session, sessionQuery),
|
|
954
1023
|
},
|
|
955
1024
|
resolved,
|
|
956
1025
|
);
|
|
@@ -1038,6 +1107,14 @@ export function apply(ctx, config) {
|
|
|
1038
1107
|
// surface is already rewritten when this hook runs).
|
|
1039
1108
|
const removed = Number.isFinite(report?.charsRemoved) ? report.charsRemoved : 0;
|
|
1040
1109
|
const prefixChars = estimatePrefixChars(session);
|
|
1110
|
+
if (typeof session?.id === "string") {
|
|
1111
|
+
const seqs = new Set();
|
|
1112
|
+
for (const item of report?.pruned ?? []) {
|
|
1113
|
+
const seq = item?.seq ?? item?.event?.seq ?? item?.payload?.seq;
|
|
1114
|
+
if (Number.isFinite(seq)) seqs.add(seq);
|
|
1115
|
+
}
|
|
1116
|
+
ownPruneSeqs.set(session.id, { seqs, at: Date.now() });
|
|
1117
|
+
}
|
|
1041
1118
|
ledger?.notePrune(session?.id, workspace, {
|
|
1042
1119
|
charsRemoved: removed,
|
|
1043
1120
|
rewriteChars: prefixChars + removed,
|
|
@@ -1063,12 +1140,22 @@ export function apply(ctx, config) {
|
|
|
1063
1140
|
ctx.inject(["toolResultPruner"], (prunerCtx) => {
|
|
1064
1141
|
const dispose = installBoundaryPrune(prunerCtx, resolved, log, boundaryHooks, {
|
|
1065
1142
|
allowFire: boundaryGate,
|
|
1143
|
+
// Durability for "armed but never fires": this plane sees the host's
|
|
1144
|
+
// own sessions; the agent-preset realm reports the same shape under
|
|
1145
|
+
// plane "preset" (lib/prune.js). Comparing the two tells us where a
|
|
1146
|
+
// missing boundary went.
|
|
1147
|
+
onAttempt: (attempt) => {
|
|
1148
|
+
void recordPruneAttempt({ plane: "host", ...attempt });
|
|
1149
|
+
},
|
|
1066
1150
|
});
|
|
1067
1151
|
// L1 provably can (or cannot) fire from here on: record it for the
|
|
1068
1152
|
// durable snapshot, then rewrite it — the first flush may have landed
|
|
1069
1153
|
// before the pruner activated, and this status line is what users
|
|
1070
1154
|
// read when "节省" stays at 0.
|
|
1071
1155
|
boundaryPruneState = boundaryPruneAvailability(prunerCtx, resolved);
|
|
1156
|
+
// Mirror it onto the live view so `/overview` can answer "is the only
|
|
1157
|
+
// ledger credit path actually able to fire?" without reading the file.
|
|
1158
|
+
service.compactionStatus.boundaryPrune = boundaryPruneState;
|
|
1072
1159
|
flushStatus();
|
|
1073
1160
|
return dispose;
|
|
1074
1161
|
});
|