dsh-engram 0.7.2 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +104 -7
- package/README.zh.md +13 -3
- package/docs/CONTEXT-GC-GUIDE.zh.md +15 -1
- package/docs/G6-quality-ab.zh.md +540 -0
- package/docs/GATES.zh.md +41 -0
- package/docs/PROPOSAL-evolution.zh.md +12 -0
- package/docs/RELEASE-NOTES-v0.7.10.md +49 -0
- package/docs/RELEASE-NOTES-v0.7.11.md +31 -0
- package/docs/RELEASE-NOTES-v0.7.12.md +22 -0
- package/docs/RELEASE-NOTES-v0.7.13.md +41 -0
- package/docs/RELEASE-NOTES-v0.7.14.md +37 -0
- package/docs/RELEASE-NOTES-v0.7.15.md +36 -0
- package/docs/RELEASE-NOTES-v0.7.16.md +42 -0
- package/docs/RELEASE-NOTES-v0.7.3.md +30 -0
- package/docs/RELEASE-NOTES-v0.7.4.md +71 -0
- package/docs/RELEASE-NOTES-v0.7.5.md +134 -0
- package/docs/RELEASE-NOTES-v0.7.6.md +74 -0
- package/docs/RELEASE-NOTES-v0.7.7.md +60 -0
- package/docs/RELEASE-NOTES-v0.7.8.md +56 -0
- package/docs/RELEASE-NOTES-v0.7.9.md +46 -0
- package/docs/RELEASE-NOTES-v0.8.0.md +59 -0
- package/docs/RELEASE-NOTES-v0.8.1.md +70 -0
- package/docs/RELEASE-NOTES-v0.8.2.md +62 -0
- package/docs/RELEASE-NOTES-v0.8.3.md +65 -0
- package/docs/RELEASE-NOTES-v0.8.4.md +56 -0
- package/docs/RELEASE-NOTES-v0.8.5.md +60 -0
- package/docs/RELEASE-NOTES-v0.8.6.md +71 -0
- package/eval/gate-audit.mjs +538 -0
- package/lib/api.js +70 -0
- package/lib/approvals.js +84 -0
- package/lib/boundary-prune.js +71 -5
- package/lib/capsule.js +134 -0
- package/lib/capture.js +85 -6
- package/lib/client.js +1813 -1745
- package/lib/compaction.js +4 -11
- package/lib/context-gc.js +63 -2
- package/lib/episode.js +10 -5
- package/lib/index-block.js +89 -8
- package/lib/index.js +108 -21
- package/lib/ledger.js +200 -10
- package/lib/pack-telemetry.js +10 -3
- package/lib/preference.js +2 -1
- package/lib/prune.js +89 -0
- package/lib/session-events.js +168 -0
- package/lib/skill-export.js +289 -0
- package/lib/status-insights.js +5 -0
- package/lib/status.js +138 -0
- package/lib/store.js +34 -0
- package/lib/todo-sink.js +3 -2
- package/lib/tools.js +2 -1
- package/lib/usage.js +32 -3
- package/lib/web-provision.js +59 -14
- package/package.json +15 -8
- package/scripts/dsh-engram.mjs +229 -4
package/lib/ledger.js
CHANGED
|
@@ -30,9 +30,15 @@ import path from "node:path";
|
|
|
30
30
|
|
|
31
31
|
const FILE_NAME = "ledger.json";
|
|
32
32
|
/** Matches the estimate used by the GUI (`chars / 4`). */
|
|
33
|
-
const CHARS_PER_TOKEN = 4;
|
|
33
|
+
export const CHARS_PER_TOKEN = 4;
|
|
34
34
|
const DEFAULT_CACHE_DISCOUNT = 0.1;
|
|
35
35
|
const DEFAULT_FLUSH_EVERY = 20;
|
|
36
|
+
/** Per-session rows kept on disk (newest activity wins; older ones drop). */
|
|
37
|
+
const DEFAULT_MAX_SESSION_ROWS = 200;
|
|
38
|
+
/** Rows returned by `summary()` — the GUI never needs the whole history. */
|
|
39
|
+
const SUMMARY_SESSION_ROWS = 50;
|
|
40
|
+
/** Session ids that would land on the prototype chain if used as row keys. */
|
|
41
|
+
const RESERVED_ROW_KEYS = new Set(["__proto__", "constructor", "prototype"]);
|
|
36
42
|
|
|
37
43
|
/** Default ledger file under the engram home. */
|
|
38
44
|
export function ledgerFile(home = path.join(os.homedir(), ".dsh", "engram")) {
|
|
@@ -84,6 +90,13 @@ function deltaOf(session) {
|
|
|
84
90
|
avoidedChars: session.avoidedChars - f.avoidedChars,
|
|
85
91
|
rewriteCount: session.rewriteCount - f.rewriteCount,
|
|
86
92
|
rewriteChars: session.rewriteChars - f.rewriteChars,
|
|
93
|
+
// These three used to be written as absolute values (`row.x = session.x`),
|
|
94
|
+
// which silently zeroed a session's history on every restart: the in-memory
|
|
95
|
+
// counters start at 0 and the first flush after boot overwrote the durable
|
|
96
|
+
// row. Deltas are the only thing a restart can't corrupt.
|
|
97
|
+
prunedChars: count(session.prunedChars) - count(f.prunedChars),
|
|
98
|
+
hostPrunedChars: count(session.hostPrunedChars) - count(f.hostPrunedChars),
|
|
99
|
+
compactedChars: count(session.compactedChars) - count(f.compactedChars),
|
|
87
100
|
};
|
|
88
101
|
}
|
|
89
102
|
|
|
@@ -93,7 +106,10 @@ function isEmptyDelta(delta) {
|
|
|
93
106
|
delta.injectedChars === 0 &&
|
|
94
107
|
delta.avoidedChars === 0 &&
|
|
95
108
|
delta.rewriteCount === 0 &&
|
|
96
|
-
delta.rewriteChars === 0
|
|
109
|
+
delta.rewriteChars === 0 &&
|
|
110
|
+
delta.prunedChars === 0 &&
|
|
111
|
+
delta.hostPrunedChars === 0 &&
|
|
112
|
+
delta.compactedChars === 0
|
|
97
113
|
);
|
|
98
114
|
}
|
|
99
115
|
|
|
@@ -119,6 +135,48 @@ export function finalizeLedgerBucket(bucket, cacheDiscount = DEFAULT_CACHE_DISCO
|
|
|
119
135
|
};
|
|
120
136
|
}
|
|
121
137
|
|
|
138
|
+
/** A durable per-session row: what ONE session spent and saved (raw counts). */
|
|
139
|
+
function emptySessionRow(sessionId, workspace) {
|
|
140
|
+
return {
|
|
141
|
+
sessionId,
|
|
142
|
+
workspace: typeof workspace === "string" ? workspace : "",
|
|
143
|
+
firstDay: "",
|
|
144
|
+
lastDay: "",
|
|
145
|
+
updatedAt: 0,
|
|
146
|
+
requests: 0,
|
|
147
|
+
injectedChars: 0,
|
|
148
|
+
avoidedChars: 0,
|
|
149
|
+
rewriteCount: 0,
|
|
150
|
+
rewriteChars: 0,
|
|
151
|
+
prunedChars: 0,
|
|
152
|
+
hostPrunedChars: 0,
|
|
153
|
+
compactedChars: 0,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Finalize a session row with the same net arithmetic as a bucket (pure).
|
|
159
|
+
*
|
|
160
|
+
* The gates in docs/PROPOSAL-evolution.zh.md §6 are per-session questions
|
|
161
|
+
* ("净收益 > 0 的会话占比", "长会话占比"), which a day-level rollup cannot
|
|
162
|
+
* answer — it aggregates them away. These rows exist so the answer is a
|
|
163
|
+
* read, not an estimate.
|
|
164
|
+
*/
|
|
165
|
+
export function finalizeSessionRow(row, cacheDiscount = DEFAULT_CACHE_DISCOUNT) {
|
|
166
|
+
const { sessions: _bucketSessions, ...base } = finalizeLedgerBucket(row, cacheDiscount);
|
|
167
|
+
return {
|
|
168
|
+
sessionId: String(row?.sessionId ?? ""),
|
|
169
|
+
workspace: String(row?.workspace ?? ""),
|
|
170
|
+
firstDay: String(row?.firstDay ?? ""),
|
|
171
|
+
lastDay: String(row?.lastDay ?? ""),
|
|
172
|
+
updatedAt: count(row?.updatedAt),
|
|
173
|
+
prunedChars: count(row?.prunedChars),
|
|
174
|
+
hostPrunedChars: count(row?.hostPrunedChars),
|
|
175
|
+
compactedChars: count(row?.compactedChars),
|
|
176
|
+
...base,
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
|
|
122
180
|
function readPersisted(file) {
|
|
123
181
|
try {
|
|
124
182
|
const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
@@ -188,6 +246,7 @@ export function estimatePrefixChars(session) {
|
|
|
188
246
|
* @param {string} [opts.home] engram home (defaults to ~/.dsh/engram)
|
|
189
247
|
* @param {number} [opts.cacheDiscount] cached-prefix price share (0..1)
|
|
190
248
|
* @param {number} [opts.flushEvery] requests between durable flushes
|
|
249
|
+
* @param {number} [opts.maxSessionRows] per-session rows kept on disk
|
|
191
250
|
* @param {boolean} [opts.persist] false = pure in-memory (tests, dry runs)
|
|
192
251
|
* @param {() => number} [opts.now] clock injection for tests
|
|
193
252
|
* @param {object} [opts.log] optional logger (warn only)
|
|
@@ -196,6 +255,7 @@ export function createLedger({
|
|
|
196
255
|
home = path.join(os.homedir(), ".dsh", "engram"),
|
|
197
256
|
cacheDiscount = DEFAULT_CACHE_DISCOUNT,
|
|
198
257
|
flushEvery = DEFAULT_FLUSH_EVERY,
|
|
258
|
+
maxSessionRows = DEFAULT_MAX_SESSION_ROWS,
|
|
199
259
|
persist = true,
|
|
200
260
|
now = Date.now,
|
|
201
261
|
log = void 0,
|
|
@@ -204,13 +264,17 @@ export function createLedger({
|
|
|
204
264
|
const discount = clampDiscount(cacheDiscount);
|
|
205
265
|
const every = Number.isFinite(flushEvery) && flushEvery > 0 ? Math.trunc(flushEvery) : 0;
|
|
206
266
|
const stored = persist ? readPersisted(file) : null;
|
|
207
|
-
const state = stored ?? { version: 1, updatedAt: 0, workspaces: {} };
|
|
267
|
+
const state = stored ?? { version: 1, updatedAt: 0, workspaces: {}, sessions: {} };
|
|
268
|
+
if (state.sessions === null || typeof state.sessions !== "object") state.sessions = {};
|
|
269
|
+
const rowCap = Number.isFinite(maxSessionRows) && maxSessionRows > 0 ? Math.trunc(maxSessionRows) : DEFAULT_MAX_SESSION_ROWS;
|
|
208
270
|
const sessions = new Map();
|
|
209
271
|
|
|
210
272
|
function ensure(sessionId, workspace) {
|
|
211
273
|
if (typeof sessionId !== "string" || sessionId.length === 0) return null;
|
|
212
274
|
let session = sessions.get(sessionId);
|
|
213
275
|
if (session === void 0) {
|
|
276
|
+
const persisted = state.sessions?.[sessionId];
|
|
277
|
+
const seed = persisted !== null && typeof persisted === "object" ? persisted : null;
|
|
214
278
|
session = {
|
|
215
279
|
sessionId,
|
|
216
280
|
workspace: typeof workspace === "string" ? workspace : "",
|
|
@@ -220,11 +284,22 @@ export function createLedger({
|
|
|
220
284
|
requests: 0,
|
|
221
285
|
injectedChars: 0,
|
|
222
286
|
avoidedChars: 0,
|
|
223
|
-
prunedChars:
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
287
|
+
prunedChars: count(seed?.prunedChars),
|
|
288
|
+
hostPrunedChars: count(seed?.hostPrunedChars),
|
|
289
|
+
compactedChars: count(seed?.compactedChars),
|
|
290
|
+
rewriteCount: count(seed?.rewriteCount),
|
|
291
|
+
rewriteChars: count(seed?.rewriteChars),
|
|
292
|
+
updatedAt: 0,
|
|
293
|
+
flushed: {
|
|
294
|
+
requests: 0,
|
|
295
|
+
injectedChars: 0,
|
|
296
|
+
avoidedChars: 0,
|
|
297
|
+
rewriteCount: count(seed?.rewriteCount),
|
|
298
|
+
rewriteChars: count(seed?.rewriteChars),
|
|
299
|
+
prunedChars: count(seed?.prunedChars),
|
|
300
|
+
hostPrunedChars: count(seed?.hostPrunedChars),
|
|
301
|
+
compactedChars: count(seed?.compactedChars),
|
|
302
|
+
},
|
|
228
303
|
firstFlush: false,
|
|
229
304
|
};
|
|
230
305
|
sessions.set(sessionId, session);
|
|
@@ -249,6 +324,34 @@ export function createLedger({
|
|
|
249
324
|
return days[day];
|
|
250
325
|
}
|
|
251
326
|
|
|
327
|
+
/** The durable row for one session, created on first flush. */
|
|
328
|
+
function rowFor(session, day) {
|
|
329
|
+
// session ids are host-generated, but a row key must never hit the
|
|
330
|
+
// prototype chain.
|
|
331
|
+
if (RESERVED_ROW_KEYS.has(session.sessionId)) return null;
|
|
332
|
+
let row = state.sessions[session.sessionId];
|
|
333
|
+
if (row === null || row === void 0 || typeof row !== "object") {
|
|
334
|
+
row = emptySessionRow(session.sessionId, session.workspace);
|
|
335
|
+
state.sessions[session.sessionId] = row;
|
|
336
|
+
}
|
|
337
|
+
if (row.firstDay === void 0 || row.firstDay.length === 0) row.firstDay = day;
|
|
338
|
+
row.lastDay = day;
|
|
339
|
+
row.updatedAt = Math.max(count(row.updatedAt), count(session.updatedAt), now());
|
|
340
|
+
if (typeof session.workspace === "string" && session.workspace.length > 0) row.workspace = session.workspace;
|
|
341
|
+
return row;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/** Keep the newest `rowCap` rows; a session that never ran is not history. */
|
|
345
|
+
function pruneRows() { const entries = Object.entries(state.sessions);
|
|
346
|
+
if (entries.length <= rowCap) return;
|
|
347
|
+
entries
|
|
348
|
+
.sort((a, b) => count(b[1]?.updatedAt) - count(a[1]?.updatedAt))
|
|
349
|
+
.slice(rowCap)
|
|
350
|
+
.forEach(([id]) => {
|
|
351
|
+
delete state.sessions[id];
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
|
|
252
355
|
function flushSession(session) {
|
|
253
356
|
if (session === null || session === void 0) return false;
|
|
254
357
|
const delta = deltaOf(session);
|
|
@@ -261,12 +364,29 @@ export function createLedger({
|
|
|
261
364
|
session.firstFlush = true;
|
|
262
365
|
}
|
|
263
366
|
addBucket(bucket, delta);
|
|
367
|
+
// Per-session row (same delta, same instant) — the gates need the
|
|
368
|
+
// distribution, not the sum.
|
|
369
|
+
const row = rowFor(session, day);
|
|
370
|
+
if (row !== null) {
|
|
371
|
+
row.requests += count(delta.requests);
|
|
372
|
+
row.injectedChars += count(delta.injectedChars);
|
|
373
|
+
row.avoidedChars += count(delta.avoidedChars);
|
|
374
|
+
row.rewriteCount += count(delta.rewriteCount);
|
|
375
|
+
row.rewriteChars += count(delta.rewriteChars);
|
|
376
|
+
row.prunedChars += count(delta.prunedChars);
|
|
377
|
+
row.hostPrunedChars += count(delta.hostPrunedChars);
|
|
378
|
+
row.compactedChars += count(delta.compactedChars);
|
|
379
|
+
pruneRows();
|
|
380
|
+
}
|
|
264
381
|
session.flushed = {
|
|
265
382
|
requests: session.requests,
|
|
266
383
|
injectedChars: session.injectedChars,
|
|
267
384
|
avoidedChars: session.avoidedChars,
|
|
268
385
|
rewriteCount: session.rewriteCount,
|
|
269
386
|
rewriteChars: session.rewriteChars,
|
|
387
|
+
prunedChars: count(session.prunedChars),
|
|
388
|
+
hostPrunedChars: count(session.hostPrunedChars),
|
|
389
|
+
compactedChars: count(session.compactedChars),
|
|
270
390
|
};
|
|
271
391
|
state.updatedAt = now();
|
|
272
392
|
if (persist) writeJsonAtomic(file, state);
|
|
@@ -281,6 +401,40 @@ export function createLedger({
|
|
|
281
401
|
for (const session of sessions.values()) flushSession(session);
|
|
282
402
|
}
|
|
283
403
|
|
|
404
|
+
/**
|
|
405
|
+
* Per-session rows for reading (persisted rows + the open sessions'
|
|
406
|
+
* unflushed tails merged in). Non-mutating, newest activity first.
|
|
407
|
+
*/
|
|
408
|
+
function sessionRows(limit = SUMMARY_SESSION_ROWS) {
|
|
409
|
+
const rows = new Map();
|
|
410
|
+
for (const [id, row] of Object.entries(state.sessions ?? {})) {
|
|
411
|
+
if (row === null || typeof row !== "object") continue;
|
|
412
|
+
rows.set(id, { ...emptySessionRow(id, row.workspace), ...row });
|
|
413
|
+
}
|
|
414
|
+
const today = ledgerDayKey(now());
|
|
415
|
+
for (const session of sessions.values()) {
|
|
416
|
+
const delta = deltaOf(session);
|
|
417
|
+
const row = rows.get(session.sessionId) ?? emptySessionRow(session.sessionId, session.workspace);
|
|
418
|
+
row.requests += count(delta.requests);
|
|
419
|
+
row.injectedChars += count(delta.injectedChars);
|
|
420
|
+
row.avoidedChars += count(delta.avoidedChars);
|
|
421
|
+
row.rewriteCount += count(delta.rewriteCount);
|
|
422
|
+
row.rewriteChars += count(delta.rewriteChars);
|
|
423
|
+
row.prunedChars = count(session.prunedChars);
|
|
424
|
+
row.hostPrunedChars = count(session.hostPrunedChars);
|
|
425
|
+
row.compactedChars = count(session.compactedChars);
|
|
426
|
+
if (session.workspace.length > 0) row.workspace = session.workspace;
|
|
427
|
+
if (typeof row.firstDay !== "string" || row.firstDay.length === 0) row.firstDay = today;
|
|
428
|
+
row.lastDay = today;
|
|
429
|
+
if (session.updatedAt > 0) row.updatedAt = Math.max(count(row.updatedAt), count(session.updatedAt));
|
|
430
|
+
rows.set(session.sessionId, row);
|
|
431
|
+
}
|
|
432
|
+
return [...rows.values()]
|
|
433
|
+
.map((row) => finalizeSessionRow(row, discount))
|
|
434
|
+
.sort((a, b) => b.updatedAt - a.updatedAt)
|
|
435
|
+
.slice(0, limit);
|
|
436
|
+
}
|
|
437
|
+
|
|
284
438
|
return {
|
|
285
439
|
/** Record the latest injected block size for one component. */
|
|
286
440
|
noteInjection(sessionId, workspace, component, chars) {
|
|
@@ -288,6 +442,7 @@ export function createLedger({
|
|
|
288
442
|
const session = ensure(sessionId, workspace);
|
|
289
443
|
if (session === null) return;
|
|
290
444
|
const size = count(chars);
|
|
445
|
+
session.updatedAt = now();
|
|
291
446
|
if (component === "esr") session.esrChars = size;
|
|
292
447
|
else if (component === "jit") session.jitChars = size;
|
|
293
448
|
else session.indexChars = size;
|
|
@@ -302,6 +457,7 @@ export function createLedger({
|
|
|
302
457
|
const session = ensure(sessionId, workspace);
|
|
303
458
|
if (session === null) return;
|
|
304
459
|
session.requests += 1;
|
|
460
|
+
session.updatedAt = now();
|
|
305
461
|
session.injectedChars += session.indexChars + session.esrChars + session.jitChars;
|
|
306
462
|
session.avoidedChars += session.prunedChars + session.compactedChars;
|
|
307
463
|
if (every > 0 && session.requests % every === 0) flushSession(session);
|
|
@@ -310,11 +466,29 @@ export function createLedger({
|
|
|
310
466
|
}
|
|
311
467
|
},
|
|
312
468
|
|
|
313
|
-
/**
|
|
314
|
-
|
|
469
|
+
/**
|
|
470
|
+
* A prune happened: its removed chars stop replaying from the next request.
|
|
471
|
+
*
|
|
472
|
+
* `source` keeps the attribution honest — only prunes engram CAUSED count
|
|
473
|
+
* toward `prunedChars` (and therefore toward avoidedChars/NetTokens). The
|
|
474
|
+
* host prunes over-threshold results on its own (observed live: 12,491
|
|
475
|
+
* tokens shadowed with no boundary in sight); those land in
|
|
476
|
+
* `hostPrunedChars`, visible but never claimed as engram's saving.
|
|
477
|
+
*/
|
|
478
|
+
notePrune(sessionId, workspace, { charsRemoved = 0, rewriteChars = 0, source = "boundary" } = {}) {
|
|
315
479
|
try {
|
|
316
480
|
const session = ensure(sessionId, workspace);
|
|
317
481
|
if (session === null) return;
|
|
482
|
+
session.updatedAt = now();
|
|
483
|
+
if (source === "host") {
|
|
484
|
+
// Symmetry: we do not claim the host's savings, so we do not pay its
|
|
485
|
+
// rewrite cost either. Charging both ways made ordinary sessions look
|
|
486
|
+
// catastrophic (1.22M chars of rewrite for prunes engram never asked
|
|
487
|
+
// for) — the exact bias that turns a neutral session into -1.7M.
|
|
488
|
+
session.hostPrunedChars += count(charsRemoved);
|
|
489
|
+
flushSession(session);
|
|
490
|
+
return;
|
|
491
|
+
}
|
|
318
492
|
session.prunedChars += count(charsRemoved);
|
|
319
493
|
session.rewriteCount += 1;
|
|
320
494
|
session.rewriteChars += count(rewriteChars);
|
|
@@ -329,6 +503,7 @@ export function createLedger({
|
|
|
329
503
|
try {
|
|
330
504
|
const session = ensure(sessionId, workspace);
|
|
331
505
|
if (session === null) return;
|
|
506
|
+
session.updatedAt = now();
|
|
332
507
|
session.compactedChars += count(charsSaved);
|
|
333
508
|
} catch (error) {
|
|
334
509
|
log?.warn?.(`engram ledger noteCompaction failed: ${String(error)}`);
|
|
@@ -343,6 +518,7 @@ export function createLedger({
|
|
|
343
518
|
try {
|
|
344
519
|
const session = ensure(sessionId, workspace);
|
|
345
520
|
if (session === null) return;
|
|
521
|
+
session.updatedAt = now();
|
|
346
522
|
session.rewriteCount += 1;
|
|
347
523
|
session.rewriteChars += count(chars);
|
|
348
524
|
flushSession(session);
|
|
@@ -402,6 +578,11 @@ export function createLedger({
|
|
|
402
578
|
const totals = emptyBucket();
|
|
403
579
|
for (const bucket of Object.values(byWorkspace)) addBucket(totals, bucket);
|
|
404
580
|
|
|
581
|
+
const rowsHostPruned = sessionRows(SUMMARY_SESSION_ROWS).reduce(
|
|
582
|
+
(sum, row) => sum + count(row.hostPrunedChars),
|
|
583
|
+
0,
|
|
584
|
+
);
|
|
585
|
+
|
|
405
586
|
const finalizedWorkspaces = {};
|
|
406
587
|
for (const [ws, bucket] of Object.entries(byWorkspace)) {
|
|
407
588
|
finalizedWorkspaces[ws] = finalizeLedgerBucket(bucket, discount);
|
|
@@ -417,8 +598,11 @@ export function createLedger({
|
|
|
417
598
|
cacheDiscount: discount,
|
|
418
599
|
openSessions: sessions.size,
|
|
419
600
|
totals: finalizeLedgerBucket(totals, discount),
|
|
601
|
+
// Savings the HOST performed on its own (visible, never claimed).
|
|
602
|
+
hostPrunedChars: rowsHostPruned,
|
|
420
603
|
byWorkspace: finalizedWorkspaces,
|
|
421
604
|
byDay: finalizedDays,
|
|
605
|
+
sessions: sessionRows(SUMMARY_SESSION_ROWS),
|
|
422
606
|
};
|
|
423
607
|
},
|
|
424
608
|
};
|
|
@@ -442,6 +626,10 @@ export function readLedger({ home, cacheDiscount = DEFAULT_CACHE_DISCOUNT } = {}
|
|
|
442
626
|
}
|
|
443
627
|
const totals = emptyBucket();
|
|
444
628
|
for (const bucket of Object.values(workspaces)) addBucket(totals, bucket);
|
|
629
|
+
const rows = Object.values(stored?.sessions ?? {})
|
|
630
|
+
.filter((row) => row !== null && typeof row === "object")
|
|
631
|
+
.map((row) => finalizeSessionRow(row, cacheDiscount))
|
|
632
|
+
.sort((a, b) => b.updatedAt - a.updatedAt);
|
|
445
633
|
return {
|
|
446
634
|
version: 1,
|
|
447
635
|
updatedAt: stored?.updatedAt ?? 0,
|
|
@@ -454,5 +642,7 @@ export function readLedger({ home, cacheDiscount = DEFAULT_CACHE_DISCOUNT } = {}
|
|
|
454
642
|
byDay: Object.entries(days)
|
|
455
643
|
.map(([day, bucket]) => ({ day, ...finalizeLedgerBucket(bucket, cacheDiscount) }))
|
|
456
644
|
.sort((a, b) => (a.day < b.day ? 1 : a.day > b.day ? -1 : 0)),
|
|
645
|
+
sessions: rows,
|
|
646
|
+
hostPrunedChars: rows.reduce((sum, row) => sum + count(row.hostPrunedChars), 0),
|
|
457
647
|
};
|
|
458
648
|
}
|
package/lib/pack-telemetry.js
CHANGED
|
@@ -21,9 +21,16 @@ import fs from "node:fs";
|
|
|
21
21
|
import os from "node:os";
|
|
22
22
|
import path from "node:path";
|
|
23
23
|
|
|
24
|
-
/** Default engram home (
|
|
25
|
-
|
|
26
|
-
|
|
24
|
+
/** Default engram home: `$DSH_HOME/engram` when set (tests, sandboxes, custom
|
|
25
|
+
* installs), else `~/.dsh/engram` (mirrors status.js / result-pack.js). A
|
|
26
|
+
* plain `~/.dsh` default made every test run write live telemetry into the
|
|
27
|
+
* developer's real install. */
|
|
28
|
+
export function engramHome(home = void 0) {
|
|
29
|
+
if (typeof home === "string" && home.length > 0) return home;
|
|
30
|
+
const dshHome = process.env.DSH_HOME;
|
|
31
|
+
return dshHome !== void 0 && dshHome.length > 0
|
|
32
|
+
? path.join(dshHome, "engram")
|
|
33
|
+
: path.join(os.homedir(), ".dsh", "engram");
|
|
27
34
|
}
|
|
28
35
|
|
|
29
36
|
function statusFile(home, name) {
|
package/lib/preference.js
CHANGED
|
@@ -28,6 +28,7 @@
|
|
|
28
28
|
*/
|
|
29
29
|
|
|
30
30
|
import { workspaceKey as wk } from "./util.js";
|
|
31
|
+
import { sessionSeq } from "./session-events.js";
|
|
31
32
|
|
|
32
33
|
/** Direction of the preference statement, used in the captured text. */
|
|
33
34
|
export const PREFERENCE_DIRECTION = {
|
|
@@ -244,7 +245,7 @@ export function installPreferenceCapture(ctx, storeAdapter, config, log, stats)
|
|
|
244
245
|
signal: PREFERENCE_SIGNAL,
|
|
245
246
|
filePath: null,
|
|
246
247
|
sessionId,
|
|
247
|
-
seq:
|
|
248
|
+
seq: sessionSeq(session),
|
|
248
249
|
},
|
|
249
250
|
config,
|
|
250
251
|
)
|
package/lib/prune.js
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-engram — BoundaryPrune (L1) as a mountable entry INSIDE the agent preset.
|
|
3
|
+
*
|
|
4
|
+
* Why this entry exists: web sessions live in the agent preset's own realm, and
|
|
5
|
+
* their `session/event`s are only observed by plugins mounted in that realm.
|
|
6
|
+
* The host-plane install armed fine (`boundaryPrune.armed: true`) and still never
|
|
7
|
+
* fired across every session on the profile: the listener was on the wrong side
|
|
8
|
+
* of the realm boundary, so a real `todo/write` boundary produced nothing — the
|
|
9
|
+
* ledger's prune credit leg was structurally dead for `dsh web`.
|
|
10
|
+
*
|
|
11
|
+
* This row sits in the same `compaction` group as `dsh-engram/compaction`
|
|
12
|
+
* (which already isolates `toolResultPruner` for the agent plane), so it sees
|
|
13
|
+
* the sessions' events and holds that realm's pruner:
|
|
14
|
+
*
|
|
15
|
+
* ```yaml
|
|
16
|
+
* - id: compaction
|
|
17
|
+
* name: cordis:group
|
|
18
|
+
* group: true
|
|
19
|
+
* isolate: { compaction: true, toolResultPruner: true }
|
|
20
|
+
* config:
|
|
21
|
+
* - id: engram-compaction
|
|
22
|
+
* name: dsh-engram/compaction
|
|
23
|
+
* - id: engram-prune
|
|
24
|
+
* name: dsh-engram/prune # this entry
|
|
25
|
+
* - id: command-compact
|
|
26
|
+
* name: '@deepseek-ai/dsh-command-compact'
|
|
27
|
+
* - id: tool-result-pruner
|
|
28
|
+
* name: '@deepseek-ai/dsh-compaction-tool-result-pruner'
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* The host plugin still installs its own L1 for host-plane (headless/TUI)
|
|
32
|
+
* sessions. Running both is safe: `toolResultPruner.pruneSession` only prunes
|
|
33
|
+
* over-threshold results, so a second pass over an already-pruned surface finds
|
|
34
|
+
* nothing and credits nothing.
|
|
35
|
+
*
|
|
36
|
+
* Config: the same keys the host entries read (`boundaryPrune`,
|
|
37
|
+
* `boundaryPruneMinRemaining`, `boundaryPruneThresholdChars`,
|
|
38
|
+
* `boundaryPruneHeadChars`, `boundaryPruneTailChars`,
|
|
39
|
+
* `boundaryPruneIdleCadence`, `boundaryPruneAdaptive`). Ledger credit goes
|
|
40
|
+
* through the host-provided `engramLedger` service so there is exactly one
|
|
41
|
+
* ledger writer.
|
|
42
|
+
*/
|
|
43
|
+
import { installBoundaryPrune, boundaryPruneAvailability } from "./boundary-prune.js";
|
|
44
|
+
import { estimatePrefixChars } from "./ledger.js";
|
|
45
|
+
import { recordBoundaryPruneFire, engramHome } from "./pack-telemetry.js";
|
|
46
|
+
import { recordPruneAttempt } from "./status.js";
|
|
47
|
+
import { workspaceKey as wk } from "./util.js";
|
|
48
|
+
|
|
49
|
+
export const name = "dsh-engram-prune";
|
|
50
|
+
export const inject = ["toolResultPruner"];
|
|
51
|
+
|
|
52
|
+
/** The plane tag every attempt from this entry carries. */
|
|
53
|
+
export const PLANE = "preset";
|
|
54
|
+
|
|
55
|
+
export function apply(ctx, config = {}) {
|
|
56
|
+
const log = typeof ctx?.logger === "function" ? ctx.logger("dsh-engram-prune") : undefined;
|
|
57
|
+
|
|
58
|
+
// Diagnostics first: proves the row loaded at all (a row that fails to load
|
|
59
|
+
// looks exactly like one that works, until nothing ever fires).
|
|
60
|
+
try {
|
|
61
|
+
ctx.logger?.info?.(`engram boundary-prune (preset plane): ${boundaryPruneAvailability(ctx, config).reason ?? "armed"}`);
|
|
62
|
+
} catch { /* never break the mount for a log line */ }
|
|
63
|
+
|
|
64
|
+
const hooks = {
|
|
65
|
+
onPruned: (session, report, boundary) => {
|
|
66
|
+
const workspace = wk(session?.header?.cwd);
|
|
67
|
+
// Durable activation telemetry (shared with the host plane; the summary
|
|
68
|
+
// reports `activation.boundaryPrune` from it).
|
|
69
|
+
recordBoundaryPruneFire({ home: engramHome(), boundary, report, workspace });
|
|
70
|
+
// The prune rewrote the prefix: credit the removed chars, price the
|
|
71
|
+
// invalidation with the pre-prune size (mirrors the host-plane path).
|
|
72
|
+
const removed = Number.isFinite(report?.charsRemoved) ? report.charsRemoved : 0;
|
|
73
|
+
const prefixChars = estimatePrefixChars(session);
|
|
74
|
+
try {
|
|
75
|
+
const ledger = ctx.get?.("engramLedger");
|
|
76
|
+
ledger?.notePrune?.(session?.id, workspace, {
|
|
77
|
+
charsRemoved: removed,
|
|
78
|
+
rewriteChars: prefixChars + removed,
|
|
79
|
+
});
|
|
80
|
+
} catch { /* accounting must never break pruning */ }
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
return installBoundaryPrune(ctx, config, log, hooks, {
|
|
85
|
+
onAttempt: (attempt) => {
|
|
86
|
+
void recordPruneAttempt({ plane: PLANE, ...attempt });
|
|
87
|
+
},
|
|
88
|
+
});
|
|
89
|
+
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Session event-log access across DSH generations.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS (2026-10-20, the second "silent zero"):
|
|
5
|
+
* DSH ≤0.1.x exposed the conversation log as `session.events`, a plain frozen
|
|
6
|
+
* array. DSH 0.2.x removed that getter — `Session.prototype` now carries only
|
|
7
|
+
* `eventAt` / `snapshotEvents` / `ownEvents` / `deriveEventMessage`, and the
|
|
8
|
+
* synchronous full-log read is deprecated host-internally in favour of
|
|
9
|
+
* `ctx.sessionQuery.live(session)` leases. Every
|
|
10
|
+
* `Array.isArray(session.events)` read in this plugin therefore yielded `[]`
|
|
11
|
+
* on the running host, silently disabling three whole features while their
|
|
12
|
+
* unit tests (which mock `{ events: [...] }`) stayed green:
|
|
13
|
+
*
|
|
14
|
+
* - `[RECALL]` session-start auto-recall never materialized (no first user
|
|
15
|
+
* message → no query), and worse, `index.js` refused to cache the
|
|
16
|
+
* `[ENGRAM]` block ("pendingRecall") on every assembly, so the index was
|
|
17
|
+
* re-rendered per turn and never metered into the ledger;
|
|
18
|
+
* - session episodes were never written → `[RESUME]` briefs never fired;
|
|
19
|
+
* - the session-end todo sink never found a pending todo → no ESR drafts.
|
|
20
|
+
*
|
|
21
|
+
* The lesson is the same one as the session-log rename (memory #8): a
|
|
22
|
+
* host-API read must feature-detect and REPORT, never silently degrade to an
|
|
23
|
+
* empty value. `sessionEventsSource()` names the path taken so the status
|
|
24
|
+
* snapshot can surface `none` instead of pretending the log was empty.
|
|
25
|
+
*
|
|
26
|
+
* Resolution order (first working wins):
|
|
27
|
+
* 1. `session.events` — legacy hosts (≤0.1.x), and every unit test;
|
|
28
|
+
* 2. `sessionQuery.live(session)`— sanctioned ≥0.2.x lease (lazy snapshot);
|
|
29
|
+
* 3. `session.snapshotEvents()` — context-free fallback (deprecated host-side,
|
|
30
|
+
* still the only sync full-log read a plugin has);
|
|
31
|
+
* 4. `[]` — nothing readable; callers degrade and
|
|
32
|
+
* `sessionEventsSource` reports "none".
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
/** Shared empty array: callers only read, so one frozen instance is enough. */
|
|
36
|
+
const NO_EVENTS = Object.freeze([]);
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Which read path a session exposes, without materializing the log.
|
|
40
|
+
* @param {object} session
|
|
41
|
+
* @param {object} [sessionQuery] the `ctx.sessionQuery` service, if mounted
|
|
42
|
+
* @returns {"legacy"|"sessionQuery"|"snapshotEvents"|"eventAt"|"none"}
|
|
43
|
+
*/
|
|
44
|
+
export function sessionEventsSource(session, sessionQuery = void 0) {
|
|
45
|
+
if (session === null || typeof session !== "object") return "none";
|
|
46
|
+
if (Array.isArray(session.events)) return "legacy";
|
|
47
|
+
if (hasLive(sessionQuery)) return "sessionQuery";
|
|
48
|
+
if (typeof session.snapshotEvents === "function") return "snapshotEvents";
|
|
49
|
+
// Last-resort hand: the per-seq accessor is the primitive every other read
|
|
50
|
+
// is built on, so a host can drop the bulk readers and still answer `eventAt`.
|
|
51
|
+
// The Windows desktop runtime (issue #9) is exactly that host.
|
|
52
|
+
if (typeof session.eventAt === "function") return "eventAt";
|
|
53
|
+
return "none";
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* What the session object actually offers, for the dormancy warning. A bare
|
|
58
|
+
* "nothing is readable" tells the reporter nothing; the method names tell us
|
|
59
|
+
* which generation of the host we are talking to.
|
|
60
|
+
* @param {object} session
|
|
61
|
+
* @returns {string[]} sorted prototype method names (own properties included)
|
|
62
|
+
*/
|
|
63
|
+
export function describeSessionSurface(session) {
|
|
64
|
+
if (session === null || typeof session !== "object") return [];
|
|
65
|
+
const names = new Set();
|
|
66
|
+
for (let proto = session; proto !== null && proto !== Object.prototype; proto = Object.getPrototypeOf(proto)) {
|
|
67
|
+
for (const name of Object.getOwnPropertyNames(proto)) {
|
|
68
|
+
if (name === "constructor" || name.startsWith("_")) continue;
|
|
69
|
+
names.add(name);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return [...names].sort();
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function hasLive(sessionQuery) {
|
|
76
|
+
return (
|
|
77
|
+
sessionQuery !== null &&
|
|
78
|
+
typeof sessionQuery === "object" &&
|
|
79
|
+
typeof sessionQuery.live === "function"
|
|
80
|
+
);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Read a session's event log across DSH generations (see module header).
|
|
85
|
+
*
|
|
86
|
+
* Returns a frozen-or-plain array of events; never throws. A session that
|
|
87
|
+
* genuinely has no events and a session whose log is unreadable both return
|
|
88
|
+
* `[]` — use {@link sessionEventsSource} when the difference matters.
|
|
89
|
+
*
|
|
90
|
+
* @param {object} session host session (`agent.session`, or the arg of
|
|
91
|
+
* `session/disposed`)
|
|
92
|
+
* @param {object} [sessionQuery] the `ctx.sessionQuery` service, if mounted
|
|
93
|
+
* @returns {ReadonlyArray<object>}
|
|
94
|
+
*/
|
|
95
|
+
export function readSessionEvents(session, sessionQuery = void 0) {
|
|
96
|
+
if (session === null || typeof session !== "object") return NO_EVENTS;
|
|
97
|
+
|
|
98
|
+
// 1. Legacy: DSH ≤0.1.x (and every unit-test double).
|
|
99
|
+
if (Array.isArray(session.events)) return session.events;
|
|
100
|
+
|
|
101
|
+
// 2. Sanctioned ≥0.2.x: a live lease exposes a lazily materialized snapshot
|
|
102
|
+
// up to the session's current seq. Dispose it immediately — we only need
|
|
103
|
+
// the array, not the retention or the projection payload.
|
|
104
|
+
if (hasLive(sessionQuery)) {
|
|
105
|
+
let lease;
|
|
106
|
+
try {
|
|
107
|
+
lease = sessionQuery.live(session);
|
|
108
|
+
const events = lease?.events;
|
|
109
|
+
if (Array.isArray(events)) return events;
|
|
110
|
+
} catch {
|
|
111
|
+
// fall through to snapshotEvents
|
|
112
|
+
} finally {
|
|
113
|
+
try {
|
|
114
|
+
lease?.[Symbol.dispose]?.();
|
|
115
|
+
} catch {
|
|
116
|
+
// best-effort lease release
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// 3. Context-free fallback.
|
|
122
|
+
if (typeof session.snapshotEvents === "function") {
|
|
123
|
+
try {
|
|
124
|
+
const end = typeof session.seq === "number" ? session.seq : void 0;
|
|
125
|
+
const events = end === void 0 ? session.snapshotEvents() : session.snapshotEvents(0, end);
|
|
126
|
+
if (Array.isArray(events)) return events;
|
|
127
|
+
} catch {
|
|
128
|
+
// unreadable — degrade
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// 4. Per-seq accessor: read the log one event at a time. Slower than the bulk
|
|
133
|
+
// readers and only used when they are gone, but it keeps [RECALL] /
|
|
134
|
+
// episode / todo-sink alive on hosts that dropped `snapshotEvents`.
|
|
135
|
+
if (typeof session.eventAt === "function") {
|
|
136
|
+
const total = typeof session.seq === "number" && session.seq >= 0 ? Math.trunc(session.seq) : Number.POSITIVE_INFINITY;
|
|
137
|
+
const events = [];
|
|
138
|
+
for (let seq = 0; seq < total; seq += 1) {
|
|
139
|
+
let event;
|
|
140
|
+
try {
|
|
141
|
+
event = session.eventAt(seq);
|
|
142
|
+
} catch {
|
|
143
|
+
return events.length > 0 ? events : NO_EVENTS;
|
|
144
|
+
}
|
|
145
|
+
if (event === undefined || event === null) break;
|
|
146
|
+
events.push(event);
|
|
147
|
+
}
|
|
148
|
+
return events;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// 5. Nothing.
|
|
152
|
+
return NO_EVENTS;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* The session's current event count, for provenance `seq` stamps.
|
|
157
|
+
*
|
|
158
|
+
* `seq` is the next free seq on ≥0.2.x (`session.seq`), so it equals the log
|
|
159
|
+
* length; on legacy hosts the array length is the same number. Kept here so
|
|
160
|
+
* the four write sites (capture / tools / preference / episode) agree.
|
|
161
|
+
*/
|
|
162
|
+
export function sessionSeq(session, sessionQuery = void 0) {
|
|
163
|
+
if (session === null || typeof session !== "object") return 0;
|
|
164
|
+
const seq = session.seq;
|
|
165
|
+
if (typeof seq === "number" && Number.isFinite(seq) && seq >= 0) return Math.trunc(seq);
|
|
166
|
+
const events = readSessionEvents(session, sessionQuery);
|
|
167
|
+
return events.length;
|
|
168
|
+
}
|