dsh-engram 0.7.4 → 0.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +100 -6
- package/README.zh.md +7 -2
- package/docs/CONTEXT-GC-GUIDE.zh.md +15 -1
- package/docs/G6-quality-ab.zh.md +540 -0
- package/docs/GATES.zh.md +41 -0
- package/docs/PROPOSAL-evolution.zh.md +12 -0
- package/docs/RELEASE-NOTES-v0.7.10.md +49 -0
- package/docs/RELEASE-NOTES-v0.7.11.md +31 -0
- package/docs/RELEASE-NOTES-v0.7.12.md +22 -0
- package/docs/RELEASE-NOTES-v0.7.13.md +41 -0
- package/docs/RELEASE-NOTES-v0.7.14.md +37 -0
- package/docs/RELEASE-NOTES-v0.7.15.md +36 -0
- package/docs/RELEASE-NOTES-v0.7.16.md +42 -0
- package/docs/RELEASE-NOTES-v0.7.5.md +134 -0
- package/docs/RELEASE-NOTES-v0.7.6.md +74 -0
- package/docs/RELEASE-NOTES-v0.7.7.md +60 -0
- package/docs/RELEASE-NOTES-v0.7.8.md +56 -0
- package/docs/RELEASE-NOTES-v0.7.9.md +46 -0
- package/docs/RELEASE-NOTES-v0.8.0.md +59 -0
- package/docs/RELEASE-NOTES-v0.8.1.md +70 -0
- package/docs/RELEASE-NOTES-v0.8.2.md +62 -0
- package/docs/RELEASE-NOTES-v0.8.3.md +65 -0
- package/docs/RELEASE-NOTES-v0.8.4.md +56 -0
- package/docs/RELEASE-NOTES-v0.8.5.md +60 -0
- package/docs/RELEASE-NOTES-v0.8.6.md +71 -0
- package/docs/RELEASE-NOTES-v0.8.7.md +69 -0
- package/eval/gate-audit.mjs +538 -0
- package/lib/api.js +70 -0
- package/lib/approvals.js +84 -0
- package/lib/boundary-prune.js +71 -5
- package/lib/capsule.js +134 -0
- package/lib/capture.js +83 -5
- package/lib/client.js +66 -0
- package/lib/compaction.js +4 -11
- package/lib/context-gc.js +63 -2
- package/lib/index-block.js +79 -4
- package/lib/index.js +105 -18
- package/lib/ledger.js +200 -10
- package/lib/pack-telemetry.js +10 -3
- package/lib/prune.js +89 -0
- package/lib/push-legs.js +121 -0
- package/lib/session-events.js +44 -2
- package/lib/skill-export.js +289 -0
- package/lib/status-insights.js +5 -0
- package/lib/status.js +138 -0
- package/lib/store.js +34 -0
- package/lib/usage.js +32 -3
- package/lib/web-provision.js +59 -14
- package/package.json +15 -8
- package/scripts/dsh-engram.mjs +247 -5
package/lib/ledger.js
CHANGED
|
@@ -30,9 +30,15 @@ import path from "node:path";
|
|
|
30
30
|
|
|
31
31
|
const FILE_NAME = "ledger.json";
|
|
32
32
|
/** Matches the estimate used by the GUI (`chars / 4`). */
|
|
33
|
-
const CHARS_PER_TOKEN = 4;
|
|
33
|
+
export const CHARS_PER_TOKEN = 4;
|
|
34
34
|
const DEFAULT_CACHE_DISCOUNT = 0.1;
|
|
35
35
|
const DEFAULT_FLUSH_EVERY = 20;
|
|
36
|
+
/** Per-session rows kept on disk (newest activity wins; older ones drop). */
|
|
37
|
+
const DEFAULT_MAX_SESSION_ROWS = 200;
|
|
38
|
+
/** Rows returned by `summary()` — the GUI never needs the whole history. */
|
|
39
|
+
const SUMMARY_SESSION_ROWS = 50;
|
|
40
|
+
/** Session ids that would land on the prototype chain if used as row keys. */
|
|
41
|
+
const RESERVED_ROW_KEYS = new Set(["__proto__", "constructor", "prototype"]);
|
|
36
42
|
|
|
37
43
|
/** Default ledger file under the engram home. */
|
|
38
44
|
export function ledgerFile(home = path.join(os.homedir(), ".dsh", "engram")) {
|
|
@@ -84,6 +90,13 @@ function deltaOf(session) {
|
|
|
84
90
|
avoidedChars: session.avoidedChars - f.avoidedChars,
|
|
85
91
|
rewriteCount: session.rewriteCount - f.rewriteCount,
|
|
86
92
|
rewriteChars: session.rewriteChars - f.rewriteChars,
|
|
93
|
+
// These three used to be written as absolute values (`row.x = session.x`),
|
|
94
|
+
// which silently zeroed a session's history on every restart: the in-memory
|
|
95
|
+
// counters start at 0 and the first flush after boot overwrote the durable
|
|
96
|
+
// row. Deltas are the only thing a restart can't corrupt.
|
|
97
|
+
prunedChars: count(session.prunedChars) - count(f.prunedChars),
|
|
98
|
+
hostPrunedChars: count(session.hostPrunedChars) - count(f.hostPrunedChars),
|
|
99
|
+
compactedChars: count(session.compactedChars) - count(f.compactedChars),
|
|
87
100
|
};
|
|
88
101
|
}
|
|
89
102
|
|
|
@@ -93,7 +106,10 @@ function isEmptyDelta(delta) {
|
|
|
93
106
|
delta.injectedChars === 0 &&
|
|
94
107
|
delta.avoidedChars === 0 &&
|
|
95
108
|
delta.rewriteCount === 0 &&
|
|
96
|
-
delta.rewriteChars === 0
|
|
109
|
+
delta.rewriteChars === 0 &&
|
|
110
|
+
delta.prunedChars === 0 &&
|
|
111
|
+
delta.hostPrunedChars === 0 &&
|
|
112
|
+
delta.compactedChars === 0
|
|
97
113
|
);
|
|
98
114
|
}
|
|
99
115
|
|
|
@@ -119,6 +135,48 @@ export function finalizeLedgerBucket(bucket, cacheDiscount = DEFAULT_CACHE_DISCO
|
|
|
119
135
|
};
|
|
120
136
|
}
|
|
121
137
|
|
|
138
|
+
/** A durable per-session row: what ONE session spent and saved (raw counts). */
|
|
139
|
+
function emptySessionRow(sessionId, workspace) {
|
|
140
|
+
return {
|
|
141
|
+
sessionId,
|
|
142
|
+
workspace: typeof workspace === "string" ? workspace : "",
|
|
143
|
+
firstDay: "",
|
|
144
|
+
lastDay: "",
|
|
145
|
+
updatedAt: 0,
|
|
146
|
+
requests: 0,
|
|
147
|
+
injectedChars: 0,
|
|
148
|
+
avoidedChars: 0,
|
|
149
|
+
rewriteCount: 0,
|
|
150
|
+
rewriteChars: 0,
|
|
151
|
+
prunedChars: 0,
|
|
152
|
+
hostPrunedChars: 0,
|
|
153
|
+
compactedChars: 0,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Finalize a session row with the same net arithmetic as a bucket (pure).
|
|
159
|
+
*
|
|
160
|
+
* The gates in docs/PROPOSAL-evolution.zh.md §6 are per-session questions
|
|
161
|
+
* ("净收益 > 0 的会话占比", "长会话占比"), which a day-level rollup cannot
|
|
162
|
+
* answer — it aggregates them away. These rows exist so the answer is a
|
|
163
|
+
* read, not an estimate.
|
|
164
|
+
*/
|
|
165
|
+
export function finalizeSessionRow(row, cacheDiscount = DEFAULT_CACHE_DISCOUNT) {
|
|
166
|
+
const { sessions: _bucketSessions, ...base } = finalizeLedgerBucket(row, cacheDiscount);
|
|
167
|
+
return {
|
|
168
|
+
sessionId: String(row?.sessionId ?? ""),
|
|
169
|
+
workspace: String(row?.workspace ?? ""),
|
|
170
|
+
firstDay: String(row?.firstDay ?? ""),
|
|
171
|
+
lastDay: String(row?.lastDay ?? ""),
|
|
172
|
+
updatedAt: count(row?.updatedAt),
|
|
173
|
+
prunedChars: count(row?.prunedChars),
|
|
174
|
+
hostPrunedChars: count(row?.hostPrunedChars),
|
|
175
|
+
compactedChars: count(row?.compactedChars),
|
|
176
|
+
...base,
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
|
|
122
180
|
function readPersisted(file) {
|
|
123
181
|
try {
|
|
124
182
|
const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
@@ -188,6 +246,7 @@ export function estimatePrefixChars(session) {
|
|
|
188
246
|
* @param {string} [opts.home] engram home (defaults to ~/.dsh/engram)
|
|
189
247
|
* @param {number} [opts.cacheDiscount] cached-prefix price share (0..1)
|
|
190
248
|
* @param {number} [opts.flushEvery] requests between durable flushes
|
|
249
|
+
* @param {number} [opts.maxSessionRows] per-session rows kept on disk
|
|
191
250
|
* @param {boolean} [opts.persist] false = pure in-memory (tests, dry runs)
|
|
192
251
|
* @param {() => number} [opts.now] clock injection for tests
|
|
193
252
|
* @param {object} [opts.log] optional logger (warn only)
|
|
@@ -196,6 +255,7 @@ export function createLedger({
|
|
|
196
255
|
home = path.join(os.homedir(), ".dsh", "engram"),
|
|
197
256
|
cacheDiscount = DEFAULT_CACHE_DISCOUNT,
|
|
198
257
|
flushEvery = DEFAULT_FLUSH_EVERY,
|
|
258
|
+
maxSessionRows = DEFAULT_MAX_SESSION_ROWS,
|
|
199
259
|
persist = true,
|
|
200
260
|
now = Date.now,
|
|
201
261
|
log = void 0,
|
|
@@ -204,13 +264,17 @@ export function createLedger({
|
|
|
204
264
|
const discount = clampDiscount(cacheDiscount);
|
|
205
265
|
const every = Number.isFinite(flushEvery) && flushEvery > 0 ? Math.trunc(flushEvery) : 0;
|
|
206
266
|
const stored = persist ? readPersisted(file) : null;
|
|
207
|
-
const state = stored ?? { version: 1, updatedAt: 0, workspaces: {} };
|
|
267
|
+
const state = stored ?? { version: 1, updatedAt: 0, workspaces: {}, sessions: {} };
|
|
268
|
+
if (state.sessions === null || typeof state.sessions !== "object") state.sessions = {};
|
|
269
|
+
const rowCap = Number.isFinite(maxSessionRows) && maxSessionRows > 0 ? Math.trunc(maxSessionRows) : DEFAULT_MAX_SESSION_ROWS;
|
|
208
270
|
const sessions = new Map();
|
|
209
271
|
|
|
210
272
|
function ensure(sessionId, workspace) {
|
|
211
273
|
if (typeof sessionId !== "string" || sessionId.length === 0) return null;
|
|
212
274
|
let session = sessions.get(sessionId);
|
|
213
275
|
if (session === void 0) {
|
|
276
|
+
const persisted = state.sessions?.[sessionId];
|
|
277
|
+
const seed = persisted !== null && typeof persisted === "object" ? persisted : null;
|
|
214
278
|
session = {
|
|
215
279
|
sessionId,
|
|
216
280
|
workspace: typeof workspace === "string" ? workspace : "",
|
|
@@ -220,11 +284,22 @@ export function createLedger({
|
|
|
220
284
|
requests: 0,
|
|
221
285
|
injectedChars: 0,
|
|
222
286
|
avoidedChars: 0,
|
|
223
|
-
prunedChars:
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
287
|
+
prunedChars: count(seed?.prunedChars),
|
|
288
|
+
hostPrunedChars: count(seed?.hostPrunedChars),
|
|
289
|
+
compactedChars: count(seed?.compactedChars),
|
|
290
|
+
rewriteCount: count(seed?.rewriteCount),
|
|
291
|
+
rewriteChars: count(seed?.rewriteChars),
|
|
292
|
+
updatedAt: 0,
|
|
293
|
+
flushed: {
|
|
294
|
+
requests: 0,
|
|
295
|
+
injectedChars: 0,
|
|
296
|
+
avoidedChars: 0,
|
|
297
|
+
rewriteCount: count(seed?.rewriteCount),
|
|
298
|
+
rewriteChars: count(seed?.rewriteChars),
|
|
299
|
+
prunedChars: count(seed?.prunedChars),
|
|
300
|
+
hostPrunedChars: count(seed?.hostPrunedChars),
|
|
301
|
+
compactedChars: count(seed?.compactedChars),
|
|
302
|
+
},
|
|
228
303
|
firstFlush: false,
|
|
229
304
|
};
|
|
230
305
|
sessions.set(sessionId, session);
|
|
@@ -249,6 +324,34 @@ export function createLedger({
|
|
|
249
324
|
return days[day];
|
|
250
325
|
}
|
|
251
326
|
|
|
327
|
+
/** The durable row for one session, created on first flush. */
|
|
328
|
+
function rowFor(session, day) {
|
|
329
|
+
// session ids are host-generated, but a row key must never hit the
|
|
330
|
+
// prototype chain.
|
|
331
|
+
if (RESERVED_ROW_KEYS.has(session.sessionId)) return null;
|
|
332
|
+
let row = state.sessions[session.sessionId];
|
|
333
|
+
if (row === null || row === void 0 || typeof row !== "object") {
|
|
334
|
+
row = emptySessionRow(session.sessionId, session.workspace);
|
|
335
|
+
state.sessions[session.sessionId] = row;
|
|
336
|
+
}
|
|
337
|
+
if (row.firstDay === void 0 || row.firstDay.length === 0) row.firstDay = day;
|
|
338
|
+
row.lastDay = day;
|
|
339
|
+
row.updatedAt = Math.max(count(row.updatedAt), count(session.updatedAt), now());
|
|
340
|
+
if (typeof session.workspace === "string" && session.workspace.length > 0) row.workspace = session.workspace;
|
|
341
|
+
return row;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/** Keep the newest `rowCap` rows; a session that never ran is not history. */
|
|
345
|
+
function pruneRows() { const entries = Object.entries(state.sessions);
|
|
346
|
+
if (entries.length <= rowCap) return;
|
|
347
|
+
entries
|
|
348
|
+
.sort((a, b) => count(b[1]?.updatedAt) - count(a[1]?.updatedAt))
|
|
349
|
+
.slice(rowCap)
|
|
350
|
+
.forEach(([id]) => {
|
|
351
|
+
delete state.sessions[id];
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
|
|
252
355
|
function flushSession(session) {
|
|
253
356
|
if (session === null || session === void 0) return false;
|
|
254
357
|
const delta = deltaOf(session);
|
|
@@ -261,12 +364,29 @@ export function createLedger({
|
|
|
261
364
|
session.firstFlush = true;
|
|
262
365
|
}
|
|
263
366
|
addBucket(bucket, delta);
|
|
367
|
+
// Per-session row (same delta, same instant) — the gates need the
|
|
368
|
+
// distribution, not the sum.
|
|
369
|
+
const row = rowFor(session, day);
|
|
370
|
+
if (row !== null) {
|
|
371
|
+
row.requests += count(delta.requests);
|
|
372
|
+
row.injectedChars += count(delta.injectedChars);
|
|
373
|
+
row.avoidedChars += count(delta.avoidedChars);
|
|
374
|
+
row.rewriteCount += count(delta.rewriteCount);
|
|
375
|
+
row.rewriteChars += count(delta.rewriteChars);
|
|
376
|
+
row.prunedChars += count(delta.prunedChars);
|
|
377
|
+
row.hostPrunedChars += count(delta.hostPrunedChars);
|
|
378
|
+
row.compactedChars += count(delta.compactedChars);
|
|
379
|
+
pruneRows();
|
|
380
|
+
}
|
|
264
381
|
session.flushed = {
|
|
265
382
|
requests: session.requests,
|
|
266
383
|
injectedChars: session.injectedChars,
|
|
267
384
|
avoidedChars: session.avoidedChars,
|
|
268
385
|
rewriteCount: session.rewriteCount,
|
|
269
386
|
rewriteChars: session.rewriteChars,
|
|
387
|
+
prunedChars: count(session.prunedChars),
|
|
388
|
+
hostPrunedChars: count(session.hostPrunedChars),
|
|
389
|
+
compactedChars: count(session.compactedChars),
|
|
270
390
|
};
|
|
271
391
|
state.updatedAt = now();
|
|
272
392
|
if (persist) writeJsonAtomic(file, state);
|
|
@@ -281,6 +401,40 @@ export function createLedger({
|
|
|
281
401
|
for (const session of sessions.values()) flushSession(session);
|
|
282
402
|
}
|
|
283
403
|
|
|
404
|
+
/**
|
|
405
|
+
* Per-session rows for reading (persisted rows + the open sessions'
|
|
406
|
+
* unflushed tails merged in). Non-mutating, newest activity first.
|
|
407
|
+
*/
|
|
408
|
+
function sessionRows(limit = SUMMARY_SESSION_ROWS) {
|
|
409
|
+
const rows = new Map();
|
|
410
|
+
for (const [id, row] of Object.entries(state.sessions ?? {})) {
|
|
411
|
+
if (row === null || typeof row !== "object") continue;
|
|
412
|
+
rows.set(id, { ...emptySessionRow(id, row.workspace), ...row });
|
|
413
|
+
}
|
|
414
|
+
const today = ledgerDayKey(now());
|
|
415
|
+
for (const session of sessions.values()) {
|
|
416
|
+
const delta = deltaOf(session);
|
|
417
|
+
const row = rows.get(session.sessionId) ?? emptySessionRow(session.sessionId, session.workspace);
|
|
418
|
+
row.requests += count(delta.requests);
|
|
419
|
+
row.injectedChars += count(delta.injectedChars);
|
|
420
|
+
row.avoidedChars += count(delta.avoidedChars);
|
|
421
|
+
row.rewriteCount += count(delta.rewriteCount);
|
|
422
|
+
row.rewriteChars += count(delta.rewriteChars);
|
|
423
|
+
row.prunedChars = count(session.prunedChars);
|
|
424
|
+
row.hostPrunedChars = count(session.hostPrunedChars);
|
|
425
|
+
row.compactedChars = count(session.compactedChars);
|
|
426
|
+
if (session.workspace.length > 0) row.workspace = session.workspace;
|
|
427
|
+
if (typeof row.firstDay !== "string" || row.firstDay.length === 0) row.firstDay = today;
|
|
428
|
+
row.lastDay = today;
|
|
429
|
+
if (session.updatedAt > 0) row.updatedAt = Math.max(count(row.updatedAt), count(session.updatedAt));
|
|
430
|
+
rows.set(session.sessionId, row);
|
|
431
|
+
}
|
|
432
|
+
return [...rows.values()]
|
|
433
|
+
.map((row) => finalizeSessionRow(row, discount))
|
|
434
|
+
.sort((a, b) => b.updatedAt - a.updatedAt)
|
|
435
|
+
.slice(0, limit);
|
|
436
|
+
}
|
|
437
|
+
|
|
284
438
|
return {
|
|
285
439
|
/** Record the latest injected block size for one component. */
|
|
286
440
|
noteInjection(sessionId, workspace, component, chars) {
|
|
@@ -288,6 +442,7 @@ export function createLedger({
|
|
|
288
442
|
const session = ensure(sessionId, workspace);
|
|
289
443
|
if (session === null) return;
|
|
290
444
|
const size = count(chars);
|
|
445
|
+
session.updatedAt = now();
|
|
291
446
|
if (component === "esr") session.esrChars = size;
|
|
292
447
|
else if (component === "jit") session.jitChars = size;
|
|
293
448
|
else session.indexChars = size;
|
|
@@ -302,6 +457,7 @@ export function createLedger({
|
|
|
302
457
|
const session = ensure(sessionId, workspace);
|
|
303
458
|
if (session === null) return;
|
|
304
459
|
session.requests += 1;
|
|
460
|
+
session.updatedAt = now();
|
|
305
461
|
session.injectedChars += session.indexChars + session.esrChars + session.jitChars;
|
|
306
462
|
session.avoidedChars += session.prunedChars + session.compactedChars;
|
|
307
463
|
if (every > 0 && session.requests % every === 0) flushSession(session);
|
|
@@ -310,11 +466,29 @@ export function createLedger({
|
|
|
310
466
|
}
|
|
311
467
|
},
|
|
312
468
|
|
|
313
|
-
/**
|
|
314
|
-
|
|
469
|
+
/**
|
|
470
|
+
* A prune happened: its removed chars stop replaying from the next request.
|
|
471
|
+
*
|
|
472
|
+
* `source` keeps the attribution honest — only prunes engram CAUSED count
|
|
473
|
+
* toward `prunedChars` (and therefore toward avoidedChars/NetTokens). The
|
|
474
|
+
* host prunes over-threshold results on its own (observed live: 12,491
|
|
475
|
+
* tokens shadowed with no boundary in sight); those land in
|
|
476
|
+
* `hostPrunedChars`, visible but never claimed as engram's saving.
|
|
477
|
+
*/
|
|
478
|
+
notePrune(sessionId, workspace, { charsRemoved = 0, rewriteChars = 0, source = "boundary" } = {}) {
|
|
315
479
|
try {
|
|
316
480
|
const session = ensure(sessionId, workspace);
|
|
317
481
|
if (session === null) return;
|
|
482
|
+
session.updatedAt = now();
|
|
483
|
+
if (source === "host") {
|
|
484
|
+
// Symmetry: we do not claim the host's savings, so we do not pay its
|
|
485
|
+
// rewrite cost either. Charging both ways made ordinary sessions look
|
|
486
|
+
// catastrophic (1.22M chars of rewrite for prunes engram never asked
|
|
487
|
+
// for) — the exact bias that turns a neutral session into -1.7M.
|
|
488
|
+
session.hostPrunedChars += count(charsRemoved);
|
|
489
|
+
flushSession(session);
|
|
490
|
+
return;
|
|
491
|
+
}
|
|
318
492
|
session.prunedChars += count(charsRemoved);
|
|
319
493
|
session.rewriteCount += 1;
|
|
320
494
|
session.rewriteChars += count(rewriteChars);
|
|
@@ -329,6 +503,7 @@ export function createLedger({
|
|
|
329
503
|
try {
|
|
330
504
|
const session = ensure(sessionId, workspace);
|
|
331
505
|
if (session === null) return;
|
|
506
|
+
session.updatedAt = now();
|
|
332
507
|
session.compactedChars += count(charsSaved);
|
|
333
508
|
} catch (error) {
|
|
334
509
|
log?.warn?.(`engram ledger noteCompaction failed: ${String(error)}`);
|
|
@@ -343,6 +518,7 @@ export function createLedger({
|
|
|
343
518
|
try {
|
|
344
519
|
const session = ensure(sessionId, workspace);
|
|
345
520
|
if (session === null) return;
|
|
521
|
+
session.updatedAt = now();
|
|
346
522
|
session.rewriteCount += 1;
|
|
347
523
|
session.rewriteChars += count(chars);
|
|
348
524
|
flushSession(session);
|
|
@@ -402,6 +578,11 @@ export function createLedger({
|
|
|
402
578
|
const totals = emptyBucket();
|
|
403
579
|
for (const bucket of Object.values(byWorkspace)) addBucket(totals, bucket);
|
|
404
580
|
|
|
581
|
+
const rowsHostPruned = sessionRows(SUMMARY_SESSION_ROWS).reduce(
|
|
582
|
+
(sum, row) => sum + count(row.hostPrunedChars),
|
|
583
|
+
0,
|
|
584
|
+
);
|
|
585
|
+
|
|
405
586
|
const finalizedWorkspaces = {};
|
|
406
587
|
for (const [ws, bucket] of Object.entries(byWorkspace)) {
|
|
407
588
|
finalizedWorkspaces[ws] = finalizeLedgerBucket(bucket, discount);
|
|
@@ -417,8 +598,11 @@ export function createLedger({
|
|
|
417
598
|
cacheDiscount: discount,
|
|
418
599
|
openSessions: sessions.size,
|
|
419
600
|
totals: finalizeLedgerBucket(totals, discount),
|
|
601
|
+
// Savings the HOST performed on its own (visible, never claimed).
|
|
602
|
+
hostPrunedChars: rowsHostPruned,
|
|
420
603
|
byWorkspace: finalizedWorkspaces,
|
|
421
604
|
byDay: finalizedDays,
|
|
605
|
+
sessions: sessionRows(SUMMARY_SESSION_ROWS),
|
|
422
606
|
};
|
|
423
607
|
},
|
|
424
608
|
};
|
|
@@ -442,6 +626,10 @@ export function readLedger({ home, cacheDiscount = DEFAULT_CACHE_DISCOUNT } = {}
|
|
|
442
626
|
}
|
|
443
627
|
const totals = emptyBucket();
|
|
444
628
|
for (const bucket of Object.values(workspaces)) addBucket(totals, bucket);
|
|
629
|
+
const rows = Object.values(stored?.sessions ?? {})
|
|
630
|
+
.filter((row) => row !== null && typeof row === "object")
|
|
631
|
+
.map((row) => finalizeSessionRow(row, cacheDiscount))
|
|
632
|
+
.sort((a, b) => b.updatedAt - a.updatedAt);
|
|
445
633
|
return {
|
|
446
634
|
version: 1,
|
|
447
635
|
updatedAt: stored?.updatedAt ?? 0,
|
|
@@ -454,5 +642,7 @@ export function readLedger({ home, cacheDiscount = DEFAULT_CACHE_DISCOUNT } = {}
|
|
|
454
642
|
byDay: Object.entries(days)
|
|
455
643
|
.map(([day, bucket]) => ({ day, ...finalizeLedgerBucket(bucket, cacheDiscount) }))
|
|
456
644
|
.sort((a, b) => (a.day < b.day ? 1 : a.day > b.day ? -1 : 0)),
|
|
645
|
+
sessions: rows,
|
|
646
|
+
hostPrunedChars: rows.reduce((sum, row) => sum + count(row.hostPrunedChars), 0),
|
|
457
647
|
};
|
|
458
648
|
}
|
package/lib/pack-telemetry.js
CHANGED
|
@@ -21,9 +21,16 @@ import fs from "node:fs";
|
|
|
21
21
|
import os from "node:os";
|
|
22
22
|
import path from "node:path";
|
|
23
23
|
|
|
24
|
-
/** Default engram home (
|
|
25
|
-
|
|
26
|
-
|
|
24
|
+
/** Default engram home: `$DSH_HOME/engram` when set (tests, sandboxes, custom
|
|
25
|
+
* installs), else `~/.dsh/engram` (mirrors status.js / result-pack.js). A
|
|
26
|
+
* plain `~/.dsh` default made every test run write live telemetry into the
|
|
27
|
+
* developer's real install. */
|
|
28
|
+
export function engramHome(home = void 0) {
|
|
29
|
+
if (typeof home === "string" && home.length > 0) return home;
|
|
30
|
+
const dshHome = process.env.DSH_HOME;
|
|
31
|
+
return dshHome !== void 0 && dshHome.length > 0
|
|
32
|
+
? path.join(dshHome, "engram")
|
|
33
|
+
: path.join(os.homedir(), ".dsh", "engram");
|
|
27
34
|
}
|
|
28
35
|
|
|
29
36
|
function statusFile(home, name) {
|
package/lib/prune.js
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-engram — BoundaryPrune (L1) as a mountable entry INSIDE the agent preset.
|
|
3
|
+
*
|
|
4
|
+
* Why this entry exists: web sessions live in the agent preset's own realm, and
|
|
5
|
+
* their `session/event`s are only observed by plugins mounted in that realm.
|
|
6
|
+
* The host-plane install armed fine (`boundaryPrune.armed: true`) and still never
|
|
7
|
+
* fired across every session on the profile: the listener was on the wrong side
|
|
8
|
+
* of the realm boundary, so a real `todo/write` boundary produced nothing — the
|
|
9
|
+
* ledger's prune credit leg was structurally dead for `dsh web`.
|
|
10
|
+
*
|
|
11
|
+
* This row sits in the same `compaction` group as `dsh-engram/compaction`
|
|
12
|
+
* (which already isolates `toolResultPruner` for the agent plane), so it sees
|
|
13
|
+
* the sessions' events and holds that realm's pruner:
|
|
14
|
+
*
|
|
15
|
+
* ```yaml
|
|
16
|
+
* - id: compaction
|
|
17
|
+
* name: cordis:group
|
|
18
|
+
* group: true
|
|
19
|
+
* isolate: { compaction: true, toolResultPruner: true }
|
|
20
|
+
* config:
|
|
21
|
+
* - id: engram-compaction
|
|
22
|
+
* name: dsh-engram/compaction
|
|
23
|
+
* - id: engram-prune
|
|
24
|
+
* name: dsh-engram/prune # this entry
|
|
25
|
+
* - id: command-compact
|
|
26
|
+
* name: '@deepseek-ai/dsh-command-compact'
|
|
27
|
+
* - id: tool-result-pruner
|
|
28
|
+
* name: '@deepseek-ai/dsh-compaction-tool-result-pruner'
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* The host plugin still installs its own L1 for host-plane (headless/TUI)
|
|
32
|
+
* sessions. Running both is safe: `toolResultPruner.pruneSession` only prunes
|
|
33
|
+
* over-threshold results, so a second pass over an already-pruned surface finds
|
|
34
|
+
* nothing and credits nothing.
|
|
35
|
+
*
|
|
36
|
+
* Config: the same keys the host entries read (`boundaryPrune`,
|
|
37
|
+
* `boundaryPruneMinRemaining`, `boundaryPruneThresholdChars`,
|
|
38
|
+
* `boundaryPruneHeadChars`, `boundaryPruneTailChars`,
|
|
39
|
+
* `boundaryPruneIdleCadence`, `boundaryPruneAdaptive`). Ledger credit goes
|
|
40
|
+
* through the host-provided `engramLedger` service so there is exactly one
|
|
41
|
+
* ledger writer.
|
|
42
|
+
*/
|
|
43
|
+
import { installBoundaryPrune, boundaryPruneAvailability } from "./boundary-prune.js";
|
|
44
|
+
import { estimatePrefixChars } from "./ledger.js";
|
|
45
|
+
import { recordBoundaryPruneFire, engramHome } from "./pack-telemetry.js";
|
|
46
|
+
import { recordPruneAttempt } from "./status.js";
|
|
47
|
+
import { workspaceKey as wk } from "./util.js";
|
|
48
|
+
|
|
49
|
+
export const name = "dsh-engram-prune";
|
|
50
|
+
export const inject = ["toolResultPruner"];
|
|
51
|
+
|
|
52
|
+
/** The plane tag every attempt from this entry carries. */
|
|
53
|
+
export const PLANE = "preset";
|
|
54
|
+
|
|
55
|
+
export function apply(ctx, config = {}) {
|
|
56
|
+
const log = typeof ctx?.logger === "function" ? ctx.logger("dsh-engram-prune") : undefined;
|
|
57
|
+
|
|
58
|
+
// Diagnostics first: proves the row loaded at all (a row that fails to load
|
|
59
|
+
// looks exactly like one that works, until nothing ever fires).
|
|
60
|
+
try {
|
|
61
|
+
ctx.logger?.info?.(`engram boundary-prune (preset plane): ${boundaryPruneAvailability(ctx, config).reason ?? "armed"}`);
|
|
62
|
+
} catch { /* never break the mount for a log line */ }
|
|
63
|
+
|
|
64
|
+
const hooks = {
|
|
65
|
+
onPruned: (session, report, boundary) => {
|
|
66
|
+
const workspace = wk(session?.header?.cwd);
|
|
67
|
+
// Durable activation telemetry (shared with the host plane; the summary
|
|
68
|
+
// reports `activation.boundaryPrune` from it).
|
|
69
|
+
recordBoundaryPruneFire({ home: engramHome(), boundary, report, workspace });
|
|
70
|
+
// The prune rewrote the prefix: credit the removed chars, price the
|
|
71
|
+
// invalidation with the pre-prune size (mirrors the host-plane path).
|
|
72
|
+
const removed = Number.isFinite(report?.charsRemoved) ? report.charsRemoved : 0;
|
|
73
|
+
const prefixChars = estimatePrefixChars(session);
|
|
74
|
+
try {
|
|
75
|
+
const ledger = ctx.get?.("engramLedger");
|
|
76
|
+
ledger?.notePrune?.(session?.id, workspace, {
|
|
77
|
+
charsRemoved: removed,
|
|
78
|
+
rewriteChars: prefixChars + removed,
|
|
79
|
+
});
|
|
80
|
+
} catch { /* accounting must never break pruning */ }
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
return installBoundaryPrune(ctx, config, log, hooks, {
|
|
85
|
+
onAttempt: (attempt) => {
|
|
86
|
+
void recordPruneAttempt({ plane: PLANE, ...attempt });
|
|
87
|
+
},
|
|
88
|
+
});
|
|
89
|
+
}
|
package/lib/push-legs.js
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Observability for the four features that read the session event log.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS (2026-10-27, issue #9): `[RECALL]`, the session episode, the
|
|
5
|
+
* resume brief and the session-end todo sink all depend on ONE host capability
|
|
6
|
+
* — a readable session event log. When a host stops exposing every read path,
|
|
7
|
+
* the four of them sleep together and the only trace is a single `warn` line on
|
|
8
|
+
* a terminal nobody kept. The reporter in issue #9 had to decompress
|
|
9
|
+
* `session.v4.jsonl.zstd` frames by hand to prove the features were not merely
|
|
10
|
+
* "not triggered yet".
|
|
11
|
+
*
|
|
12
|
+
* A warning is not an artifact. This module records which read path the live
|
|
13
|
+
* process actually took, once per session, into
|
|
14
|
+
* `$DSH_HOME/engram/push-legs.jsonl` so `dsh-engram status` can answer
|
|
15
|
+
* "were the push legs awake?" without a session, a log capture, or a report.
|
|
16
|
+
*
|
|
17
|
+
* The record is append-only and bounded (tail kept, like the other telemetry
|
|
18
|
+
* files): diagnostics must never grow without limit or throw into the
|
|
19
|
+
* assembly path.
|
|
20
|
+
* @module dsh-engram/push-legs
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { appendFileSync, existsSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs";
|
|
24
|
+
import { mkdirSync } from "node:fs";
|
|
25
|
+
import { dirname, join } from "node:path";
|
|
26
|
+
|
|
27
|
+
/** Rows kept before the file is rewritten (tail-first). */
|
|
28
|
+
export const PUSH_LEGS_KEEP = 50;
|
|
29
|
+
/** Size at which the file is rewritten to the kept tail. */
|
|
30
|
+
const PUSH_LEGS_MAX_BYTES = 32 * 1024;
|
|
31
|
+
|
|
32
|
+
/** `<home>/engram/push-legs.jsonl` */
|
|
33
|
+
export function pushLegsPath(home) {
|
|
34
|
+
return join(home, "engram", "push-legs.jsonl");
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Record the read path taken for one session. Best effort: never throws.
|
|
39
|
+
*
|
|
40
|
+
* @param {object} options
|
|
41
|
+
* @param {string} options.home DSH home
|
|
42
|
+
* @param {string} options.source `sessionEventsSource()` verdict
|
|
43
|
+
* @param {string} [options.sessionId]
|
|
44
|
+
* @param {number} [options.pid]
|
|
45
|
+
* @param {number} [options.now]
|
|
46
|
+
* @returns {boolean} whether the row was written
|
|
47
|
+
*/
|
|
48
|
+
export function noteSessionEventsSource({ home, source, sessionId, pid = process.pid, now = Date.now() } = {}) {
|
|
49
|
+
if (typeof home !== "string" || home.length === 0) return false;
|
|
50
|
+
if (typeof source !== "string" || source.length === 0) return false;
|
|
51
|
+
const path = pushLegsPath(home);
|
|
52
|
+
const row = `${JSON.stringify({ at: now, pid, sessionId: sessionId ?? null, source })}\n`;
|
|
53
|
+
try {
|
|
54
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
55
|
+
if (existsSync(path) && statSync(path).size > PUSH_LEGS_MAX_BYTES) {
|
|
56
|
+
const kept = readFileSync(path, "utf8").split("\n").filter((line) => line.length > 0).slice(-PUSH_LEGS_KEEP);
|
|
57
|
+
writeFileSync(`${path}.tmp`, `${kept.join("\n")}\n`, "utf8");
|
|
58
|
+
renameSync(`${path}.tmp`, path);
|
|
59
|
+
}
|
|
60
|
+
appendFileSync(path, row, "utf8");
|
|
61
|
+
return true;
|
|
62
|
+
} catch {
|
|
63
|
+
return false;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Newest recorded read path (and how many rows mention each verdict).
|
|
69
|
+
* Never throws; returns `undefined` when nothing was ever recorded.
|
|
70
|
+
*
|
|
71
|
+
* @param {string} home DSH home
|
|
72
|
+
* @returns {{at: number, source: string, sessionId: string|null, pid: number, counts: Record<string, number>} | undefined}
|
|
73
|
+
*/
|
|
74
|
+
export function readPushLegsHealth(home) {
|
|
75
|
+
if (typeof home !== "string" || home.length === 0) return undefined;
|
|
76
|
+
let raw;
|
|
77
|
+
try {
|
|
78
|
+
raw = readFileSync(pushLegsPath(home), "utf8");
|
|
79
|
+
} catch {
|
|
80
|
+
return undefined;
|
|
81
|
+
}
|
|
82
|
+
const counts = {};
|
|
83
|
+
let newest;
|
|
84
|
+
for (const line of raw.split("\n")) {
|
|
85
|
+
if (line.length === 0) continue;
|
|
86
|
+
let row;
|
|
87
|
+
try {
|
|
88
|
+
row = JSON.parse(line);
|
|
89
|
+
} catch {
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
if (typeof row?.source !== "string") continue;
|
|
93
|
+
counts[row.source] = (counts[row.source] ?? 0) + 1;
|
|
94
|
+
if (newest === undefined || (typeof row.at === "number" && row.at >= (newest.at ?? 0))) newest = row;
|
|
95
|
+
}
|
|
96
|
+
if (newest === undefined) return undefined;
|
|
97
|
+
return {
|
|
98
|
+
at: typeof newest.at === "number" ? newest.at : 0,
|
|
99
|
+
source: newest.source,
|
|
100
|
+
sessionId: typeof newest.sessionId === "string" ? newest.sessionId : null,
|
|
101
|
+
pid: typeof newest.pid === "number" ? newest.pid : 0,
|
|
102
|
+
counts,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* One-line human summary for `dsh-engram status`.
|
|
108
|
+
* @param {ReturnType<typeof readPushLegsHealth>} health
|
|
109
|
+
* @returns {string}
|
|
110
|
+
*/
|
|
111
|
+
export function describePushLegs(health) {
|
|
112
|
+
if (health === undefined) return "推送腿:还没记录(本机还没有会话装配过)";
|
|
113
|
+
const when = new Date(health.at).toLocaleString("zh-CN", { timeZone: "Asia/Shanghai" });
|
|
114
|
+
const tally = Object.entries(health.counts)
|
|
115
|
+
.map(([source, count]) => `${source}×${count}`)
|
|
116
|
+
.join(" · ");
|
|
117
|
+
if (health.source === "none") {
|
|
118
|
+
return `推送腿:⚠ 事件日志不可读(none)—— [RECALL]/episode/resume/todo-sink 休眠中 · ${when} · ${tally}`;
|
|
119
|
+
}
|
|
120
|
+
return `推送腿:事件日志读法 ${health.source} · 最近 ${when} · ${tally}`;
|
|
121
|
+
}
|
package/lib/session-events.js
CHANGED
|
@@ -39,16 +39,39 @@ const NO_EVENTS = Object.freeze([]);
|
|
|
39
39
|
* Which read path a session exposes, without materializing the log.
|
|
40
40
|
* @param {object} session
|
|
41
41
|
* @param {object} [sessionQuery] the `ctx.sessionQuery` service, if mounted
|
|
42
|
-
* @returns {"legacy"|"sessionQuery"|"snapshotEvents"|"none"}
|
|
42
|
+
* @returns {"legacy"|"sessionQuery"|"snapshotEvents"|"eventAt"|"none"}
|
|
43
43
|
*/
|
|
44
44
|
export function sessionEventsSource(session, sessionQuery = void 0) {
|
|
45
45
|
if (session === null || typeof session !== "object") return "none";
|
|
46
46
|
if (Array.isArray(session.events)) return "legacy";
|
|
47
47
|
if (hasLive(sessionQuery)) return "sessionQuery";
|
|
48
48
|
if (typeof session.snapshotEvents === "function") return "snapshotEvents";
|
|
49
|
+
// Last-resort hand: the per-seq accessor is the primitive every other read
|
|
50
|
+
// is built on, so a host can drop the bulk readers and still answer `eventAt`.
|
|
51
|
+
// The Windows desktop runtime (issue #9) is exactly that host.
|
|
52
|
+
if (typeof session.eventAt === "function") return "eventAt";
|
|
49
53
|
return "none";
|
|
50
54
|
}
|
|
51
55
|
|
|
56
|
+
/**
|
|
57
|
+
* What the session object actually offers, for the dormancy warning. A bare
|
|
58
|
+
* "nothing is readable" tells the reporter nothing; the method names tell us
|
|
59
|
+
* which generation of the host we are talking to.
|
|
60
|
+
* @param {object} session
|
|
61
|
+
* @returns {string[]} sorted prototype method names (own properties included)
|
|
62
|
+
*/
|
|
63
|
+
export function describeSessionSurface(session) {
|
|
64
|
+
if (session === null || typeof session !== "object") return [];
|
|
65
|
+
const names = new Set();
|
|
66
|
+
for (let proto = session; proto !== null && proto !== Object.prototype; proto = Object.getPrototypeOf(proto)) {
|
|
67
|
+
for (const name of Object.getOwnPropertyNames(proto)) {
|
|
68
|
+
if (name === "constructor" || name.startsWith("_")) continue;
|
|
69
|
+
names.add(name);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return [...names].sort();
|
|
73
|
+
}
|
|
74
|
+
|
|
52
75
|
function hasLive(sessionQuery) {
|
|
53
76
|
return (
|
|
54
77
|
sessionQuery !== null &&
|
|
@@ -106,7 +129,26 @@ export function readSessionEvents(session, sessionQuery = void 0) {
|
|
|
106
129
|
}
|
|
107
130
|
}
|
|
108
131
|
|
|
109
|
-
// 4.
|
|
132
|
+
// 4. Per-seq accessor: read the log one event at a time. Slower than the bulk
|
|
133
|
+
// readers and only used when they are gone, but it keeps [RECALL] /
|
|
134
|
+
// episode / todo-sink alive on hosts that dropped `snapshotEvents`.
|
|
135
|
+
if (typeof session.eventAt === "function") {
|
|
136
|
+
const total = typeof session.seq === "number" && session.seq >= 0 ? Math.trunc(session.seq) : Number.POSITIVE_INFINITY;
|
|
137
|
+
const events = [];
|
|
138
|
+
for (let seq = 0; seq < total; seq += 1) {
|
|
139
|
+
let event;
|
|
140
|
+
try {
|
|
141
|
+
event = session.eventAt(seq);
|
|
142
|
+
} catch {
|
|
143
|
+
return events.length > 0 ? events : NO_EVENTS;
|
|
144
|
+
}
|
|
145
|
+
if (event === undefined || event === null) break;
|
|
146
|
+
events.push(event);
|
|
147
|
+
}
|
|
148
|
+
return events;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// 5. Nothing.
|
|
110
152
|
return NO_EVENTS;
|
|
111
153
|
}
|
|
112
154
|
|