dsh-engram 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +180 -39
- package/README.zh.md +149 -40
- package/docs/CONTEXT-GC-GUIDE.zh.md +15 -9
- package/docs/PROPOSAL-evolution.zh.md +708 -0
- package/docs/RELEASE-NOTES-v0.6.0.md +64 -0
- package/docs/RELEASE-NOTES-v0.7.0.md +57 -0
- package/lib/anchor.js +124 -0
- package/lib/api.js +5 -0
- package/lib/boundary-prune.js +27 -1
- package/lib/client.js +92 -2
- package/lib/entity-page.js +200 -0
- package/lib/episode.js +163 -0
- package/lib/index-block.js +74 -4
- package/lib/index.js +247 -19
- package/lib/jit.js +224 -0
- package/lib/ledger.js +458 -0
- package/lib/settings.js +18 -4
- package/lib/store.js +48 -5
- package/lib/tools.js +25 -3
- package/package.json +1 -1
package/lib/ledger.js
ADDED
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context Ledger — the accounting layer of the "context portfolio"
|
|
3
|
+
* (docs/PROPOSAL-evolution.zh.md §4 v0.6-A).
|
|
4
|
+
*
|
|
5
|
+
* Every engram mechanism either spends context (injected index / recall / ESR
|
|
6
|
+
* blocks, held in the prompt prefix for the rest of the session) or saves it
|
|
7
|
+
* (pruned / compacted tool results that stop replaying). The ledger turns that
|
|
8
|
+
* into one auditable number per session, workspace and day:
|
|
9
|
+
*
|
|
10
|
+
* net = avoided − injected − rewriteChars × (1 − cacheDiscount)
|
|
11
|
+
*
|
|
12
|
+
* where `rewriteChars` is the prefix size invalidated by a prune (KV cache
|
|
13
|
+
* break). `cacheDiscount` is the fraction a cached prefix would have cost
|
|
14
|
+
* (0.1 = 10%, the conservative L0 assumption), so the penalty is the
|
|
15
|
+
* non-discounted remainder.
|
|
16
|
+
*
|
|
17
|
+
* Zero LLM, deterministic, fail-contained. Two layers:
|
|
18
|
+
* - live per-session accumulator, ticked by `assistant/message` events
|
|
19
|
+
* - durable daily rollup at `~/.dsh/engram/ledger.json`, delta-flushed so a
|
|
20
|
+
* restart loses at most `flushEvery` requests of accounting
|
|
21
|
+
*
|
|
22
|
+
* Deliberately NOT here: model-quality deltas, actual provider billing, and
|
|
23
|
+
* compaction savings whose before/after size the host does not expose yet.
|
|
24
|
+
* Those stay labelled assumptions, never invented numbers.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import fs from "node:fs";
|
|
28
|
+
import os from "node:os";
|
|
29
|
+
import path from "node:path";
|
|
30
|
+
|
|
31
|
+
const FILE_NAME = "ledger.json";
|
|
32
|
+
/** Matches the estimate used by the GUI (`chars / 4`). */
|
|
33
|
+
const CHARS_PER_TOKEN = 4;
|
|
34
|
+
const DEFAULT_CACHE_DISCOUNT = 0.1;
|
|
35
|
+
const DEFAULT_FLUSH_EVERY = 20;
|
|
36
|
+
|
|
37
|
+
/** Default ledger file under the engram home. */
|
|
38
|
+
export function ledgerFile(home = path.join(os.homedir(), ".dsh", "engram")) {
|
|
39
|
+
return path.join(home, FILE_NAME);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Local YYYY-MM-DD (calendar days for the user, mirrors usage.dayKey). */
|
|
43
|
+
export function ledgerDayKey(now = Date.now()) {
|
|
44
|
+
const d = new Date(now);
|
|
45
|
+
const p = (n) => String(n).padStart(2, "0");
|
|
46
|
+
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}`;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function clampDiscount(value) {
|
|
50
|
+
if (!Number.isFinite(value)) return DEFAULT_CACHE_DISCOUNT;
|
|
51
|
+
return Math.min(1, Math.max(0, value));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function count(value) {
|
|
55
|
+
return Number.isFinite(value) && value > 0 ? Math.trunc(value) : 0;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function emptyBucket() {
|
|
59
|
+
return {
|
|
60
|
+
sessions: 0,
|
|
61
|
+
requests: 0,
|
|
62
|
+
injectedChars: 0,
|
|
63
|
+
avoidedChars: 0,
|
|
64
|
+
rewriteCount: 0,
|
|
65
|
+
rewriteChars: 0,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function addBucket(target, delta) {
|
|
70
|
+
target.sessions += count(delta.sessions);
|
|
71
|
+
target.requests += count(delta.requests);
|
|
72
|
+
target.injectedChars += count(delta.injectedChars);
|
|
73
|
+
target.avoidedChars += count(delta.avoidedChars);
|
|
74
|
+
target.rewriteCount += count(delta.rewriteCount);
|
|
75
|
+
target.rewriteChars += count(delta.rewriteChars);
|
|
76
|
+
return target;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function deltaOf(session) {
|
|
80
|
+
const f = session.flushed;
|
|
81
|
+
return {
|
|
82
|
+
requests: session.requests - f.requests,
|
|
83
|
+
injectedChars: session.injectedChars - f.injectedChars,
|
|
84
|
+
avoidedChars: session.avoidedChars - f.avoidedChars,
|
|
85
|
+
rewriteCount: session.rewriteCount - f.rewriteCount,
|
|
86
|
+
rewriteChars: session.rewriteChars - f.rewriteChars,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function isEmptyDelta(delta) {
|
|
91
|
+
return (
|
|
92
|
+
delta.requests === 0 &&
|
|
93
|
+
delta.injectedChars === 0 &&
|
|
94
|
+
delta.avoidedChars === 0 &&
|
|
95
|
+
delta.rewriteCount === 0 &&
|
|
96
|
+
delta.rewriteChars === 0
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Finalize one bucket with the net-token arithmetic (pure). */
|
|
101
|
+
export function finalizeLedgerBucket(bucket, cacheDiscount = DEFAULT_CACHE_DISCOUNT) {
|
|
102
|
+
const discount = clampDiscount(cacheDiscount);
|
|
103
|
+
const injectedChars = count(bucket.injectedChars);
|
|
104
|
+
const avoidedChars = count(bucket.avoidedChars);
|
|
105
|
+
const rewriteChars = count(bucket.rewriteChars);
|
|
106
|
+
const netChars = avoidedChars - injectedChars - Math.round(rewriteChars * (1 - discount));
|
|
107
|
+
return {
|
|
108
|
+
sessions: count(bucket.sessions),
|
|
109
|
+
requests: count(bucket.requests),
|
|
110
|
+
injectedChars,
|
|
111
|
+
avoidedChars,
|
|
112
|
+
rewriteCount: count(bucket.rewriteCount),
|
|
113
|
+
rewriteChars,
|
|
114
|
+
injectedTokens: Math.ceil(injectedChars / CHARS_PER_TOKEN),
|
|
115
|
+
avoidedTokens: Math.ceil(avoidedChars / CHARS_PER_TOKEN),
|
|
116
|
+
netChars,
|
|
117
|
+
netTokens: Math.round(netChars / CHARS_PER_TOKEN),
|
|
118
|
+
rewritesCostChars: Math.round(rewriteChars * (1 - discount)),
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function readPersisted(file) {
|
|
123
|
+
try {
|
|
124
|
+
const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
125
|
+
if (parsed === null || typeof parsed !== "object") return null;
|
|
126
|
+
if (parsed.version !== 1) return null;
|
|
127
|
+
if (parsed.workspaces === null || typeof parsed.workspaces !== "object") return null;
|
|
128
|
+
return parsed;
|
|
129
|
+
} catch {
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Atomic-ish write: tmp + rename, so a crash never leaves a torn file. */
|
|
135
|
+
function writeJsonAtomic(file, payload) {
|
|
136
|
+
try {
|
|
137
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
138
|
+
const tmp = `${file}.${process.pid}.tmp`;
|
|
139
|
+
fs.writeFileSync(tmp, `${JSON.stringify(payload)}\n`, "utf8");
|
|
140
|
+
fs.renameSync(tmp, file);
|
|
141
|
+
return true;
|
|
142
|
+
} catch {
|
|
143
|
+
// best-effort accounting — never throw into the session pipeline
|
|
144
|
+
return false;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function contentChars(content) {
|
|
149
|
+
if (!Array.isArray(content)) return 0;
|
|
150
|
+
let chars = 0;
|
|
151
|
+
for (const block of content) {
|
|
152
|
+
if (block === null || typeof block !== "object") continue;
|
|
153
|
+
if (typeof block.text === "string") chars += [...block.text].length;
|
|
154
|
+
}
|
|
155
|
+
return chars;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Estimate the live prefix size a prune invalidates (in Unicode code points).
|
|
160
|
+
*
|
|
161
|
+
* Walks the session surface through the same accessors the host pruner uses
|
|
162
|
+
* (`surface.nodes` → `eventAt` → `deriveEventMessage`). Fail-contained: an
|
|
163
|
+
* unknown session shape yields 0, which makes the ledger conservative about
|
|
164
|
+
* claiming savings.
|
|
165
|
+
*/
|
|
166
|
+
export function estimatePrefixChars(session) {
|
|
167
|
+
try {
|
|
168
|
+
const seqs = session?.surface?.nodes;
|
|
169
|
+
if (seqs === void 0 || seqs === null) return 0;
|
|
170
|
+
if (typeof session.eventAt !== "function" || typeof session.deriveEventMessage !== "function") return 0;
|
|
171
|
+
let chars = 0;
|
|
172
|
+
for (const seq of [...seqs]) {
|
|
173
|
+
const event = session.eventAt(seq);
|
|
174
|
+
if (event === void 0 || event === null) continue;
|
|
175
|
+
const message = session.deriveEventMessage(event);
|
|
176
|
+
chars += contentChars(message?.content);
|
|
177
|
+
}
|
|
178
|
+
return chars;
|
|
179
|
+
} catch {
|
|
180
|
+
return 0;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Create the in-memory ledger + durable rollup writer.
|
|
186
|
+
*
|
|
187
|
+
* @param {object} [opts]
|
|
188
|
+
* @param {string} [opts.home] engram home (defaults to ~/.dsh/engram)
|
|
189
|
+
* @param {number} [opts.cacheDiscount] cached-prefix price share (0..1)
|
|
190
|
+
* @param {number} [opts.flushEvery] requests between durable flushes
|
|
191
|
+
* @param {boolean} [opts.persist] false = pure in-memory (tests, dry runs)
|
|
192
|
+
* @param {() => number} [opts.now] clock injection for tests
|
|
193
|
+
* @param {object} [opts.log] optional logger (warn only)
|
|
194
|
+
*/
|
|
195
|
+
export function createLedger({
|
|
196
|
+
home = path.join(os.homedir(), ".dsh", "engram"),
|
|
197
|
+
cacheDiscount = DEFAULT_CACHE_DISCOUNT,
|
|
198
|
+
flushEvery = DEFAULT_FLUSH_EVERY,
|
|
199
|
+
persist = true,
|
|
200
|
+
now = Date.now,
|
|
201
|
+
log = void 0,
|
|
202
|
+
} = {}) {
|
|
203
|
+
const file = ledgerFile(home);
|
|
204
|
+
const discount = clampDiscount(cacheDiscount);
|
|
205
|
+
const every = Number.isFinite(flushEvery) && flushEvery > 0 ? Math.trunc(flushEvery) : 0;
|
|
206
|
+
const stored = persist ? readPersisted(file) : null;
|
|
207
|
+
const state = stored ?? { version: 1, updatedAt: 0, workspaces: {} };
|
|
208
|
+
const sessions = new Map();
|
|
209
|
+
|
|
210
|
+
function ensure(sessionId, workspace) {
|
|
211
|
+
if (typeof sessionId !== "string" || sessionId.length === 0) return null;
|
|
212
|
+
let session = sessions.get(sessionId);
|
|
213
|
+
if (session === void 0) {
|
|
214
|
+
session = {
|
|
215
|
+
sessionId,
|
|
216
|
+
workspace: typeof workspace === "string" ? workspace : "",
|
|
217
|
+
indexChars: 0,
|
|
218
|
+
esrChars: 0,
|
|
219
|
+
jitChars: 0,
|
|
220
|
+
requests: 0,
|
|
221
|
+
injectedChars: 0,
|
|
222
|
+
avoidedChars: 0,
|
|
223
|
+
prunedChars: 0,
|
|
224
|
+
compactedChars: 0,
|
|
225
|
+
rewriteCount: 0,
|
|
226
|
+
rewriteChars: 0,
|
|
227
|
+
flushed: { requests: 0, injectedChars: 0, avoidedChars: 0, rewriteCount: 0, rewriteChars: 0 },
|
|
228
|
+
firstFlush: false,
|
|
229
|
+
};
|
|
230
|
+
sessions.set(sessionId, session);
|
|
231
|
+
} else if (typeof workspace === "string" && workspace.length > 0) {
|
|
232
|
+
session.workspace = workspace;
|
|
233
|
+
}
|
|
234
|
+
return session;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function bucketFor(workspace, day) {
|
|
238
|
+
const ws = typeof workspace === "string" && workspace.length > 0 ? workspace : "(unknown)";
|
|
239
|
+
const workspaces = state.workspaces;
|
|
240
|
+
if (workspaces[ws] === void 0 || workspaces[ws] === null || typeof workspaces[ws] !== "object") {
|
|
241
|
+
workspaces[ws] = {};
|
|
242
|
+
}
|
|
243
|
+
const days = workspaces[ws];
|
|
244
|
+
if (days[day] === void 0 || days[day] === null || typeof days[day] !== "object") {
|
|
245
|
+
days[day] = emptyBucket();
|
|
246
|
+
} else {
|
|
247
|
+
days[day] = { ...emptyBucket(), ...days[day] };
|
|
248
|
+
}
|
|
249
|
+
return days[day];
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function flushSession(session) {
|
|
253
|
+
if (session === null || session === void 0) return false;
|
|
254
|
+
const delta = deltaOf(session);
|
|
255
|
+
if (isEmptyDelta(delta)) return false;
|
|
256
|
+
try {
|
|
257
|
+
const day = ledgerDayKey(now());
|
|
258
|
+
const bucket = bucketFor(session.workspace, day);
|
|
259
|
+
if (session.firstFlush === false) {
|
|
260
|
+
bucket.sessions += 1;
|
|
261
|
+
session.firstFlush = true;
|
|
262
|
+
}
|
|
263
|
+
addBucket(bucket, delta);
|
|
264
|
+
session.flushed = {
|
|
265
|
+
requests: session.requests,
|
|
266
|
+
injectedChars: session.injectedChars,
|
|
267
|
+
avoidedChars: session.avoidedChars,
|
|
268
|
+
rewriteCount: session.rewriteCount,
|
|
269
|
+
rewriteChars: session.rewriteChars,
|
|
270
|
+
};
|
|
271
|
+
state.updatedAt = now();
|
|
272
|
+
if (persist) writeJsonAtomic(file, state);
|
|
273
|
+
return true;
|
|
274
|
+
} catch (error) {
|
|
275
|
+
log?.warn?.(`engram ledger flush failed: ${String(error)}`);
|
|
276
|
+
return false;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
function flush() {
|
|
281
|
+
for (const session of sessions.values()) flushSession(session);
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
return {
|
|
285
|
+
/** Record the latest injected block size for one component. */
|
|
286
|
+
noteInjection(sessionId, workspace, component, chars) {
|
|
287
|
+
try {
|
|
288
|
+
const session = ensure(sessionId, workspace);
|
|
289
|
+
if (session === null) return;
|
|
290
|
+
const size = count(chars);
|
|
291
|
+
if (component === "esr") session.esrChars = size;
|
|
292
|
+
else if (component === "jit") session.jitChars = size;
|
|
293
|
+
else session.indexChars = size;
|
|
294
|
+
} catch (error) {
|
|
295
|
+
log?.warn?.(`engram ledger noteInjection failed: ${String(error)}`);
|
|
296
|
+
}
|
|
297
|
+
},
|
|
298
|
+
|
|
299
|
+
/** One model request completed: charge injected chars, credit savings. */
|
|
300
|
+
tick(sessionId, workspace) {
|
|
301
|
+
try {
|
|
302
|
+
const session = ensure(sessionId, workspace);
|
|
303
|
+
if (session === null) return;
|
|
304
|
+
session.requests += 1;
|
|
305
|
+
session.injectedChars += session.indexChars + session.esrChars + session.jitChars;
|
|
306
|
+
session.avoidedChars += session.prunedChars + session.compactedChars;
|
|
307
|
+
if (every > 0 && session.requests % every === 0) flushSession(session);
|
|
308
|
+
} catch (error) {
|
|
309
|
+
log?.warn?.(`engram ledger tick failed: ${String(error)}`);
|
|
310
|
+
}
|
|
311
|
+
},
|
|
312
|
+
|
|
313
|
+
/** A prune fired: its removed chars stop replaying from the next request. */
|
|
314
|
+
notePrune(sessionId, workspace, { charsRemoved = 0, rewriteChars = 0 } = {}) {
|
|
315
|
+
try {
|
|
316
|
+
const session = ensure(sessionId, workspace);
|
|
317
|
+
if (session === null) return;
|
|
318
|
+
session.prunedChars += count(charsRemoved);
|
|
319
|
+
session.rewriteCount += 1;
|
|
320
|
+
session.rewriteChars += count(rewriteChars);
|
|
321
|
+
flushSession(session);
|
|
322
|
+
} catch (error) {
|
|
323
|
+
log?.warn?.(`engram ledger notePrune failed: ${String(error)}`);
|
|
324
|
+
}
|
|
325
|
+
},
|
|
326
|
+
|
|
327
|
+
/** A compaction replaced history: the saved chars stop replaying too. */
|
|
328
|
+
noteCompaction(sessionId, workspace, { charsSaved = 0 } = {}) {
|
|
329
|
+
try {
|
|
330
|
+
const session = ensure(sessionId, workspace);
|
|
331
|
+
if (session === null) return;
|
|
332
|
+
session.compactedChars += count(charsSaved);
|
|
333
|
+
} catch (error) {
|
|
334
|
+
log?.warn?.(`engram ledger noteCompaction failed: ${String(error)}`);
|
|
335
|
+
}
|
|
336
|
+
},
|
|
337
|
+
|
|
338
|
+
/**
|
|
339
|
+
* A prefix rewrite that is not a prune (e.g. a JIT push appended to the
|
|
340
|
+
* prompt): charge the full prefix invalidation, credit nothing.
|
|
341
|
+
*/
|
|
342
|
+
noteRewrite(sessionId, workspace, { chars = 0 } = {}) {
|
|
343
|
+
try {
|
|
344
|
+
const session = ensure(sessionId, workspace);
|
|
345
|
+
if (session === null) return;
|
|
346
|
+
session.rewriteCount += 1;
|
|
347
|
+
session.rewriteChars += count(chars);
|
|
348
|
+
flushSession(session);
|
|
349
|
+
} catch (error) {
|
|
350
|
+
log?.warn?.(`engram ledger noteRewrite failed: ${String(error)}`);
|
|
351
|
+
}
|
|
352
|
+
},
|
|
353
|
+
|
|
354
|
+
/** Session ended: fold the tail into the durable rollup and forget it. */
|
|
355
|
+
dispose(sessionId) {
|
|
356
|
+
try {
|
|
357
|
+
const session = sessions.get(sessionId);
|
|
358
|
+
if (session === void 0) return;
|
|
359
|
+
flushSession(session);
|
|
360
|
+
sessions.delete(sessionId);
|
|
361
|
+
} catch (error) {
|
|
362
|
+
log?.warn?.(`engram ledger dispose failed: ${String(error)}`);
|
|
363
|
+
}
|
|
364
|
+
},
|
|
365
|
+
|
|
366
|
+
flush,
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* Read the ledger: persisted rollup + unflushed live deltas. Pure (never
|
|
370
|
+
* mutates state), so /stats can call it on every request.
|
|
371
|
+
*/
|
|
372
|
+
summary() {
|
|
373
|
+
const byWorkspace = {};
|
|
374
|
+
const byDay = {};
|
|
375
|
+
|
|
376
|
+
// persisted floor
|
|
377
|
+
for (const [ws, days] of Object.entries(state.workspaces ?? {})) {
|
|
378
|
+
if (days === null || typeof days !== "object") continue;
|
|
379
|
+
const wsBucket = (byWorkspace[ws] ??= emptyBucket());
|
|
380
|
+
for (const [day, bucket] of Object.entries(days)) {
|
|
381
|
+
if (bucket === null || typeof bucket !== "object") continue;
|
|
382
|
+
addBucket(wsBucket, bucket);
|
|
383
|
+
const dayBucket = (byDay[day] ??= emptyBucket());
|
|
384
|
+
addBucket(dayBucket, bucket);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
// live, not-yet-flushed tail
|
|
389
|
+
const today = ledgerDayKey(now());
|
|
390
|
+
for (const session of sessions.values()) {
|
|
391
|
+
const delta = deltaOf(session);
|
|
392
|
+
if (isEmptyDelta(delta)) continue;
|
|
393
|
+
const ws = session.workspace.length > 0 ? session.workspace : "(unknown)";
|
|
394
|
+
const wsBucket = (byWorkspace[ws] ??= emptyBucket());
|
|
395
|
+
if (session.firstFlush === false) wsBucket.sessions += 1;
|
|
396
|
+
addBucket(wsBucket, delta);
|
|
397
|
+
const dayBucket = (byDay[today] ??= emptyBucket());
|
|
398
|
+
if (session.firstFlush === false) dayBucket.sessions += 1;
|
|
399
|
+
addBucket(dayBucket, delta);
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
const totals = emptyBucket();
|
|
403
|
+
for (const bucket of Object.values(byWorkspace)) addBucket(totals, bucket);
|
|
404
|
+
|
|
405
|
+
const finalizedWorkspaces = {};
|
|
406
|
+
for (const [ws, bucket] of Object.entries(byWorkspace)) {
|
|
407
|
+
finalizedWorkspaces[ws] = finalizeLedgerBucket(bucket, discount);
|
|
408
|
+
}
|
|
409
|
+
const finalizedDays = Object.entries(byDay)
|
|
410
|
+
.map(([day, bucket]) => ({ day, ...finalizeLedgerBucket(bucket, discount) }))
|
|
411
|
+
.sort((a, b) => (a.day < b.day ? 1 : a.day > b.day ? -1 : 0));
|
|
412
|
+
|
|
413
|
+
return {
|
|
414
|
+
version: 1,
|
|
415
|
+
updatedAt: state.updatedAt ?? 0,
|
|
416
|
+
charsPerToken: CHARS_PER_TOKEN,
|
|
417
|
+
cacheDiscount: discount,
|
|
418
|
+
openSessions: sessions.size,
|
|
419
|
+
totals: finalizeLedgerBucket(totals, discount),
|
|
420
|
+
byWorkspace: finalizedWorkspaces,
|
|
421
|
+
byDay: finalizedDays,
|
|
422
|
+
};
|
|
423
|
+
},
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/** Read the durable ledger without creating a live instance (GUI / tests). */
|
|
428
|
+
export function readLedger({ home, cacheDiscount = DEFAULT_CACHE_DISCOUNT } = {}) {
|
|
429
|
+
const stored = readPersisted(ledgerFile(home));
|
|
430
|
+
const workspaces = {};
|
|
431
|
+
const days = {};
|
|
432
|
+
if (stored !== null) {
|
|
433
|
+
for (const [ws, wsDays] of Object.entries(stored.workspaces)) {
|
|
434
|
+
if (wsDays === null || typeof wsDays !== "object") continue;
|
|
435
|
+
const wsBucket = (workspaces[ws] ??= emptyBucket());
|
|
436
|
+
for (const [day, bucket] of Object.entries(wsDays)) {
|
|
437
|
+
if (bucket === null || typeof bucket !== "object") continue;
|
|
438
|
+
addBucket(wsBucket, bucket);
|
|
439
|
+
addBucket((days[day] ??= emptyBucket()), bucket);
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
const totals = emptyBucket();
|
|
444
|
+
for (const bucket of Object.values(workspaces)) addBucket(totals, bucket);
|
|
445
|
+
return {
|
|
446
|
+
version: 1,
|
|
447
|
+
updatedAt: stored?.updatedAt ?? 0,
|
|
448
|
+
charsPerToken: CHARS_PER_TOKEN,
|
|
449
|
+
cacheDiscount: clampDiscount(cacheDiscount),
|
|
450
|
+
totals: finalizeLedgerBucket(totals, cacheDiscount),
|
|
451
|
+
byWorkspace: Object.fromEntries(
|
|
452
|
+
Object.entries(workspaces).map(([ws, bucket]) => [ws, finalizeLedgerBucket(bucket, cacheDiscount)]),
|
|
453
|
+
),
|
|
454
|
+
byDay: Object.entries(days)
|
|
455
|
+
.map(([day, bucket]) => ({ day, ...finalizeLedgerBucket(bucket, cacheDiscount) }))
|
|
456
|
+
.sort((a, b) => (a.day < b.day ? 1 : a.day > b.day ? -1 : 0)),
|
|
457
|
+
};
|
|
458
|
+
}
|
package/lib/settings.js
CHANGED
|
@@ -43,6 +43,7 @@ export const SETTINGS_KEYS = [
|
|
|
43
43
|
"autoCapture",
|
|
44
44
|
"autoCapturePreferences",
|
|
45
45
|
"boundaryPrune",
|
|
46
|
+
"boundaryPruneAdaptive",
|
|
46
47
|
"boundaryPruneThresholdChars",
|
|
47
48
|
"boundaryPruneIdleCadence",
|
|
48
49
|
"resultPack",
|
|
@@ -66,6 +67,12 @@ export const SETTINGS_KEYS = [
|
|
|
66
67
|
"autoSupersede",
|
|
67
68
|
"autoSinkTodosOnEnd",
|
|
68
69
|
"autoWebCompaction",
|
|
70
|
+
"ledgerEnabled",
|
|
71
|
+
"ledgerCacheDiscount",
|
|
72
|
+
"jitEnabled",
|
|
73
|
+
"episodeEnabled",
|
|
74
|
+
"resumeBriefEnabled",
|
|
75
|
+
"entityCardsEnabled",
|
|
69
76
|
"trustedHosts",
|
|
70
77
|
];
|
|
71
78
|
|
|
@@ -83,10 +90,11 @@ export function makeSettingsSchema(defaults) {
|
|
|
83
90
|
return z.object({
|
|
84
91
|
autoCapture: z.boolean().default(defaults.autoCapture),
|
|
85
92
|
autoCapturePreferences: z.boolean().default(defaults.autoCapturePreferences).description("用户纠正/偏好学习(零 LLM):检测用户消息里的显式 prefer/avoid/switch-to 句式,自动沉淀为 preference 记忆并置顶 [ENGRAM] 索引(correx/PAHF 式偏好提取;关掉=只靠手动 engram_store)"),
|
|
86
|
-
boundaryPrune: z.boolean().default(defaults.boundaryPrune).description("ResultPack L1(BoundaryPrune 边界剪枝):在 todo 完成推进 / goal 终态这些子任务边界,主动调用 DSH 的确定性 toolResultPruner 剪掉超阈值旧工具结果(SoL-Pi Online Context Compact
|
|
93
|
+
boundaryPrune: z.boolean().default(defaults.boundaryPrune).description("ResultPack L1(BoundaryPrune 边界剪枝):在 todo 完成推进 / goal 终态这些子任务边界,主动调用 DSH 的确定性 toolResultPruner 剪掉超阈值旧工具结果(SoL-Pi Online Context Compact;默认开,false=关)"),
|
|
94
|
+
boundaryPruneAdaptive: z.boolean().default(defaults.boundaryPruneAdaptive).description("剪枝自适应(账本否决):当 Context Ledger 显示该 workspace 在真实剪枝后净收益为负(≥3 次剪枝)时自动停止继续剪;证据只能关、不会自行开;关掉=只受偿还门约束"),
|
|
87
95
|
boundaryPruneThresholdChars: z.natural().default(0).description("边界剪枝阈值覆盖(字符):宿主 pruner 固定 8192;设更低(如 2048)可更早开剪——L0 仿真显示 T2048 净收益最高(+61.6Mc),前提是 L2 归档已开启可找回被剪中段;0=用宿主 8192"),
|
|
88
96
|
boundaryPruneIdleCadence: z.natural().default(0).description("空闲步数边界(请求计数):最多每 N 个请求触发一次剪枝,覆盖无 todo/goal 的会话(约 62% 会话永不触发事件边界);仍受偿还门约束;0=仅事件边界"),
|
|
89
|
-
resultPack: z.boolean().default(defaults.resultPack).description("ResultPack L2(结果归档召回):BoundaryPrune 剪掉的中段脱敏归档到本地,剪枝节点尾部追加 handle,模型可用 result_recall(callId) 按页召回(完整 ObservationPack
|
|
97
|
+
resultPack: z.boolean().default(defaults.resultPack).description("ResultPack L2(结果归档召回):BoundaryPrune 剪掉的中段脱敏归档到本地,剪枝节点尾部追加 handle,模型可用 result_recall(callId) 按页召回(完整 ObservationPack;默认开,保证默认剪枝无损;需与 boundaryPrune 一起开)"),
|
|
90
98
|
sessionSearch: z.boolean().default(defaults.sessionSearch),
|
|
91
99
|
recallScope: z.union(["workspace", "global"]).default(defaults.recallScope ?? "workspace").description("召回范围:workspace=严格按当前工作区隔离(默认、token 纪律);global=本工作区命中弱/空时,从其它工作区池按 recency/origin 加权召回并标 [W:<workspace>] 来源。开启会打破跨工作区隔离,请按需选用"),
|
|
92
100
|
autoRecallOnStart: z.boolean().default(defaults.autoRecallOnStart).description("新会话开始时,宿主按首条消息对工作区做确定性召回,把命中记忆直接注入 [RECALL] 块(无需模型自己想起调 engram_recall;关掉=回到纯拉取)"),
|
|
@@ -101,12 +109,18 @@ export function makeSettingsSchema(defaults) {
|
|
|
101
109
|
maxMemoriesPerWorkspace: z.natural().max(10000).default(defaults.maxMemoriesPerWorkspace),
|
|
102
110
|
gcEnabled: z.boolean().default(defaults.gcEnabled),
|
|
103
111
|
gcStableRetentionDays: z.natural().max(3650).default(defaults.gcStableRetentionDays),
|
|
104
|
-
gcReplacesCompaction: z.boolean().default(defaults.gcReplacesCompaction),
|
|
112
|
+
gcReplacesCompaction: z.boolean().default(defaults.gcReplacesCompaction).description("Context GC 接管 DSH 自动 compact(指针摘要替代有损 LLM 摘要);默认关=保留宿主 summarizer。改写宿主预设是最高维护风险,需显式开启"),
|
|
105
113
|
gcNarrative: z.boolean().default(defaults.gcNarrative),
|
|
106
114
|
verifyArtifact: z.boolean().default(defaults.verifyArtifact).description("ESR 闭环时校验 artifact 路径真实存在"),
|
|
107
115
|
autoSupersede: z.boolean().default(defaults.autoSupersede).description("实体锚定 + 替换式更新(改用/不再…)自动把旧记忆标为 superseded(默认关)"),
|
|
108
116
|
autoSinkTodosOnEnd: z.boolean().default(defaults.autoSinkTodosOnEnd).description("会话结束时把仍 pending 的 todo 自动沉淀为 ESR draft 任务(去重、受容量上限约束;关掉=会话结束即弃)"),
|
|
109
|
-
autoWebCompaction: z.boolean().default(defaults.autoWebCompaction).description("web 面启动时把全部 stock 布局的 agent 预设(shipped + 用户根)的 compaction 组自动换成 dsh-engram/compaction(带备份、幂等、不碰自定义/无 compaction
|
|
117
|
+
autoWebCompaction: z.boolean().default(defaults.autoWebCompaction).description("web 面启动时把全部 stock 布局的 agent 预设(shipped + 用户根)的 compaction 组自动换成 dsh-engram/compaction(带备份、幂等、不碰自定义/无 compaction 预设)。默认关(opt-in):不主动改写任何预设;已接管的预设保持 Context GC、需 npm run web-compaction:revert 还原,host 面不受影响"),
|
|
118
|
+
ledgerEnabled: z.boolean().default(defaults.ledgerEnabled).description("上下文账本:按会话统计注入成本、剪枝/压缩节省与前缀重写罚金,输出净 token(/stats.ledger 与概览);关掉=不记账、零写入"),
|
|
119
|
+
ledgerCacheDiscount: z.number().min(0).max(1).default(defaults.ledgerCacheDiscount).description("账本缓存折价(0..1):被剪枝前缀的重写按 (1−该系数) 计罚金;0.1 = 缓存命中只付 10%,重写付 90%(L0 保守假设)"),
|
|
120
|
+
jitEnabled: z.boolean().default(defaults.jitEnabled).description("JIT 推送:错误复发 / 文件焦点 / goal 完成时把已有记忆压成一行推进上下文(append-only、每会话最多 3 行、无命中沉默;每次推送按前缀重写计入账本)"),
|
|
121
|
+
episodeEnabled: z.boolean().default(defaults.episodeEnabled).description("会话 episode:会话结束时用零 LLM 规则生成一条紧凑纪要(文件/命令/失败→修复/决策/未闭环),不进 [ENGRAM] 索引,供 resume brief 与实体页使用"),
|
|
122
|
+
resumeBriefEnabled: z.boolean().default(defaults.resumeBriefEnabled).description("恢复简报:新会话启动时注入最近 7 天的最新 episode(≤240 字符,含 engram_detail 指针),让隔天继续的项目不用重新交代背景"),
|
|
123
|
+
entityCardsEnabled: z.boolean().default(defaults.entityCardsEnabled).description("实体卡:[ENGRAM] 索引为活跃实体(有锚定记忆或未闭环任务)渲染一行「当前真相 + 打开任务 + engram_detail(entity=…) 指针」,替代平铺记忆行;最多 2 张"),
|
|
110
124
|
// Security knob, not rendered in the card: hostnames allowed past the
|
|
111
125
|
// loopback fence for /api/dsh-engram (operator-authorized tunnel hosts).
|
|
112
126
|
trustedHosts: z.array(z.string()).default([]),
|
package/lib/store.js
CHANGED
|
@@ -22,6 +22,7 @@ import path from "node:path";
|
|
|
22
22
|
import { defineDomain, domainTable } from "@deepseek-ai/dsh-storage-domain";
|
|
23
23
|
import { hashText, byRecency, uuid, now, truncate as truncateText, bm25Rank, tokenize as tokenizeLocal, dedupeBySession } from "./util.js";
|
|
24
24
|
import { deriveObservations } from "./obs.js";
|
|
25
|
+
import { makeFileAnchor } from "./anchor.js";
|
|
25
26
|
import { redactText } from "./redact.js";
|
|
26
27
|
|
|
27
28
|
export const DOMAIN_VERSION = 1;
|
|
@@ -36,8 +37,21 @@ const MEMORY_KINDS = [
|
|
|
36
37
|
"insight",
|
|
37
38
|
"handoff",
|
|
38
39
|
"task",
|
|
40
|
+
"episode",
|
|
39
41
|
];
|
|
40
42
|
|
|
43
|
+
const AnchorSchema = z.object({
|
|
44
|
+
/** Absolute file path the claim was written against. */
|
|
45
|
+
path: z.string().min(1),
|
|
46
|
+
/** Stat identity at write time — cheap revalidation (no read). */
|
|
47
|
+
mtimeMs: z.number().nonnegative(),
|
|
48
|
+
size: z.number().int().nonnegative(),
|
|
49
|
+
/** sha256[0..32] for files under the hash cap; null for large files. */
|
|
50
|
+
hash: z.string().nullable().default(null),
|
|
51
|
+
/** When the anchor was captured (epoch ms). */
|
|
52
|
+
at: z.number().int().nonnegative(),
|
|
53
|
+
});
|
|
54
|
+
|
|
41
55
|
const MemorySchema = z.object({
|
|
42
56
|
/** Stable id (table key). */
|
|
43
57
|
id: z.string().min(1),
|
|
@@ -61,6 +75,8 @@ const MemorySchema = z.object({
|
|
|
61
75
|
lastAccessAt: z.number().int().nonnegative().nullable().default(null),
|
|
62
76
|
/** Optional file anchor (absolute path). Sharpens error revival and recall display. */
|
|
63
77
|
filePath: z.string().nullable().default(null),
|
|
78
|
+
/** Revalidatable stat/hash identity of `filePath` at write time (v0.7-C). */
|
|
79
|
+
anchor: AnchorSchema.nullable().default(null),
|
|
64
80
|
/** 0..1 static signal; index threshold is minIndexSignal. */
|
|
65
81
|
signal: z.number().min(0).max(1).default(0.5),
|
|
66
82
|
status: z.enum(["active", "archived"]).default("active"),
|
|
@@ -448,6 +464,12 @@ export async function openEngramDomain(facility) {
|
|
|
448
464
|
if ([...text].length > config.maxMemoryChars) {
|
|
449
465
|
throw new EngramError("ENGRAM_CAP_EXCEEDED", `memory text is ${[...text].length} characters, over the cap of ${config.maxMemoryChars}`);
|
|
450
466
|
}
|
|
467
|
+
const kind = entry.kind ?? "fact";
|
|
468
|
+
const anchor = entry.anchor !== void 0
|
|
469
|
+
? entry.anchor
|
|
470
|
+
: (typeof entry.filePath === "string" && entry.filePath.length > 0
|
|
471
|
+
? makeFileAnchor(entry.filePath, { baseDir: entry.workspace, now: now() })
|
|
472
|
+
: null);
|
|
451
473
|
const all = allMemories(entry.workspace);
|
|
452
474
|
let supersedes = validateRelationTarget(entry.workspace, entry.supersedes ?? null, "supersedes", all);
|
|
453
475
|
const contradicts = validateRelationTarget(entry.workspace, entry.contradicts ?? null, "contradicts", all);
|
|
@@ -456,7 +478,12 @@ export async function openEngramDomain(facility) {
|
|
|
456
478
|
// Repeat of the exact same failure is itself a signal: refresh recency
|
|
457
479
|
// AND climb the hit counter (errors only) so the [ENGRAM] index can
|
|
458
480
|
// eventually surface it via promoteHits.
|
|
459
|
-
const next = {
|
|
481
|
+
const next = {
|
|
482
|
+
...dup,
|
|
483
|
+
updatedAt: now(),
|
|
484
|
+
...(dup.kind === "error" ? { hits: (dup.hits ?? 0) + 1 } : {}),
|
|
485
|
+
...(anchor !== null ? { anchor, filePath: dup.filePath ?? entry.filePath ?? null } : {}),
|
|
486
|
+
};
|
|
460
487
|
await memories.put(dup.id, next);
|
|
461
488
|
return { stored: next, duplicated: true, id: dup.id, revived: dup.kind === "error" };
|
|
462
489
|
}
|
|
@@ -472,7 +499,13 @@ export async function openEngramDomain(facility) {
|
|
|
472
499
|
prev.filePath !== null && entry.filePath !== null && prev.filePath === entry.filePath;
|
|
473
500
|
if (!samePath && ratio < REPEAT_OVERLAP) continue;
|
|
474
501
|
if (samePath && ratio < SAME_PATH_OVERLAP) continue;
|
|
475
|
-
const revived = {
|
|
502
|
+
const revived = {
|
|
503
|
+
...prev,
|
|
504
|
+
updatedAt: now(),
|
|
505
|
+
hits: (prev.hits ?? 0) + 1,
|
|
506
|
+
filePath: prev.filePath ?? entry.filePath ?? null,
|
|
507
|
+
...(anchor !== null ? { anchor } : {}),
|
|
508
|
+
};
|
|
476
509
|
await memories.put(prev.id, revived);
|
|
477
510
|
return { stored: revived, duplicated: true, id: prev.id, revived: true };
|
|
478
511
|
}
|
|
@@ -487,16 +520,25 @@ export async function openEngramDomain(facility) {
|
|
|
487
520
|
const auto = autoSupersedeCandidate(all, entry, text);
|
|
488
521
|
if (auto !== null) supersedes = auto.id;
|
|
489
522
|
}
|
|
523
|
+
// Layered TTL (v0.7-C): user rules and proven procedures are permanent;
|
|
524
|
+
// session episodes are short-lived; errors age out faster than facts.
|
|
525
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
526
|
+
const ttlDays = (() => {
|
|
527
|
+
if (kind === "preference" || kind === "procedure") return 0;
|
|
528
|
+
if (kind === "episode") return config.episodeTtlDays ?? 14;
|
|
529
|
+
if (kind === "error") return config.errorTtlDays ?? 60;
|
|
530
|
+
return config.expireDays ?? 180;
|
|
531
|
+
})();
|
|
490
532
|
const expiresAt =
|
|
491
533
|
entry.expiresAt !== void 0 && entry.expiresAt !== null
|
|
492
534
|
? entry.expiresAt
|
|
493
|
-
:
|
|
494
|
-
? now() +
|
|
535
|
+
: ttlDays > 0
|
|
536
|
+
? now() + ttlDays * DAY_MS
|
|
495
537
|
: null;
|
|
496
538
|
const stored = {
|
|
497
539
|
id: uuid(),
|
|
498
540
|
workspace: entry.workspace,
|
|
499
|
-
kind
|
|
541
|
+
kind,
|
|
500
542
|
text,
|
|
501
543
|
tags: entry.tags ?? [],
|
|
502
544
|
entity: entry.entity ?? null,
|
|
@@ -507,6 +549,7 @@ export async function openEngramDomain(facility) {
|
|
|
507
549
|
hits: 0,
|
|
508
550
|
lastAccessAt: null,
|
|
509
551
|
filePath: entry.filePath ?? null,
|
|
552
|
+
anchor: anchor ?? null,
|
|
510
553
|
signal: typeof entry.signal === "number" ? Math.max(0, Math.min(1, entry.signal)) : 0.5,
|
|
511
554
|
status: "active",
|
|
512
555
|
supersedes,
|
package/lib/tools.js
CHANGED
|
@@ -19,6 +19,7 @@ import { z } from "zod";
|
|
|
19
19
|
import { basename } from "node:path";
|
|
20
20
|
import { EngramError, hasCode } from "./store.js";
|
|
21
21
|
import { buildEsrStatusView } from "./index-block.js";
|
|
22
|
+
import { buildEntityPage, renderEntityPage } from "./entity-page.js";
|
|
22
23
|
import { truncate, shortId, slugId, fmtDate, escapeLt, workspaceKey as wk, uuid } from "./util.js";
|
|
23
24
|
import { hasCJK, searchSessionsCJK } from "./session-cjk.js";
|
|
24
25
|
|
|
@@ -285,19 +286,40 @@ export function registerTools(ctx, service) {
|
|
|
285
286
|
// ── engram_detail ───────────────────────────────────────────────────
|
|
286
287
|
registerTool({
|
|
287
288
|
name: "engram_detail",
|
|
288
|
-
description: "Full record of one memory id
|
|
289
|
+
description: "Full record of one memory id, or an entity page (truth/timeline/open loops/conflicts/relations) when `entity` is given.",
|
|
289
290
|
parameters: {
|
|
290
291
|
type: "object",
|
|
291
292
|
properties: {
|
|
292
293
|
id: { type: "string", description: "Memory id (e.g. the #xxxxx marker in the [ENGRAM] index is the first 8 chars)" },
|
|
294
|
+
entity: { type: "string", description: "Entity id or name — returns the full page (truth / timeline / open loops / conflicts / relations)" },
|
|
293
295
|
},
|
|
294
|
-
required: ["id"],
|
|
295
296
|
},
|
|
296
297
|
output: TEXT_OUTPUT,
|
|
297
298
|
execute: async (raw, exec) => {
|
|
298
|
-
const args = parse(
|
|
299
|
+
const args = parse(
|
|
300
|
+
z.object({ id: z.string().optional(), entity: z.string().optional() }),
|
|
301
|
+
raw,
|
|
302
|
+
"engram_detail",
|
|
303
|
+
);
|
|
299
304
|
const caller = callerOf(exec);
|
|
300
305
|
const domain = await requireDomain(service);
|
|
306
|
+
if (typeof args.entity === "string" && args.entity.trim().length > 0) {
|
|
307
|
+
const page = buildEntityPage(domain, caller.cwd, args.entity);
|
|
308
|
+
const registered = (domain.listEntities?.(caller.cwd) ?? []).some(
|
|
309
|
+
(entity) => entity.id === page?.entity?.id || entity.name === page?.entity?.name,
|
|
310
|
+
);
|
|
311
|
+
if (page === null || (!registered && page.counts.memories === 0 && page.counts.openTasks === 0)) {
|
|
312
|
+
const names = (domain.listEntities?.(caller.cwd) ?? []).slice(0, 8).map((entity) => entity.name);
|
|
313
|
+
throw new EngramError(
|
|
314
|
+
"ENGRAM_NOT_FOUND",
|
|
315
|
+
`no entity "${args.entity}" in this workspace${names.length > 0 ? ` (known: ${names.join(", ")})` : ""}`,
|
|
316
|
+
);
|
|
317
|
+
}
|
|
318
|
+
return renderEntityPage(page);
|
|
319
|
+
}
|
|
320
|
+
if (typeof args.id !== "string" || args.id.length === 0) {
|
|
321
|
+
throw new EngramError("ENGRAM_INVALID_ARGS", "engram_detail requires `id` or `entity`");
|
|
322
|
+
}
|
|
301
323
|
const id = resolveId(domain, caller.cwd, args.id);
|
|
302
324
|
const m = domain.getMemory(caller.cwd, id);
|
|
303
325
|
if (m === void 0) throw new EngramError("ENGRAM_NOT_FOUND", `no active memory "${args.id}" in this workspace`);
|