dsh-engram 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -17
- package/README.zh.md +110 -15
- package/docs/PROPOSAL-resultpack.zh.md +7 -3
- package/docs/RELEASE-NOTES-v0.5.0.md +58 -0
- package/docs/RELEASE-NOTES-v0.5.1.md +54 -0
- package/lib/api.js +19 -6
- package/lib/boundary-prune.js +115 -14
- package/lib/client.js +253 -3
- package/lib/index-block.js +27 -5
- package/lib/index.js +181 -5
- package/lib/pack-telemetry.js +115 -0
- package/lib/preference.js +276 -0
- package/lib/result-pack.js +276 -0
- package/lib/session-cjk.js +2 -2
- package/lib/settings.js +115 -31
- package/lib/store.js +1 -0
- package/lib/tools.js +2 -1
- package/lib/usage.js +47 -4
- package/package.json +8 -7
- package/scripts/prepublish-check.mjs +7 -4
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-engram: user-correction / preference learning — zero-LLM.
|
|
3
|
+
*
|
|
4
|
+
* Trend-driven (correx / PAHF(Meta, arXiv 2602.16173) / Mem0 preference
|
|
5
|
+
* extraction): when the user corrects or states a preference to the agent,
|
|
6
|
+
* that lesson should persist — "the AI makes a mistake and gets corrected,
|
|
7
|
+
* the lesson survives". The pull-only model left this entirely to manual
|
|
8
|
+
* engram_store calls; this module closes the gap by watching `user/message`
|
|
9
|
+
* session events and turning deterministic correction/preference cues into
|
|
10
|
+
* `preference` memories — no model call, no embeddings, pure pattern rules.
|
|
11
|
+
*
|
|
12
|
+
* Signal design (avoid false positives):
|
|
13
|
+
* - Only strong, explicit cues are treated as preference statements
|
|
14
|
+
* (negation "不要用X/别再用X/avoid/never use", substitution "改用X/换成X/
|
|
15
|
+
* instead of", affirmation "prefer/please use/always use/应该用/统一用").
|
|
16
|
+
* - A one-shot operation instruction ("运行 npm test") carries no
|
|
17
|
+
* preference cue and is never captured.
|
|
18
|
+
* - Interrogatives are skipped — asking "是不是用X?" is not a statement.
|
|
19
|
+
* - The extracted clause is bounded (4..120 chars); empty/pointless clauses
|
|
20
|
+
* fall back to no capture.
|
|
21
|
+
* - A per-session cap and a recent-window dedup mirror the auto-capture
|
|
22
|
+
* discipline of lib/capture.js (the store layer re-dedups by hash too).
|
|
23
|
+
* - Captured text is <direction marker> + the extracted clause, so recall
|
|
24
|
+
* lines read as standing rules ("prefer", "avoid", "switch-to").
|
|
25
|
+
*
|
|
26
|
+
* All functions are pure and deterministic; `now`/weights are injectable for
|
|
27
|
+
* node:test (matching lib/capture.js and lib/rerank.js).
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import { workspaceKey as wk } from "./util.js";
|
|
31
|
+
|
|
32
|
+
/** Direction of the preference statement, used in the captured text. */
|
|
33
|
+
export const PREFERENCE_DIRECTION = {
|
|
34
|
+
prefer: "prefer", // affirmative: use / prefer / always use
|
|
35
|
+
avoid: "avoid", // negative: don't use / never / stop
|
|
36
|
+
switch: "switch-to", // substitution: use X instead of Y / 改用 X
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
const PREFERENCE_SIGNAL = 0.55;
|
|
40
|
+
/** Cap the per-session count of auto-captured preference memories. */
|
|
41
|
+
export const DEFAULT_PREFERENCE_PER_SESSION = 8;
|
|
42
|
+
|
|
43
|
+
/** Strip leading punctuation/whitespace before a clause. */
|
|
44
|
+
function trimLead(s) {
|
|
45
|
+
return s.replace(/^[\s,,。;;::、!!??'"“”‘’()()]+/, "").trim();
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Trim a captured clause to a bounded, single-line window. */
|
|
49
|
+
function boundClause(s) {
|
|
50
|
+
let t = String(s ?? "").replace(/\s*\n+\s*/g, " ").replace(/[。;;!!??]+$/, "").trim();
|
|
51
|
+
t = trimLead(t);
|
|
52
|
+
// trailing sentence particles / connective fillers that are noise in a rule clause
|
|
53
|
+
t = t.replace(/(?:的|了|吧|啊|呢|呀|哦)$/, "").trim();
|
|
54
|
+
if (t.length > 120) t = `${t.slice(0, 120)}…`;
|
|
55
|
+
return t;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Is this user message an interrogative (asking, not stating)? */
|
|
59
|
+
function isInterrogative(text) {
|
|
60
|
+
if (/[??]\s*$/.test(text)) return true;
|
|
61
|
+
return /(是不是|可不可以|能不能|是否|有没有|好不好|行不行|该不该|要不要)/.test(text);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Skip meta-talk about our own memory tools (not a real preference). */
|
|
65
|
+
function isToolMeta(text) {
|
|
66
|
+
return /(engram_|esr_|\[ENGRAM\]|\[ESR\]|\[RECALL\])/.test(text) && /(记|存|调|删|用|store|recall|remember|delete|remove)/.test(text);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** A single extractor: cue regex + how to derive the clause + direction. */
|
|
70
|
+
const EXTRACTORS = [
|
|
71
|
+
// ── Chinese: negation (avoidance) ────────────────────────────────────────
|
|
72
|
+
// 不要用X / 别用X / 别再用X / 不要再X / 请勿使用X / 避免用X / 不要选X…
|
|
73
|
+
{
|
|
74
|
+
re: /(?:不要|别再?|请勿|切勿|避免|禁止|严禁|尽量别)(?:再|去|要)?\s*(?:用|使用|采用|选|选择|用?到){0,2}\s*([^,。;;!!??\n]{2,80})/,
|
|
75
|
+
kind: PREFERENCE_DIRECTION.avoid,
|
|
76
|
+
},
|
|
77
|
+
// 不要再这样 / 别这么干 (clause after a bare negation)
|
|
78
|
+
{
|
|
79
|
+
re: /(?:不要再|别这样|别这么|不要这样|不要这么)([^,。;;!!??\n]{0,60})/,
|
|
80
|
+
kind: PREFERENCE_DIRECTION.avoid,
|
|
81
|
+
},
|
|
82
|
+
// 别/不要 bare + trailing object ("别 X" where X is a 2+ char target)
|
|
83
|
+
{
|
|
84
|
+
re: /(?:^|[\s,,;;])(?:别|不要)\s+([^,。;;!!??\n]{2,80})/,
|
|
85
|
+
kind: PREFERENCE_DIRECTION.avoid,
|
|
86
|
+
},
|
|
87
|
+
// ── Chinese: substitution (switch-to) ─────────────────────────────────────
|
|
88
|
+
// 改用X / 换成X / 用X而不是Y / 应该用X / 统一用X / 优先用X / 尽量用X
|
|
89
|
+
{
|
|
90
|
+
re: /(?:改用|换成|换用|切到|迁移到|改用为|建议用|优先用|统一用|尽量用|应该用|应当用|就得用|最好用|记得用|请用|就用|用用看)\s*([^,。;;!!??\n]{2,80})/,
|
|
91
|
+
kind: PREFERENCE_DIRECTION.switch,
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
re: /(?:应该|建议|可以|记得|统一|优先|最好|尽量)\s*都?\s*(?:用|选|采用?|使?用?到)\s*([^,。;;!!??\n]{2,80})/,
|
|
95
|
+
kind: PREFERENCE_DIRECTION.switch,
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
re: /(?:用|选|采用?)\s*([^,。;;!!??\n]{2,80})\s*(?:而|而不是|而非|别|不要)\s*(?:用|选|采用?)?\s*([^,。;;!!??\n]{1,40})/,
|
|
99
|
+
kind: PREFERENCE_DIRECTION.switch,
|
|
100
|
+
},
|
|
101
|
+
// ── Chinese: affirmation (prefer) ────────────────────────────────────────
|
|
102
|
+
// 喜欢用X / 习惯用X / 更倾向X / 偏好X
|
|
103
|
+
{
|
|
104
|
+
re: /(?:偏好|更倾向(?:于)?|喜欢用|习惯用|爱用|常用)\s*([^,。;;!!??\n]{2,80})/,
|
|
105
|
+
kind: PREFERENCE_DIRECTION.prefer,
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
re: /(?:以后|从现在起|之后|接下来|后面|下次)\s*(?:都|一律|统一)?\s*(?:用|选|采用?)\s*([^,。;;!!??\n]{2,80})/,
|
|
109
|
+
kind: PREFERENCE_DIRECTION.switch,
|
|
110
|
+
},
|
|
111
|
+
|
|
112
|
+
// ── English: negation (avoidance) ─────────────────────────────────────────
|
|
113
|
+
// don't use X / do not use X / never use X / avoid X / stop using X.
|
|
114
|
+
// The clause runs to the next sentence/beginning-of-comma boundary so a
|
|
115
|
+
// short target like "var" in "never use var, always use let" survives.
|
|
116
|
+
{
|
|
117
|
+
re: /\b(?:don'?t|do not|never|stop|avoid|please don'?t|do not ever)\s+(?:use|using|pick|choose|select|go with|use?ing)?\s*([A-Za-z0-9_./@-]+(?:\s+[A-Za-z0-9_./'()@-]{1,60}){0,5})/i,
|
|
118
|
+
kind: PREFERENCE_DIRECTION.avoid,
|
|
119
|
+
},
|
|
120
|
+
// X is not allowed / no more X
|
|
121
|
+
{
|
|
122
|
+
re: /\b(?:no more|not allowed|clear|drop)\s+([A-Za-z0-9_./@-]+(?:\s+[A-Za-z0-9_./'()@-]{1,60}){0,5})/i,
|
|
123
|
+
kind: PREFERENCE_DIRECTION.avoid,
|
|
124
|
+
},
|
|
125
|
+
// ── English: substitution (switch-to) ─────────────────────────────────────
|
|
126
|
+
// use X instead (of Y) / switch to X / go with X / move to X
|
|
127
|
+
{
|
|
128
|
+
re: /\b(?:use|using|switch to|switch over to|go with|move to|migrate to|default to|prefer to use)\s+([A-Za-z0-9_./@-]+(?:\s+[A-Za-z0-9_./'()@-]{1,60}){0,5})\s*(?:instead|rather|over|from now on)?/i,
|
|
129
|
+
kind: PREFERENCE_DIRECTION.switch,
|
|
130
|
+
},
|
|
131
|
+
// instead of / rather than (primary target elsewhere)
|
|
132
|
+
{
|
|
133
|
+
re: /\b(?:instead of|rather than|as opposed to)\s+([A-Za-z0-9_./@-]+(?:\s+[A-Za-z0-9_./'()@-]{1,60}){0,5})/i,
|
|
134
|
+
kind: PREFERENCE_DIRECTION.avoid,
|
|
135
|
+
},
|
|
136
|
+
// ── English: affirmation (prefer) ─────────────────────────────────────────
|
|
137
|
+
{
|
|
138
|
+
re: /\b(?:i prefer|prefer|i like|like using|usually use|always use|we use|please use)\s+([A-Za-z0-9_./@-]+(?:\s+[A-Za-z0-9_./'()@-]{1,60}){0,5})/i,
|
|
139
|
+
kind: PREFERENCE_DIRECTION.prefer,
|
|
140
|
+
},
|
|
141
|
+
];
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Extract a preference statement from user message text, or null when the
|
|
145
|
+
* message carries no explicit correction/preference cue. Pure + deterministic.
|
|
146
|
+
*
|
|
147
|
+
* @param {string} text user message text
|
|
148
|
+
* @returns {{ clause: string, direction: string, text: string, tags: string[] } | null}
|
|
149
|
+
*/
|
|
150
|
+
export function extractPreference(text) {
|
|
151
|
+
if (typeof text !== "string" || text.trim().length === 0) return null;
|
|
152
|
+
const src = text.trim();
|
|
153
|
+
if (src.length < 6 || src.length > 2000) return null;
|
|
154
|
+
if (isInterrogative(src)) return null;
|
|
155
|
+
if (isToolMeta(src)) return null;
|
|
156
|
+
|
|
157
|
+
for (const ex of EXTRACTORS) {
|
|
158
|
+
const m = src.match(ex.re);
|
|
159
|
+
if (!m) continue;
|
|
160
|
+
// A clause may be a short but meaningful target ("var"); 2 chars minimum.
|
|
161
|
+
let clause = boundClause(m[1] ?? "");
|
|
162
|
+
if (clause.length < 2 || clause.length > 120) continue;
|
|
163
|
+
const direction = ex.kind;
|
|
164
|
+
const marker = direction === PREFERENCE_DIRECTION.avoid ? "avoid" : direction === PREFERENCE_DIRECTION.switch ? "switch-to" : "prefer";
|
|
165
|
+
const lang = /[\u4e00-\u9fff]/.test(clause) ? "zh" : "en";
|
|
166
|
+
return {
|
|
167
|
+
clause,
|
|
168
|
+
direction,
|
|
169
|
+
text: `user ${marker}: ${clause}`,
|
|
170
|
+
tags: ["preference", "user", marker, lang, "auto-captured"],
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Install the `session/event` listener that captures user corrections and
|
|
178
|
+
* preferences as `preference` memories. Mirrors installGoalCapture's
|
|
179
|
+
* containment + lazy adapter pattern: `storeAdapter` is any
|
|
180
|
+
* { storeMemory(entry) -> Promise } face, resolved per event, with a
|
|
181
|
+
* per-session capture cap and a recent-window dedup. Returns the disposer.
|
|
182
|
+
*
|
|
183
|
+
* @param {object} ctx cordis scoped context (ctx.on available)
|
|
184
|
+
* @param {object} storeAdapter { storeMemory({workspace,kind,text,tags,signal,filePath,sessionId,seq}) -> Promise }
|
|
185
|
+
* @param {object} config { autoCapturePreferences, preferenceCapturePerSession }
|
|
186
|
+
* @param {object} [log]
|
|
187
|
+
* @param {object} [stats] optional mutable {total, preference} counters
|
|
188
|
+
*/
|
|
189
|
+
export function installPreferenceCapture(ctx, storeAdapter, config, log, stats) {
|
|
190
|
+
if (config?.autoCapturePreferences === false) {
|
|
191
|
+
// still install a no-op listener? No — return a no-op disposer so callers
|
|
192
|
+
// can always dispose without branching on config.
|
|
193
|
+
return () => {};
|
|
194
|
+
}
|
|
195
|
+
const cap = Number.isInteger(config?.preferenceCapturePerSession)
|
|
196
|
+
? config.preferenceCapturePerSession
|
|
197
|
+
: DEFAULT_PREFERENCE_PER_SESSION;
|
|
198
|
+
const perSession = new Map(); // sessionId -> count
|
|
199
|
+
const recent = new Set(); // `${workspace}|${text}` recent-window dedup
|
|
200
|
+
|
|
201
|
+
const onEvent = (session, event) => {
|
|
202
|
+
try {
|
|
203
|
+
if (event?.type !== "user/message") return;
|
|
204
|
+
// Only learn from the human user's own messages. The harness also emits
|
|
205
|
+
// user/message events for injected context (runtime-context, skill-catalog
|
|
206
|
+
// snapshots, etc.) — those are instructions to the model, not corrections
|
|
207
|
+
// the user is teaching us, and must never become preference memories.
|
|
208
|
+
if (event.data?.source?.kind !== "user") return;
|
|
209
|
+
const cwd = session?.header?.cwd;
|
|
210
|
+
const sessionId = session?.id;
|
|
211
|
+
if (typeof cwd !== "string" || cwd.length === 0) return;
|
|
212
|
+
if (typeof sessionId !== "string" || sessionId.length === 0) return;
|
|
213
|
+
|
|
214
|
+
const text = messageText(event.data);
|
|
215
|
+
if (text.length === 0) return;
|
|
216
|
+
|
|
217
|
+
const extracted = extractPreference(text);
|
|
218
|
+
if (extracted === null) return;
|
|
219
|
+
|
|
220
|
+
const used = perSession.get(sessionId) ?? 0;
|
|
221
|
+
if (used >= cap) return;
|
|
222
|
+
perSession.set(sessionId, used + 1);
|
|
223
|
+
|
|
224
|
+
const key = `${wk(cwd)}|${extracted.text}`;
|
|
225
|
+
if (recent.has(key)) return;
|
|
226
|
+
recent.add(key);
|
|
227
|
+
if (recent.size > 500) {
|
|
228
|
+
const first = recent.values().next().value;
|
|
229
|
+
recent.delete(first);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
if (stats) {
|
|
233
|
+
stats.total = (stats.total ?? 0) + 1;
|
|
234
|
+
stats.preference = (stats.preference ?? 0) + 1;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
storeAdapter
|
|
238
|
+
.storeMemory(
|
|
239
|
+
{
|
|
240
|
+
workspace: wk(cwd),
|
|
241
|
+
kind: "preference",
|
|
242
|
+
text: extracted.text,
|
|
243
|
+
tags: extracted.tags,
|
|
244
|
+
signal: PREFERENCE_SIGNAL,
|
|
245
|
+
filePath: null,
|
|
246
|
+
sessionId,
|
|
247
|
+
seq: Array.isArray(session.events) ? session.events.length : 0,
|
|
248
|
+
},
|
|
249
|
+
config,
|
|
250
|
+
)
|
|
251
|
+
.then(() => {})
|
|
252
|
+
.catch((error) => log?.warn?.(`engram preference capture failed: ${String(error)}`));
|
|
253
|
+
} catch (error) {
|
|
254
|
+
// fully contained — a capture bug must never disturb the session pipeline
|
|
255
|
+
log?.warn?.(`engram preference capture threw: ${String(error)}`);
|
|
256
|
+
}
|
|
257
|
+
};
|
|
258
|
+
return ctx.on("session/event", onEvent);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Extract plain text from a `user/message` event data payload. Mirrors the
|
|
263
|
+
* shape understood by index-block.firstUserMessageText: `content` is an
|
|
264
|
+
* array of blocks each with `.text`, or a single `.text` field. Pure.
|
|
265
|
+
*/
|
|
266
|
+
export function messageText(data) {
|
|
267
|
+
if (data === null || typeof data !== "object") return "";
|
|
268
|
+
const blocks = Array.isArray(data.content) ? data.content : [];
|
|
269
|
+
if (blocks.length > 0) {
|
|
270
|
+
return blocks
|
|
271
|
+
.map((b) => (b !== null && typeof b === "object" && typeof b.text === "string" ? b.text : ""))
|
|
272
|
+
.join(" ")
|
|
273
|
+
.trim();
|
|
274
|
+
}
|
|
275
|
+
return typeof data.text === "string" ? data.text.trim() : "";
|
|
276
|
+
}
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-engram ResultPack L2 (PROPOSAL-resultpack §L2): archive pruned tool-result
|
|
3
|
+
* middles + `result_recall` paged recall — the ObservationPack pattern SoL-Pi
|
|
4
|
+
* mechanism ③ (load archived, keep a handle + digest on the surface, page back
|
|
5
|
+
* on demand; its paired EdgeBench: bill −23.58%).
|
|
6
|
+
*
|
|
7
|
+
* L1 (lib/boundary-prune.js) prunes over-threshold tool results at todo/goal
|
|
8
|
+
* boundaries through DSH's own deterministic `toolResultPruner`. The L1 pruned
|
|
9
|
+
* middle was simply gone — the model could not reach it again. L2 makes the
|
|
10
|
+
* prune lossless:
|
|
11
|
+
*
|
|
12
|
+
* 1. When L1 fires, we read the ORIGINAL result from the session log
|
|
13
|
+
* (`session.eventAt(originalSeq)` + `deriveEventMessage`), extract the
|
|
14
|
+
* removed middle span, redact secrets (same rules as the memory write
|
|
15
|
+
* path), and archive it to a per-callId JSON file under
|
|
16
|
+
* `~/.dsh/engram/resultpack/`.
|
|
17
|
+
* 2. We append a tail handle to the pruned surface node via the host's own
|
|
18
|
+
* surface-replace protocol (append tool/result + surfaceOp replace,
|
|
19
|
+
* replay-safe): `[middle archived: result_recall(<callId>) 可取]`.
|
|
20
|
+
* The model decides whether to drill down — the same progressive
|
|
21
|
+
* disclosure philosophy as [ENGRAM] → engram_detail.
|
|
22
|
+
* 3. `result_recall(callId, offset?)` returns the archived middle in paged
|
|
23
|
+
* text blocks, so a very large result is never re-materialized whole.
|
|
24
|
+
*
|
|
25
|
+
* Zero-LLM, deterministic, fail-contained (never throws into the session
|
|
26
|
+
* pipeline); every write is atomic temp-file + rename. Retention is a simple
|
|
27
|
+
* mtime sweep (`sweepExpired`), wired to the same GC cadence as memory GC.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import fs from "node:fs";
|
|
31
|
+
import os from "node:os";
|
|
32
|
+
import path from "node:path";
|
|
33
|
+
|
|
34
|
+
import { redactText } from "./redact.js";
|
|
35
|
+
import { hashText, workspaceKey as wk, truncate } from "./util.js";
|
|
36
|
+
|
|
37
|
+
/** Default page budget for result_recall (chars per page). */
|
|
38
|
+
export const DEFAULT_PAGE_CHARS = 4000;
|
|
39
|
+
/** Default retention for archived results (mtime-based sweep). */
|
|
40
|
+
export const DEFAULT_RETENTION_DAYS = 7;
|
|
41
|
+
/** Handle line appended to the pruned surface node. */
|
|
42
|
+
const HANDLE_PREFIX = "[middle archived: result_recall(";
|
|
43
|
+
|
|
44
|
+
/** Archived-result directory under the engram home (~/.dsh/engram/resultpack). */
|
|
45
|
+
export function resultPackDir(home = path.join(os.homedir(), ".dsh", "engram")) {
|
|
46
|
+
return path.join(home, "resultpack");
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Stable, path-safe archive id derived from the tool callId. */
|
|
50
|
+
export function archiveId(callId) {
|
|
51
|
+
return `${hashText(String(callId))}`;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Read one archived result back (parsed), or undefined when absent/broken. */
|
|
55
|
+
export function readArchived(dir, callId) {
|
|
56
|
+
try {
|
|
57
|
+
const file = path.join(dir, `${archiveId(callId)}.json`);
|
|
58
|
+
if (!fs.existsSync(file)) return void 0;
|
|
59
|
+
return JSON.parse(fs.readFileSync(file, "utf8"));
|
|
60
|
+
} catch {
|
|
61
|
+
return void 0;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Extract the removed middle text from an original tool-result content and the
|
|
67
|
+
* host pruner budgets. Mirrors host arithmetic (code-point slicing across text
|
|
68
|
+
* blocks: everything between headChars and total-tailChars is the middle).
|
|
69
|
+
* @param blocks - original message content (blocks with `.type`/`.text`).
|
|
70
|
+
* @param headChars - pruner head budget.
|
|
71
|
+
* @param tailChars - pruner tail budget.
|
|
72
|
+
* @returns { removed: string, head: string, tail: string } or null when nothing removed.
|
|
73
|
+
*/
|
|
74
|
+
export function extractMiddle(blocks, headChars, tailChars) {
|
|
75
|
+
if (!Array.isArray(blocks)) return null;
|
|
76
|
+
const total = blocks.reduce((n, b) => (b && b.type === "text" ? n + Array.from(String(b.text)).length : n), 0);
|
|
77
|
+
const removedStart = headChars;
|
|
78
|
+
const removedEnd = total - tailChars;
|
|
79
|
+
if (total <= removedStart + tailChars || removedEnd <= removedStart) return null;
|
|
80
|
+
const removed = [];
|
|
81
|
+
const head = [];
|
|
82
|
+
const tail = [];
|
|
83
|
+
let consumed = 0;
|
|
84
|
+
for (const block of blocks) {
|
|
85
|
+
if (!block || block.type !== "text") continue;
|
|
86
|
+
const points = Array.from(String(block.text));
|
|
87
|
+
const blockStart = consumed;
|
|
88
|
+
const blockEnd = blockStart + points.length;
|
|
89
|
+
const headEnd = Math.min(points.length, Math.max(0, removedStart - blockStart));
|
|
90
|
+
const tailStart = Math.min(points.length, Math.max(0, removedEnd - blockStart));
|
|
91
|
+
head.push(points.slice(0, headEnd).join(""));
|
|
92
|
+
if (tailStart > headEnd) removed.push(points.slice(headEnd, tailStart).join(""));
|
|
93
|
+
tail.push(points.slice(tailStart).join(""));
|
|
94
|
+
consumed = blockEnd;
|
|
95
|
+
}
|
|
96
|
+
return {
|
|
97
|
+
removed: removed.join(""),
|
|
98
|
+
head: head.join(""),
|
|
99
|
+
tail: tail.join(""),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Archive the pruned middles from one L1 prune report, appending a tail handle
|
|
105
|
+
* on each pruned surface node. Returns the count archived. Never throws.
|
|
106
|
+
*
|
|
107
|
+
* @param {object} opts
|
|
108
|
+
* @param {string} opts.dir archive directory
|
|
109
|
+
* @param {object} opts.session the session being pruned (eventAt + deriveEventMessage)
|
|
110
|
+
* @param {object} opts.report the host pruner's return `{ pruned, charsRemoved }`
|
|
111
|
+
* @param {string} [opts.workspace] workspace key used on the archive metadata
|
|
112
|
+
* @param {object} [opts.log]
|
|
113
|
+
* @returns {number} archives written
|
|
114
|
+
*/
|
|
115
|
+
export function archivePrunedResults({ dir, session, report, workspace, log }) {
|
|
116
|
+
const pruned = report?.pruned;
|
|
117
|
+
if (!Array.isArray(pruned) || pruned.length === 0) return 0;
|
|
118
|
+
let archived = 0;
|
|
119
|
+
try {
|
|
120
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
121
|
+
} catch (error) {
|
|
122
|
+
log?.warn?.(`engram result-pack: archive dir unavailable: ${String(error)}`);
|
|
123
|
+
return 0;
|
|
124
|
+
}
|
|
125
|
+
for (const item of pruned) {
|
|
126
|
+
try {
|
|
127
|
+
const callId = item?.callId;
|
|
128
|
+
if (typeof callId !== "string" || callId.length === 0) continue;
|
|
129
|
+
const original = item.originalSeq !== void 0 ? session?.eventAt?.(item.originalSeq) : void 0;
|
|
130
|
+
const message = original ? session?.deriveEventMessage?.(original) : void 0;
|
|
131
|
+
const blocks = message?.content;
|
|
132
|
+
// Fall back to a default budget when the report lacks head/tail (the host
|
|
133
|
+
// report carries charsBefore/After only; budgets default to host DEFAULTS).
|
|
134
|
+
const mine = extractMiddle(blocks, item.headChars ?? 4096, item.tailChars ?? 1024);
|
|
135
|
+
const removed = mine?.removed ?? "";
|
|
136
|
+
if (removed.length === 0) continue;
|
|
137
|
+
const record = {
|
|
138
|
+
v: 1,
|
|
139
|
+
callId,
|
|
140
|
+
workspace: workspace ?? wk(message?.source?.workspace ?? ""),
|
|
141
|
+
sessionId: item.sessionId ?? session?.id ?? null,
|
|
142
|
+
originalSeq: item.originalSeq ?? null,
|
|
143
|
+
createdAt: Date.now(),
|
|
144
|
+
headChars: item.headChars ?? 4096,
|
|
145
|
+
tailChars: item.tailChars ?? 1024,
|
|
146
|
+
charsBefore: item.charsBefore ?? removed.length,
|
|
147
|
+
charsAfter: item.charsAfter ?? (item.headChars ?? 4096) + (item.tailChars ?? 1024),
|
|
148
|
+
head: truncate(mine?.head ?? "", 2000),
|
|
149
|
+
tail: truncate(mine?.tail ?? "", 2000),
|
|
150
|
+
// Redacted before persistence — the memory path's guarantee holds here.
|
|
151
|
+
removed: redactText(removed),
|
|
152
|
+
};
|
|
153
|
+
const file = path.join(dir, `${archiveId(callId)}.json`);
|
|
154
|
+
const tmp = `${file}.${hashText(String(Date.now() + Math.random()))}.tmp`;
|
|
155
|
+
fs.writeFileSync(tmp, JSON.stringify(record), "utf8");
|
|
156
|
+
fs.renameSync(tmp, file);
|
|
157
|
+
archived += 1;
|
|
158
|
+
appendHandle({ session, callId, replacementSeq: item.replacementSeq, log });
|
|
159
|
+
} catch (error) {
|
|
160
|
+
log?.warn?.(`engram result-pack: archive failed: ${String(error)}`);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return archived;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Append a `[middle archived: result_recall(callId) 可取]` tail to one pruned
|
|
168
|
+
* surface node using the host's own surface-replace protocol (append a
|
|
169
|
+
* tool/result +++ surfaceOp replace; replay-safe, zero-LLM). Never throws.
|
|
170
|
+
*/
|
|
171
|
+
export function appendHandle({ session, callId, replacementSeq, log }) {
|
|
172
|
+
try {
|
|
173
|
+
if (!session || typeof session.append !== "function") return;
|
|
174
|
+
const seq = replacementSeq;
|
|
175
|
+
if (seq === void 0) return;
|
|
176
|
+
const node = session.eventAt?.(seq);
|
|
177
|
+
if (!node || node.type !== "tool/result") return;
|
|
178
|
+
const current = session.deriveEventMessage?.(node);
|
|
179
|
+
if (!current) return;
|
|
180
|
+
const handleText = `${HANDLE_PREFIX}${callId}) 可取]`;
|
|
181
|
+
const content = Array.isArray(current.content) ? [...current.content] : [];
|
|
182
|
+
// Avoid stacking duplicate handles on repeated prunes of the same node.
|
|
183
|
+
if (content.some((b) => b && typeof b.text === "string" && b.text.includes(HANDLE_PREFIX))) return;
|
|
184
|
+
content.push({ type: "text", text: handleText });
|
|
185
|
+
const message = { ...current, content };
|
|
186
|
+
session.append("tool/result", { ...node.data, message }, {
|
|
187
|
+
surfaceOp: { op: "replace", startSeq: seq, endSeq: seq },
|
|
188
|
+
sourceEventSeqs: [seq],
|
|
189
|
+
});
|
|
190
|
+
} catch (error) {
|
|
191
|
+
log?.warn?.(`engram result-pack: handle append failed: ${String(error)}`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Page out one archived middle. Pure + deterministic.
|
|
197
|
+
* @param {object} archived parsed archive record
|
|
198
|
+
* @param {number} [offset] zero-based page index
|
|
199
|
+
* @param {number} [pageChars] chars per page
|
|
200
|
+
* @returns {{ total, totalChars, offset, page, hasMore, text, callId }}
|
|
201
|
+
*/
|
|
202
|
+
export function pageArchived(archived, offset = 0, pageChars = DEFAULT_PAGE_CHARS) {
|
|
203
|
+
const totalChars = Array.from(String(archived?.removed ?? "")).length;
|
|
204
|
+
const total = Math.max(1, Math.ceil(totalChars / pageChars));
|
|
205
|
+
const idx = Number.isFinite(offset) && offset >= 0 ? Math.floor(offset) : 0;
|
|
206
|
+
const p = Math.min(idx, total - 1);
|
|
207
|
+
const chars = Array.from(String(archived?.removed ?? ""));
|
|
208
|
+
const text = chars.slice(p * pageChars, (p + 1) * pageChars).join("");
|
|
209
|
+
return {
|
|
210
|
+
callId: archived?.callId ?? null,
|
|
211
|
+
total,
|
|
212
|
+
totalChars,
|
|
213
|
+
offset: p,
|
|
214
|
+
page: p + 1,
|
|
215
|
+
hasMore: p + 1 < total,
|
|
216
|
+
text,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Sweep archives whose mtime is older than `retentionDays`. Returns deleted
|
|
222
|
+
* count. Never throws.
|
|
223
|
+
*/
|
|
224
|
+
export function sweepExpired(dir, retentionDays = DEFAULT_RETENTION_DAYS) {
|
|
225
|
+
let deleted = 0;
|
|
226
|
+
try {
|
|
227
|
+
if (!fs.existsSync(dir)) return 0;
|
|
228
|
+
const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
|
|
229
|
+
for (const name of fs.readdirSync(dir)) {
|
|
230
|
+
if (!name.endsWith(".json")) continue;
|
|
231
|
+
const file = path.join(dir, name);
|
|
232
|
+
try {
|
|
233
|
+
if (fs.statSync(file).mtimeMs < cutoff) {
|
|
234
|
+
fs.unlinkSync(file);
|
|
235
|
+
deleted += 1;
|
|
236
|
+
}
|
|
237
|
+
} catch { /* racing removal — ignore */ }
|
|
238
|
+
}
|
|
239
|
+
} catch { /* unreadable dir — ignore */ }
|
|
240
|
+
return deleted;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* The model-facing tool definition: `result_recall(callId, offset?)`.
|
|
245
|
+
* Independent of the memory domain (archives live on the filesystem). Reads
|
|
246
|
+
* are workspace-sandboxed by callerOf-style cwd resolution, but the archive is
|
|
247
|
+
* keyed by globally-unique callId, so we just return the archive when found.
|
|
248
|
+
*/
|
|
249
|
+
export function resultRecallTool(dir, pageChars = DEFAULT_PAGE_CHARS) {
|
|
250
|
+
return {
|
|
251
|
+
name: "result_recall",
|
|
252
|
+
description: "Retrieve an archived pruned tool-result middle by its callId (page 1 by default). The handle line inside a pruned result names the callId. Returns one page of the original middle text; paging is 0-based via offset.",
|
|
253
|
+
parameters: {
|
|
254
|
+
type: "object",
|
|
255
|
+
properties: {
|
|
256
|
+
callId: { type: "string", description: "callId from a `[middle archived: result_recall(<callId>) 可取]` handle (required)" },
|
|
257
|
+
offset: { type: "number", description: "Zero-based page index to return (default 0)", minimum: 0 },
|
|
258
|
+
},
|
|
259
|
+
required: ["callId"],
|
|
260
|
+
},
|
|
261
|
+
output: {
|
|
262
|
+
schema: { type: "string" },
|
|
263
|
+
render: (_args, value) => [{ type: "text", text: String(value) }],
|
|
264
|
+
},
|
|
265
|
+
execute: async (raw) => {
|
|
266
|
+
const callId = typeof raw?.callId === "string" ? raw.callId.trim() : "";
|
|
267
|
+
if (callId.length === 0) return "result_recall: callId is required";
|
|
268
|
+
const offset = Number.isFinite(raw?.offset) ? Math.floor(raw.offset) : 0;
|
|
269
|
+
const archived = readArchived(dir, callId);
|
|
270
|
+
if (archived === void 0) return `result_recall: no archived middle for callId ${callId} (expired or never archived)`;
|
|
271
|
+
const page = pageArchived(archived, offset, pageChars);
|
|
272
|
+
const header = `result_recall ${callId} · page ${page.page}/${page.total} · ${page.totalChars} chars archived${page.hasMore ? ` · offset=${page.offset + 1} 取下一页` : ""}\n\n`;
|
|
273
|
+
return header + page.text;
|
|
274
|
+
},
|
|
275
|
+
};
|
|
276
|
+
}
|
package/lib/session-cjk.js
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
import fs from "node:fs";
|
|
23
23
|
import path from "node:path";
|
|
24
24
|
|
|
25
|
-
import { decodeZstd, sessionsRoot } from "./usage.js";
|
|
25
|
+
import { decodeZstd, sessionLogFile, sessionsRoot } from "./usage.js";
|
|
26
26
|
import { redactText } from "./redact.js";
|
|
27
27
|
|
|
28
28
|
/** CJK ideographs (incl. Ext-A and compatibility ideographs). */
|
|
@@ -135,7 +135,7 @@ export function searchSessionsCJK({
|
|
|
135
135
|
}
|
|
136
136
|
const files = [];
|
|
137
137
|
for (const sid of sids) {
|
|
138
|
-
const file = path.join(bucketDir, sid
|
|
138
|
+
const file = sessionLogFile(path.join(bucketDir, sid));
|
|
139
139
|
let st;
|
|
140
140
|
try {
|
|
141
141
|
st = fs.statSync(file);
|