@hicaru/pi-rlm 0.3.16 → 0.3.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +0 -4
  2. package/README.ru.md +56 -66
  3. package/README.zh-CN.md +61 -65
  4. package/package.json +5 -5
  5. package/src/bridge/add-context.ts +1 -1
  6. package/src/bridge/handlers/await.ts +13 -22
  7. package/src/bridge/handlers/completion.ts +27 -5
  8. package/src/bridge/handlers/emitting.ts +2 -2
  9. package/src/bridge/handlers/llm-query.ts +46 -68
  10. package/src/bridge/handlers/rlm-query.ts +14 -84
  11. package/src/bridge/handlers/task-registry.ts +22 -17
  12. package/src/bridge/handlers/types.ts +8 -6
  13. package/src/bridge/model.ts +6 -3
  14. package/src/commands/rlm-llm.ts +1 -10
  15. package/src/commands/rlm-rlm.ts +1 -8
  16. package/src/config/defaults.ts +28 -12
  17. package/src/config/settings.ts +41 -31
  18. package/src/config/skillstate.ts +465 -0
  19. package/src/context/md-cache.ts +1 -1
  20. package/src/context/merge.ts +1 -1
  21. package/src/context/namespace.ts +2 -2
  22. package/src/context/refresh.ts +1 -1
  23. package/src/context/source-dir.ts +21 -11
  24. package/src/context/source-doc.ts +1 -1
  25. package/src/context/source-git.ts +3 -15
  26. package/src/context/source-text.ts +1 -1
  27. package/src/context/walk.ts +6 -14
  28. package/src/core/budget.ts +107 -21
  29. package/src/core/compaction.ts +44 -1
  30. package/src/core/engine.ts +141 -84
  31. package/src/core/iteration.ts +1 -1
  32. package/src/core/ledger.ts +10 -13
  33. package/src/core/limits.ts +1 -1
  34. package/src/core/model-registry.ts +1 -1
  35. package/src/core/resource-limits.ts +1 -1
  36. package/src/core/root-context.ts +126 -0
  37. package/src/core/root-digest.ts +213 -0
  38. package/src/core/root-state.ts +240 -0
  39. package/src/core/run-state.ts +577 -0
  40. package/src/core/types.ts +51 -12
  41. package/src/index.ts +167 -36
  42. package/src/mode/llm-model.ts +13 -1
  43. package/src/mode/native-guards.ts +0 -6
  44. package/src/mode/rlm-mode.ts +34 -11
  45. package/src/mode/subagent.ts +5 -5
  46. package/src/prompts/glossary.ts +41 -25
  47. package/src/prompts/native.ts +1 -3
  48. package/src/prompts/system.ts +12 -4
  49. package/src/sandbox/context-file.ts +1 -1
  50. package/src/sandbox/interrupts.ts +25 -31
  51. package/src/sandbox/protocol.ts +14 -20
  52. package/src/sandbox/py/__pycache__/guards.cpython-314.pyc +0 -0
  53. package/src/sandbox/py/__pycache__/scaffold.cpython-314.pyc +0 -0
  54. package/src/sandbox/py/__pycache__/worker.cpython-314.pyc +0 -0
  55. package/src/sandbox/py/guards.py +1 -1
  56. package/src/sandbox/py/scaffold.py +24 -31
  57. package/src/sandbox/py/worker.py +3 -1
  58. package/src/sandbox/sandbox-manager.ts +2 -2
  59. package/src/sandbox/sandbox.ts +21 -4
  60. package/src/text/agent-text.ts +58 -0
  61. package/src/text/parsing.ts +35 -3
  62. package/src/text/preview.ts +3 -0
  63. package/src/text/repl-output.ts +1 -1
  64. package/src/tool/background-tasks.ts +1 -1
  65. package/src/tool/repl-render.ts +1 -1
  66. package/src/tool/repl-result.ts +1 -1
  67. package/src/tool/repl-tool.ts +50 -26
  68. package/src/tool/rlm-tool.ts +4 -5
  69. package/src/tool/subcall-render.ts +1 -1
  70. package/src/tool/subcall-store.ts +2 -2
  71. package/src/tool/tool-utils.ts +5 -5
  72. package/src/ui/intro.ts +1 -1
  73. package/src/ui/modal/timeline-store.ts +1 -1
  74. package/src/ui/model-picker/drilldown.ts +1 -1
  75. package/src/ui/model-picker/levels.ts +1 -1
  76. package/src/ui/panel/run-registry.ts +1 -1
  77. package/src/ui/tree/tree-rows.ts +1 -1
  78. package/src/ui/tree/tree-widget.ts +1 -1
  79. package/src/util/bm25.ts +97 -0
  80. package/src/util/concurrency.ts +1 -1
  81. package/src/util/errors.ts +1 -1
  82. package/src/util/retry.ts +22 -7
  83. package/src/util/state-merge.ts +34 -0
  84. package/src/util/throttle.ts +1 -1
  85. package/src/util/type-guards.ts +6 -0
  86. package/src/core/memory.ts +0 -589
package/src/util/retry.ts CHANGED
@@ -95,8 +95,10 @@ export interface RetryPolicy {
95
95
  }
96
96
 
97
97
  export const DEFAULT_RETRY_POLICY: Readonly<RetryPolicy> = Object.freeze({
98
- maxAttempts: 3,
99
- rateLimitMaxAttempts: 8,
98
+ // DOCTRINE: 15 attempts on the SAME model, then the call fails loudly. No cross-model
99
+ // or cross-provider fallback exists anywhere in the runtime — by design.
100
+ maxAttempts: 15,
101
+ rateLimitMaxAttempts: 15,
100
102
  baseDelayMs: 500,
101
103
  maxDelayMs: 15_000,
102
104
  throttleBaseMs: 2_000,
@@ -104,7 +106,7 @@ export const DEFAULT_RETRY_POLICY: Readonly<RetryPolicy> = Object.freeze({
104
106
  });
105
107
 
106
108
  /** Shape of the optional retry knobs on RlmConfig — kept structural to avoid a cycle. */
107
- export interface RetryConfigNumbers {
109
+ interface RetryConfigNumbers {
108
110
  readonly retryMaxAttempts?: number;
109
111
  readonly rateLimitMaxAttempts?: number;
110
112
  readonly retryBaseDelayMs?: number;
@@ -142,7 +144,16 @@ export async function completeWithRetry<T>(
142
144
  readonly onRelease?: () => void;
143
145
  },
144
146
  ): Promise<T> {
145
- const { policy, provider, signal, onPark, onRelease } = opts;
147
+ const { policy, provider, signal } = opts;
148
+ // Bench rec #5: a silent park is indistinguishable from a hang (the campaign watched 15
149
+ // attempts × 15 cooldown windows with zero output). Callers may own observability via
150
+ // opts.onPark; otherwise one [rlm]-prefixed warn fires per park — fail-soft, never throws.
151
+ const onPark =
152
+ opts.onPark ??
153
+ ((ms: number): void => {
154
+ console.warn(`[rlm] ${provider} parked ${Math.round(ms)}ms on provider cooldown`);
155
+ });
156
+ const { onRelease } = opts;
146
157
  const cooldown = policy.cooldown ?? sharedCooldown;
147
158
  let status: number | undefined;
148
159
  let headers: Record<string, string> | undefined;
@@ -174,10 +185,14 @@ export async function completeWithRetry<T>(
174
185
  }
175
186
  if (tries + 1 >= policy.maxAttempts) throw err;
176
187
  if (!retryableError(status, msg)) throw err;
177
- await sleepMs(
178
- Math.min(retryAfterMs(headers) ?? backoffMs(tries, policy.baseDelayMs, policy.maxDelayMs), policy.maxDelayMs),
179
- signal,
188
+ const delay = Math.min(
189
+ retryAfterMs(headers) ?? backoffMs(tries, policy.baseDelayMs, policy.maxDelayMs),
190
+ policy.maxDelayMs,
180
191
  );
192
+ console.warn(
193
+ `[rlm] ${provider} attempt ${tries + 1}/${policy.maxAttempts} failed (${msg.slice(0, 140)}) — retrying in ${delay}ms`,
194
+ );
195
+ await sleepMs(delay, signal);
181
196
  }
182
197
  }
183
198
  }
@@ -0,0 +1,34 @@
1
+ /**
2
+ * THE deep-merge with null-deletion (SKILL.state paper §3.2, the `⊕` operator) — one
3
+ * implementation shared by the RunState patch path (core/run-state.ts, Workstream A) and the
4
+ * SkillState note merge (config/skillstate.ts, Workstream B). Never inline a second merge.
5
+ *
6
+ * x ⊕ {k: null} = x without k (delete — must be explicit, never silent)
7
+ * x ⊕ {k: v} = x[k] ⊕ v if both are plain objects (deep merge)
8
+ * else v (replace)
9
+ *
10
+ * Arrays are replaced wholesale: array semantics (append, index-set, dedup, eviction) are
11
+ * domain policy and live with the callers that understand them (see core/run-state.ts).
12
+ * A non-object patch replaces the base as-is.
13
+ */
14
+
15
+ function isPlainObject(value: unknown): value is Record<string, unknown> {
16
+ return typeof value === "object" && value !== null && !Array.isArray(value);
17
+ }
18
+
19
+ export function deepMergeWithNullDeletion(base: unknown, patch: unknown): unknown {
20
+ if (!isPlainObject(base) || !isPlainObject(patch)) return patch;
21
+ const out: Record<string, unknown> = { ...base };
22
+ for (const [key, value] of Object.entries(patch)) {
23
+ if (value === null) {
24
+ // eslint-disable-next-line @typescript-eslint/no-dynamic-delete -- ⊕ explicit null = delete (paper §3.2 +)
25
+ delete out[key]; // explicit null = delete (paper §3.2)
26
+ continue;
27
+ }
28
+ const current = out[key];
29
+ out[key] = isPlainObject(current) && isPlainObject(value)
30
+ ? deepMergeWithNullDeletion(current, value)
31
+ : value;
32
+ }
33
+ return out;
34
+ }
@@ -81,7 +81,7 @@ export class ProviderCooldown {
81
81
  }
82
82
 
83
83
  /** Mirrored into DEFAULT_RETRY_POLICY (util/retry.ts imports these — keep one-way). */
84
- export const THROTTLE_DEFAULTS = Object.freeze({ baseMs: 2_000, maxMs: 60_000 } as const);
84
+ const THROTTLE_DEFAULTS = Object.freeze({ baseMs: 2_000, maxMs: 60_000 } as const);
85
85
 
86
86
  /** Process-wide instance: every completion in this pi session shares it. */
87
87
  export const sharedCooldown: ProviderCooldown = new ProviderCooldown(
@@ -0,0 +1,6 @@
1
+ /** Shared runtime type guards — one implementation, imported everywhere (DRY). */
2
+
3
+ /** True for any non-null object value; the standard probe before field checks. */
4
+ export function isRecord(value: unknown): value is Record<string, unknown> {
5
+ return typeof value === "object" && value !== null;
6
+ }
@@ -1,589 +0,0 @@
1
- /**
2
- * Durable memory (port of the v5 `memory/store.py` engine).
3
- *
4
- * L1 episodes: content-addressed replay — a recorded child/root answer replays for ZERO
5
- * API calls while every file it touched still hashes to the recorded sha256.
6
- * L2 notes: A-MEM-lite BM25 notes over {content, context, keywords, tags, paths, symbols},
7
- * batched consolidation (no per-write evolve), link-on-write to top-4 neighbors.
8
- *
9
- * Layout (v5-identical): `<root>/.rlm/memory/{episodes.jsonl, notes.json}`.
10
- * All I/O is fail-soft: writers return booleans and never throw — a corrupt or missing
11
- * store degrades to a no-op, it never takes a run down.
12
- */
13
-
14
- import { createHash } from "node:crypto";
15
- import { closeSync, openSync, readSync } from "node:fs";
16
- import { readFileSync, mkdirSync, writeFileSync } from "node:fs";
17
- import { dirname, join, resolve, sep } from "node:path";
18
- import { formatError } from "../util/errors.ts";
19
-
20
- const TOK = /[a-z0-9]{2,}/g;
21
- const EPISODE_CAP = 4_000;
22
- const INJECT_HEADER = "[memory] retrieved notes (do not restudy these paths unless hashes went stale):";
23
- const NOTE_CONTENT_CHARS = 280;
24
- const LINK_NEIGHBORS = 4;
25
- const KEYWORDS_MAX = 12;
26
-
27
- export interface Episode {
28
- readonly key: string;
29
- readonly kind: "rlm" | "root";
30
- readonly model: string;
31
- readonly prompt: string;
32
- readonly paths: readonly string[];
33
- readonly pathHashes: Readonly<Record<string, string>>;
34
- readonly result: string;
35
- readonly tokensIn: number;
36
- readonly tokensOut: number;
37
- readonly ts: number;
38
- }
39
-
40
- export interface Note {
41
- readonly id: string;
42
- readonly content: string;
43
- readonly timestamp: number;
44
- readonly keywords: readonly string[];
45
- readonly tags: readonly string[];
46
- readonly context: string;
47
- readonly paths: readonly string[];
48
- readonly symbols: readonly string[];
49
- readonly links: readonly string[];
50
- readonly sourceKeys: readonly string[];
51
- }
52
-
53
- export interface MemoryStats {
54
- readonly episodes: number;
55
- readonly notes: number;
56
- readonly hits: number;
57
- readonly misses: number;
58
- readonly notesInjected: number;
59
- }
60
-
61
- /** Single completion seam for consolidation (wired to bridge/model at the composition root). */
62
- export type MemoryLlm = (prompt: string) => Promise<string>;
63
-
64
- export interface MemoryOptions {
65
- /** Override directory; default resolves to `<root>/.rlm/memory`. */
66
- readonly dir?: string;
67
- readonly injectNoteTokens?: number;
68
- readonly evolveEvery?: number;
69
- readonly llm?: MemoryLlm;
70
- }
71
-
72
- function tokenize(text: string): readonly string[] {
73
- return (text.toLowerCase().replace(/_/g, " ").replace(/-/g, " ").match(TOK) ?? []) as readonly string[];
74
- }
75
-
76
- function noteBlob(n: Note): string {
77
- return [n.content, n.context, n.keywords.join(" "), n.tags.join(" "), n.paths.join(" "), n.symbols.join(" ")].join(" ");
78
- }
79
-
80
- /** Sync streamed sha256 (64KiB chunks via readSync — audit H7): a huge path must never
81
- * buffer whole in memory, and no async contagion into recordEpisode/replay. */
82
- export function fileSha256(path: string): string | undefined {
83
- let fd: number | undefined;
84
- try {
85
- fd = openSync(path, "r");
86
- const h = createHash("sha256");
87
- const buf = Buffer.allocUnsafe(1 << 16);
88
- for (;;) {
89
- const n = readSync(fd, buf, 0, buf.length, null);
90
- if (n <= 0) break;
91
- h.update(n === buf.length ? buf : buf.subarray(0, n));
92
- }
93
- return h.digest("hex");
94
- } catch {
95
- return undefined;
96
- } finally {
97
- if (fd !== undefined) {
98
- try {
99
- closeSync(fd);
100
- } catch {
101
- // already closed — nothing to do
102
- }
103
- }
104
- }
105
- }
106
-
107
- function isRecord(value: unknown): value is Record<string, unknown> {
108
- return typeof value === "object" && value !== null;
109
- }
110
-
111
- /** H6 (audit): the real-file slice of a root context — cwd-seeded entries (un-prefixed
112
- * paths with string content), bounded, `ctx/<id>/…` virtual sources excluded (they have no
113
- * disk file to hash). These are what a root episode snapshots for replay invalidation. */
114
- export function rootContextPaths(context: unknown, max: number): readonly string[] {
115
- if (!Array.isArray(context)) return Object.freeze([]);
116
- const out: string[] = [];
117
- for (const item of context) {
118
- if (out.length >= max) break;
119
- if (isRecord(item) && typeof item.path === "string" && typeof item.content === "string") {
120
- const p = item.path;
121
- if (p !== "" && !p.startsWith("ctx/") && !p.includes("/ctx/") && !p.startsWith("/")) out.push(p);
122
- }
123
- }
124
- return Object.freeze(out);
125
- }
126
-
127
- /** Who is calling serviceOp — delegation children read durable notes but never write them
128
- * (their findings persist via recordEpisode in the rlm_query handler, one per run). */
129
- export type MemoryScope = "root" | "child";
130
-
131
- export class MemoryStore {
132
- readonly enabled: boolean;
133
- private dir: string | undefined;
134
- private readonly pinnedDir: boolean;
135
- private readonly injectNoteTokens: number;
136
- private readonly evolveEvery: number;
137
- private llm: MemoryLlm | undefined;
138
- private root: string;
139
- private episodes = new Map<string, Episode>();
140
- private notes = new Map<string, Note>();
141
- private pending: readonly string[] = [];
142
- private hits = 0;
143
- private misses = 0;
144
- private notesInjected = 0;
145
- private loaded = false;
146
-
147
- constructor(root: string, opts: MemoryOptions = {}, enabled = true) {
148
- this.root = root;
149
- this.enabled = enabled && opts.dir !== null;
150
- this.pinnedDir = opts.dir !== undefined;
151
- this.dir = this.enabled ? (opts.dir ?? join(root, ".rlm", "memory")) : undefined;
152
- this.injectNoteTokens = opts.injectNoteTokens ?? 2_000;
153
- this.evolveEvery = opts.evolveEvery ?? 8;
154
- this.llm = opts.llm;
155
- }
156
-
157
- /** Session hooks: the consolidation model + real workspace root arrive after construction. */
158
- setLlm(llm: MemoryLlm): void {
159
- this.llm = llm;
160
- }
161
-
162
- setRoot(root: string): void {
163
- if (root === this.root) return;
164
- this.root = root;
165
- if (!this.pinnedDir && this.enabled) this.dir = join(root, ".rlm", "memory");
166
- this.loaded = false;
167
- this.episodes = new Map();
168
- this.notes = new Map();
169
- this.pending = [];
170
- }
171
-
172
- // ── L1: episodes ────────────────────────────────────────────────────────────
173
-
174
- /** H7 (audit): resolve a rel-path INSIDE the root only — `../` traversal gets no digest. */
175
- private safeAbs(rel: string): string | undefined {
176
- const rootAbs = resolve(this.root);
177
- const abs = resolve(rootAbs, rel);
178
- return abs === rootAbs || abs.startsWith(rootAbs + sep) ? abs : undefined;
179
- }
180
-
181
- /** Record a completed run. Snapshots path hashes; triggers consolidation on threshold. */
182
- recordEpisode(req: {
183
- readonly key: string;
184
- readonly kind: "rlm" | "root";
185
- readonly model: string;
186
- readonly prompt: string;
187
- readonly paths: readonly string[];
188
- readonly result: string;
189
- readonly tokensIn?: number;
190
- readonly tokensOut?: number;
191
- }): boolean {
192
- if (!this.enabled || this.dir === undefined || req.result === "") return false;
193
- this.load();
194
- const pathHashes: Record<string, string> = {};
195
- for (const rel of req.paths) {
196
- const abs = this.safeAbs(rel);
197
- const digest = abs === undefined ? undefined : fileSha256(abs);
198
- if (digest !== undefined) pathHashes[rel.replace(/\\/g, "/")] = digest;
199
- }
200
- const ep: Episode = {
201
- key: req.key,
202
- kind: req.kind,
203
- model: req.model,
204
- prompt: req.prompt,
205
- paths: Object.freeze([...req.paths]),
206
- pathHashes: Object.freeze(pathHashes),
207
- result: req.result,
208
- tokensIn: req.tokensIn ?? 0,
209
- tokensOut: req.tokensOut ?? 0,
210
- ts: Date.now(),
211
- };
212
- this.episodes.set(ep.key, ep);
213
- this.pending = [...this.pending, ep.key];
214
- const appended = this.appendEpisode(ep);
215
- if (this.pending.length >= this.evolveEvery) void this.consolidate();
216
- return appended;
217
- }
218
-
219
- /** Zero-API-call replay — only while every recorded hash still matches. */
220
- replay(key: string): Episode | undefined {
221
- if (!this.enabled) return undefined;
222
- this.load();
223
- const ep = this.episodes.get(key);
224
- if (ep === undefined) {
225
- this.misses++;
226
- return undefined;
227
- }
228
- if (!this.hashesFresh(ep.pathHashes)) {
229
- this.misses++;
230
- return undefined;
231
- }
232
- this.hits++;
233
- return ep;
234
- }
235
-
236
- private hashesFresh(pathHashes: Readonly<Record<string, string>>): boolean {
237
- for (const [rel, digest] of Object.entries(pathHashes)) {
238
- const abs = this.safeAbs(rel);
239
- if (abs === undefined) return false; // path escaped the root — treat as drifted
240
- if (fileSha256(abs) !== digest) return false;
241
- }
242
- return true;
243
- }
244
-
245
- private appendEpisode(ep: Episode): boolean {
246
- if (this.dir === undefined) return false;
247
- try {
248
- mkdirSync(dirname(join(this.dir, "episodes.jsonl")), { recursive: true });
249
- writeFileSync(join(this.dir, "episodes.jsonl"), `${JSON.stringify(ep)}\n`, { flag: "a" });
250
- if (this.episodes.size > EPISODE_CAP) this.rewriteEpisodes();
251
- return true;
252
- } catch {
253
- return false; // fail-soft: warn-free degradation
254
- }
255
- }
256
-
257
- private rewriteEpisodes(): void {
258
- if (this.dir === undefined) return;
259
- const keep = [...this.episodes.values()].sort((a, b) => b.ts - a.ts).slice(0, EPISODE_CAP);
260
- this.episodes = new Map(keep.map((e) => [e.key, e]));
261
- try {
262
- writeFileSync(
263
- join(this.dir, "episodes.jsonl"),
264
- keep.map((e) => JSON.stringify(e)).join("\n") + (keep.length > 0 ? "\n" : ""),
265
- );
266
- } catch {
267
- // fail-soft: the append already succeeded; the trim retries on the next overflow
268
- }
269
- }
270
-
271
- // ── L2: notes ───────────────────────────────────────────────────────────────
272
-
273
- addNote(req: {
274
- readonly content: string;
275
- readonly paths?: readonly string[];
276
- readonly tags?: readonly string[];
277
- readonly context?: string;
278
- readonly symbols?: readonly string[];
279
- readonly sourceKeys?: readonly string[];
280
- readonly noteId?: string;
281
- }): Note | undefined {
282
- if (!this.enabled || this.dir === undefined || req.content.trim() === "") return undefined;
283
- this.load();
284
- const paths = req.paths ?? [];
285
- const id =
286
- req.noteId ?? createHash("sha256").update(`${req.content}|${paths.join(",")}`).digest("hex").slice(0, 16);
287
- const existing = this.notes.get(id);
288
- const toks = tokenize(req.content);
289
- const note: Note = {
290
- id,
291
- content: req.content.trim(),
292
- timestamp: Date.now(),
293
- keywords: Object.freeze(existing?.keywords ?? toks.slice(0, KEYWORDS_MAX)),
294
- tags: Object.freeze([...(req.tags ?? [])]),
295
- context: req.context ?? "",
296
- paths: Object.freeze([...paths]),
297
- symbols: Object.freeze([...(req.symbols ?? [])]),
298
- links: Object.freeze([...(existing?.links ?? [])]),
299
- sourceKeys: Object.freeze([...new Set([...(existing?.sourceKeys ?? []), ...(req.sourceKeys ?? [])])]),
300
- };
301
- // link-on-write: top-4 BM25 neighbors, bidirectional (merge for re-writes)
302
- const neighbors = this.bm25(note.content, [...this.notes.values()].filter((n) => n.id !== id), LINK_NEIGHBORS);
303
- const links = [...new Set([...note.links, ...neighbors.map(([n]) => n.id)])];
304
- this.notes.set(id, { ...note, links: Object.freeze(links) });
305
- for (const [n] of neighbors) {
306
- const back = this.notes.get(n.id);
307
- if (back !== undefined) {
308
- this.notes.set(n.id, { ...back, links: Object.freeze([...new Set([...back.links, id])]) });
309
- }
310
- }
311
- this.saveNotes();
312
- return this.notes.get(id);
313
- }
314
-
315
- /** BM25 retrieval over note blobs. */
316
- query(text: string, k = 8): readonly Note[] {
317
- if (!this.enabled) return [];
318
- this.load();
319
- return Object.freeze(this.bm25(text, [...this.notes.values()], k).map(([n]) => n));
320
- }
321
-
322
- private bm25(query: string, notes: readonly Note[], k: number): readonly [Note, number][] {
323
- if (notes.length === 0 || query.trim() === "") return [];
324
- const qTokens = new Set(tokenize(query));
325
- if (qTokens.size === 0) return [];
326
- const docs = notes.map((n) => tokenize(noteBlob(n)));
327
- const df = new Map<string, number>();
328
- for (const toks of docs) {
329
- for (const t of new Set(toks)) df.set(t, (df.get(t) ?? 0) + 1);
330
- }
331
- const n = docs.length;
332
- const avgdl = docs.reduce((s, t) => s + t.length, 0) / Math.max(1, n);
333
- const k1 = 1.5;
334
- const b = 0.75;
335
- const scored = new Array<[Note, number]>(n);
336
- for (let i = 0; i < n; i++) {
337
- const toks = docs[i];
338
- const tf = new Map<string, number>();
339
- for (const t of toks) tf.set(t, (tf.get(t) ?? 0) + 1);
340
- const dl = toks.length || 1;
341
- let score = 0;
342
- for (const q of qTokens) {
343
- const f = tf.get(q);
344
- if (f === undefined) continue;
345
- const idf = Math.log(1 + (n - (df.get(q) ?? 0) + 0.5) / ((df.get(q) ?? 0) + 0.5));
346
- score += idf * ((f * (k1 + 1)) / (f + k1 * (1 - b + (b * dl) / avgdl)));
347
- }
348
- scored[i] = [notes[i], score];
349
- }
350
- scored.sort((a, c) => c[1] - a[1] || (a[0].id < c[0].id ? -1 : 1));
351
- return scored.slice(0, k).filter(([, s]) => s > 0);
352
- }
353
-
354
- /** Batched A-MEM-lite: one prompt over every pending episode → note list; verbatim fallback.
355
- * Single-flight (audit M2): overlapping recordEpisode bursts share ONE consolidation. */
356
- async consolidate(): Promise<number> {
357
- if (this.consolidating !== undefined) return this.consolidating;
358
- const run = this.doConsolidate().finally(() => {
359
- this.consolidating = undefined;
360
- });
361
- this.consolidating = run;
362
- return run;
363
- }
364
-
365
- private consolidating: Promise<number> | undefined;
366
-
367
- private async doConsolidate(): Promise<number> {
368
- if (!this.enabled || this.dir === undefined) return 0;
369
- this.load();
370
- // Snapshot the batch we are about to distill. Mid-flight recordEpisode
371
- // keys must survive — wiping `this.pending = []` at the end dropped them (audit R5).
372
- const batch = this.pending;
373
- const pendingEps = batch
374
- .map((k) => this.episodes.get(k))
375
- .filter((e): e is Episode => e !== undefined);
376
- if (pendingEps.length === 0) return 0;
377
- let made = 0;
378
- if (this.llm !== undefined) {
379
- try {
380
- const payload = JSON.stringify(
381
- pendingEps.map((e) => ({ prompt: e.prompt.slice(0, 200), answer: e.result.slice(0, 600), paths: e.paths })),
382
- );
383
- const raw = await this.llm(
384
- "Distill the following completed research episodes into durable notes. " +
385
- "Return STRICT JSON: an array of {content, tags, paths} objects (max 12, one line each, " +
386
- "no preamble). Episodes:\n" + payload,
387
- );
388
- // M2 (audit): try the whole reply as JSON first; only then fall back to slicing the
389
- // first bracket span out of prose. Item-level validation below rejects junk either way.
390
- const parsed: unknown = await (async (): Promise<unknown> => {
391
- try {
392
- return JSON.parse(raw.trim());
393
- } catch {
394
- const slice = raw.slice(raw.indexOf("["), raw.lastIndexOf("]") + 1);
395
- return slice === "" ? null : JSON.parse(slice);
396
- }
397
- })();
398
- if (Array.isArray(parsed)) {
399
- for (const item of parsed) {
400
- if (typeof item === "object" && item !== null) {
401
- const r = item as Record<string, unknown>;
402
- if (typeof r.content === "string") {
403
- this.addNote({
404
- content: r.content,
405
- tags: Array.isArray(r.tags) ? r.tags.filter((t): t is string => typeof t === "string") : [],
406
- paths: Array.isArray(r.paths) ? r.paths.filter((p): p is string => typeof p === "string") : [],
407
- sourceKeys: pendingEps.map((e) => e.key),
408
- });
409
- made++;
410
- }
411
- }
412
- }
413
- }
414
- } catch {
415
- made = 0; // fall through to the verbatim fallback below
416
- }
417
- }
418
- if (made === 0) {
419
- for (const e of pendingEps) {
420
- this.addNote({
421
- content: `${e.prompt.slice(0, 200)} → ${e.result.slice(0, 400)}`,
422
- paths: e.paths,
423
- tags: ["episode"],
424
- sourceKeys: [e.key],
425
- });
426
- made++;
427
- }
428
- }
429
- this.pending = this.pending.filter((k) => !batch.includes(k));
430
- return made;
431
- }
432
-
433
- /** v2 rule: silent when empty (v1 burned ~90 chars/turn teaching an empty store). */
434
- injectBlock(query: string, k = 6): string {
435
- const notes = this.query(query, k);
436
- if (notes.length === 0) {
437
- this.notesInjected = 0;
438
- return "";
439
- }
440
- const budget = this.injectNoteTokens * 4; // chars
441
- const lines: string[] = [INJECT_HEADER];
442
- let used = INJECT_HEADER.length;
443
- let kept = 0;
444
- for (const n of notes) {
445
- const chunk = `- ${n.id} tags=${n.tags.join(",") || "-"} paths=${n.paths.join(",") || "-"}: ${n.content.slice(0, NOTE_CONTENT_CHARS)}`;
446
- if (used + chunk.length > budget) break;
447
- lines.push(chunk);
448
- used += chunk.length;
449
- kept++;
450
- }
451
- this.notesInjected = kept;
452
- return kept > 0 ? lines.join("\n") : "";
453
- }
454
-
455
- stats(): MemoryStats {
456
- return Object.freeze({
457
- episodes: this.episodes.size,
458
- notes: this.notes.size,
459
- hits: this.hits,
460
- misses: this.misses,
461
- notesInjected: this.notesInjected,
462
- });
463
- }
464
-
465
- /** The ONE implementation of the sandbox `memory.query/add/stats` surface — the engine
466
- * and the native repl tool both route their `memoryOp` interrupt here. */
467
- serviceOp(
468
- op: "query" | "add" | "stats",
469
- args: { readonly query?: string; readonly k?: number; readonly content?: string; readonly paths?: readonly string[]; readonly tags?: readonly string[] },
470
- scope: MemoryScope = "root",
471
- ): string {
472
- if (!this.enabled) return "memory disabled";
473
- if (op === "stats") return JSON.stringify(this.stats());
474
- if (op === "add" && scope === "child") {
475
- return formatError(
476
- "memory.add is root-only — query durable notes with memory.query; " +
477
- "your final answer is recorded as an episode automatically",
478
- );
479
- }
480
- if (op === "add") {
481
- const n = this.addNote({ content: args.content ?? "", paths: args.paths ?? [], tags: args.tags ?? [] });
482
- return n === undefined ? "add skipped (empty content)" : `ok note ${n.id}`;
483
- }
484
- const notes = this.query(args.query ?? "", args.k ?? 8);
485
- if (notes.length === 0) return "no notes match";
486
- const lines = new Array<string>(notes.length);
487
- for (let i = 0; i < notes.length; i++) {
488
- const n = notes[i];
489
- lines[i] = `- ${n.id} tags=${n.tags.join(",") || "-"} paths=${n.paths.join(",") || "-"}: ${n.content.slice(0, NOTE_CONTENT_CHARS)}`;
490
- }
491
- return lines.join("\n");
492
- }
493
-
494
- // ── persistence (fail-soft) ─────────────────────────────────────────────────
495
-
496
- private saveNotes(): void {
497
- if (this.dir === undefined) return;
498
- try {
499
- mkdirSync(this.dir, { recursive: true });
500
- const obj: Record<string, Note> = {};
501
- for (const [id, n] of this.notes) obj[id] = n;
502
- writeFileSync(join(this.dir, "notes.json"), JSON.stringify(obj));
503
- } catch {
504
- // fail-soft
505
- }
506
- }
507
-
508
- private load(): void {
509
- if (this.loaded || this.dir === undefined) return;
510
- this.loaded = true;
511
- try {
512
- const raw = readFileSync(join(this.dir, "episodes.jsonl"), "utf8");
513
- for (const line of raw.split("\n")) {
514
- if (line.trim() === "") continue;
515
- try {
516
- const ep = parseEpisode(JSON.parse(line) as unknown);
517
- if (ep !== undefined) this.episodes.set(ep.key, ep);
518
- } catch {
519
- // skip corrupt line — one bad append must not lose the store
520
- }
521
- }
522
- } catch {
523
- // no episodes yet
524
- }
525
- try {
526
- const rawNotes = JSON.parse(readFileSync(join(this.dir, "notes.json"), "utf8")) as unknown;
527
- if (typeof rawNotes === "object" && rawNotes !== null) {
528
- for (const v of Object.values(rawNotes as Record<string, unknown>)) {
529
- const n = parseNote(v);
530
- if (n !== undefined) this.notes.set(n.id, n);
531
- }
532
- }
533
- } catch {
534
- // no notes yet
535
- }
536
- }
537
- }
538
-
539
- function str(v: unknown, fallback = ""): string {
540
- return typeof v === "string" ? v : fallback;
541
- }
542
- function strArr(v: unknown): readonly string[] {
543
- return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
544
- }
545
- function num(v: unknown): number {
546
- return typeof v === "number" && Number.isFinite(v) ? v : 0;
547
- }
548
-
549
- function parseEpisode(v: unknown): Episode | undefined {
550
- if (typeof v !== "object" || v === null) return undefined;
551
- const r = v as Record<string, unknown>;
552
- if (typeof r.key !== "string" || typeof r.result !== "string") return undefined;
553
- const hashes: Record<string, string> = {};
554
- if (typeof r.pathHashes === "object" && r.pathHashes !== null) {
555
- for (const [k, h] of Object.entries(r.pathHashes as Record<string, unknown>)) {
556
- if (typeof h === "string") hashes[k] = h;
557
- }
558
- }
559
- return {
560
- key: r.key,
561
- kind: r.kind === "root" ? "root" : "rlm",
562
- model: str(r.model),
563
- prompt: str(r.prompt),
564
- paths: strArr(r.paths),
565
- pathHashes: hashes,
566
- result: r.result,
567
- tokensIn: num(r.tokensIn),
568
- tokensOut: num(r.tokensOut),
569
- ts: num(r.ts),
570
- };
571
- }
572
-
573
- function parseNote(v: unknown): Note | undefined {
574
- if (typeof v !== "object" || v === null) return undefined;
575
- const r = v as Record<string, unknown>;
576
- if (typeof r.id !== "string" || typeof r.content !== "string") return undefined;
577
- return {
578
- id: r.id,
579
- content: r.content,
580
- timestamp: num(r.timestamp),
581
- keywords: strArr(r.keywords),
582
- tags: strArr(r.tags),
583
- context: str(r.context),
584
- paths: strArr(r.paths),
585
- symbols: strArr(r.symbols),
586
- links: strArr(r.links),
587
- sourceKeys: strArr(r.sourceKeys),
588
- };
589
- }