@hicaru/pi-rlm 0.3.20 → 0.3.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/commands/rlm.ts +14 -7
- package/src/config/defaults.ts +33 -10
- package/src/config/settings.ts +6 -0
- package/src/config/skillstate.ts +236 -44
- package/src/core/budget.ts +7 -3
- package/src/core/compaction.ts +2 -2
- package/src/core/engine.ts +87 -19
- package/src/core/root-context.ts +74 -21
- package/src/core/root-digest.ts +48 -11
- package/src/core/root-state.ts +39 -12
- package/src/core/run-state.ts +86 -14
- package/src/core/session-archive.ts +174 -0
- package/src/core/types.ts +6 -0
- package/src/index.ts +142 -12
- package/src/mode/rlm-mode.ts +2 -2
- package/src/prompts/glossary.ts +34 -5
- package/src/prompts/native.ts +8 -2
- package/src/prompts/user.ts +4 -3
- package/src/sandbox/py/__pycache__/retrieval.cpython-314.pyc +0 -0
- package/src/sandbox/py/__pycache__/scaffold.cpython-314.pyc +0 -0
- package/src/sandbox/py/retrieval.py +202 -36
- package/src/sandbox/py/scaffold.py +20 -5
- package/src/sandbox/py/worker.py +1 -1
- package/src/sandbox/sandbox-manager.ts +19 -0
- package/src/text/parsing.ts +133 -2
- package/src/text/tokens.ts +39 -4
- package/src/tool/repl-render.ts +38 -2
- package/src/tool/repl-tool.ts +34 -18
- package/src/tool/subcall-render.ts +7 -4
- package/src/ui/config-panel.ts +2 -2
- package/src/ui/intro.ts +1 -1
- package/src/ui/python-highlight.ts +49 -0
- package/src/ui/stage-cards.ts +192 -0
- package/src/ui/tree/tree-model.ts +69 -19
- package/src/ui/tree/tree-rows.ts +2 -1
- package/src/util/abort.ts +34 -0
- package/src/util/bm25.ts +170 -21
- package/src/util/errors.ts +1 -1
package/package.json
CHANGED
package/src/commands/rlm.ts
CHANGED
|
@@ -4,7 +4,12 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
|
4
4
|
import type { RlmController } from "../mode/rlm-mode.ts";
|
|
5
5
|
import { setRlmModeStatus } from "../ui/status.ts";
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
/**
|
|
8
|
+
* `stopNative` aborts the native-mode session work (repl cells, child engines, detached
|
|
9
|
+
* spawn() tasks) and reports whether any was in flight. Optional so tests can register the
|
|
10
|
+
* bare command; production wires the closure accessors from src/index.ts.
|
|
11
|
+
*/
|
|
12
|
+
export function registerRlmCommand(pi: ExtensionAPI, controller: RlmController, stopNative?: () => boolean): void {
|
|
8
13
|
pi.registerCommand("rlm", {
|
|
9
14
|
description: "Toggle persistent RLM mode (route plain prompts through the RLM engine).",
|
|
10
15
|
handler: async (_args, ctx) => {
|
|
@@ -15,14 +20,16 @@ export function registerRlmCommand(pi: ExtensionAPI, controller: RlmController):
|
|
|
15
20
|
});
|
|
16
21
|
|
|
17
22
|
pi.registerCommand("rlm-stop", {
|
|
18
|
-
description: "Abort
|
|
23
|
+
description: "Abort in-progress RLM work: RLM runs, native repl cells, background tasks.",
|
|
19
24
|
handler: async (_args, ctx) => {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
25
|
+
const rlmBusy = controller.isBusy();
|
|
26
|
+
if (rlmBusy) controller.abort();
|
|
27
|
+
const nativeBusy = stopNative?.() ?? false;
|
|
28
|
+
if (rlmBusy || nativeBusy) {
|
|
29
|
+
ctx.ui.notify("RLM work aborted — runs, repl cells and background tasks stopped.", "info");
|
|
30
|
+
} else {
|
|
31
|
+
ctx.ui.notify("No RLM work in progress.", "info");
|
|
23
32
|
}
|
|
24
|
-
controller.abort();
|
|
25
|
-
ctx.ui.notify("RLM run aborted.", "info");
|
|
26
33
|
},
|
|
27
34
|
});
|
|
28
35
|
|
package/src/config/defaults.ts
CHANGED
|
@@ -1,10 +1,18 @@
|
|
|
1
1
|
import type { RlmConfig } from "../core/types.ts";
|
|
2
2
|
|
|
3
|
-
/**
|
|
3
|
+
/**
|
|
4
|
+
* Frozen default sub-LLM system prompt — avoids re-allocation on every llm_query call.
|
|
5
|
+
* Anthropic prompting canon applied: a role ("precise extraction assistant") and the
|
|
6
|
+
* hallucination out ("reply exactly NOT_FOUND when the material lacks the answer") — the
|
|
7
|
+
* convention the glossary documents so roots can branch on a leaf's NOT_FOUND instead of
|
|
8
|
+
* retrying identical prompts against slices that cannot answer.
|
|
9
|
+
*/
|
|
4
10
|
const DEFAULT_SUB_SYSTEM_PROMPT =
|
|
5
|
-
"
|
|
6
|
-
"
|
|
7
|
-
"
|
|
11
|
+
"You are a precise extraction and analysis assistant. " +
|
|
12
|
+
"Answer directly and concisely from the material provided in the prompt. " +
|
|
13
|
+
"Return only the requested information — no preamble, no meta-commentary, no explanation of your approach. " +
|
|
14
|
+
"If listing items, use compact bullet form. " +
|
|
15
|
+
"If the material does not contain the answer, reply exactly: NOT_FOUND";
|
|
8
16
|
|
|
9
17
|
export const DEFAULT_CONFIG: Readonly<RlmConfig> = Object.freeze({
|
|
10
18
|
enabled: true,
|
|
@@ -64,9 +72,11 @@ export const DEFAULT_CONFIG: Readonly<RlmConfig> = Object.freeze({
|
|
|
64
72
|
// v5 TaskLedger blackboard
|
|
65
73
|
enableLedger: true,
|
|
66
74
|
rlmBudget: 8,
|
|
67
|
-
// Verification-discipline nudge —
|
|
68
|
-
//
|
|
69
|
-
|
|
75
|
+
// Verification-discipline nudge — default ON (knowlange rec: 28/33 bench failures were
|
|
76
|
+
// early confident wrong answers; the RLM paper ships the same discipline). An early bare
|
|
77
|
+
// finalize — or one from a run that never inspected its context — gets ONE coached redo
|
|
78
|
+
// before the answer is accepted. Opt out via rlm.json (`"enableVerificationNudge": false`).
|
|
79
|
+
enableVerificationNudge: true,
|
|
70
80
|
// SKILL.state integration: Σ_t execution state + cross-session distilled knowledge.
|
|
71
81
|
// Paradigm flags are ENFORCED (R0, /tmp/ROOT_FULL_SKILLSTATE_PLAN.md) — validateEnforcedOn
|
|
72
82
|
// forces true whatever rlm.json carries; only calibrations are tunable.
|
|
@@ -78,7 +88,12 @@ export const DEFAULT_CONFIG: Readonly<RlmConfig> = Object.freeze({
|
|
|
78
88
|
enableSkillStateDistill: true, // ENFORCED — see /tmp/ROOT_FULL_SKILLSTATE_PLAN.md R0
|
|
79
89
|
skillStateMaxTokens: 1_200,
|
|
80
90
|
skillStateLeafTokens: 200,
|
|
81
|
-
|
|
91
|
+
// 2.5 (was 4.0 — recall W3): at 4.0 grounding silently no-oped on most prompts; 2.5 keeps
|
|
92
|
+
// the byte-identical-below-threshold contract while letting genuinely-relevant facts land.
|
|
93
|
+
skillStateMinScore: 2.5,
|
|
94
|
+
// Ξ block floor (recall W3): was Number.MIN_VALUE — every positively-scored stale note
|
|
95
|
+
// rode every root prompt. 2.0 admits relevant notes without the cross-session noise.
|
|
96
|
+
skillStateXiMinScore: 2.0,
|
|
82
97
|
skillStateNotesPerProject: 128,
|
|
83
98
|
// Root Σ integration (WS-2..WS-4): every LLM call assembles A_t = (P, Σ_t, O_t) — discard
|
|
84
99
|
// semantics on stale payloads + exactly one Σ snapshot splice, and model-proposed ΔΣ_t
|
|
@@ -89,8 +104,16 @@ export const DEFAULT_CONFIG: Readonly<RlmConfig> = Object.freeze({
|
|
|
89
104
|
rootDigestKeepRecentChars: 12_000,
|
|
90
105
|
rootDigestMaxChars: 8_000,
|
|
91
106
|
enableRootContextTransform: true, // ENFORCED — see /tmp/ROOT_FULL_SKILLSTATE_PLAN.md R0 (was soak-OFF pre-v2)
|
|
92
|
-
|
|
93
|
-
|
|
107
|
+
// Recall W4 calibration: 4 verbatim turns (was 2). SKILL.state's budget-matched ablation
|
|
108
|
+
// (Table 5/11) shows truncated windows collapse recall (0.18 vs structured 0.94); a 2-turn
|
|
109
|
+
// window sat dangerously close to that shape. Still O(1) per call; the session archive
|
|
110
|
+
// (rootArchiveMaxChars) makes anything older dereferenceable instead of gone.
|
|
111
|
+
rootContextKeepTurns: 4,
|
|
112
|
+
rootContextElideChars: 3_000,
|
|
113
|
+
// Recall W1: elided turns archive into the sandbox (ctx/session-log/*) so search()/
|
|
114
|
+
// grep_context() recall them — elision becomes dereferenceable, and the stubs stay honest.
|
|
115
|
+
// 0 disables the archive (stubs degrade to the plain Σ line).
|
|
116
|
+
rootArchiveMaxChars: 2_000_000,
|
|
94
117
|
rootContextSnapshot: true,
|
|
95
118
|
enableRootStateFences: true, // ENFORCED — see /tmp/ROOT_FULL_SKILLSTATE_PLAN.md R0 (was soak-OFF pre-v2)
|
|
96
119
|
});
|
package/src/config/settings.ts
CHANGED
|
@@ -167,6 +167,8 @@ export function validateConfig(raw: unknown): Partial<RlmConfig> {
|
|
|
167
167
|
if (skillStateLeafTokens !== undefined) out.skillStateLeafTokens = skillStateLeafTokens;
|
|
168
168
|
const skillStateMinScore = validateNumber(r.skillStateMinScore, 0);
|
|
169
169
|
if (skillStateMinScore !== undefined) out.skillStateMinScore = skillStateMinScore;
|
|
170
|
+
const skillStateXiMinScore = validateNumber(r.skillStateXiMinScore, 0);
|
|
171
|
+
if (skillStateXiMinScore !== undefined) out.skillStateXiMinScore = skillStateXiMinScore;
|
|
170
172
|
const skillStateNotesPerProject = validateNumber(r.skillStateNotesPerProject, 1);
|
|
171
173
|
if (skillStateNotesPerProject !== undefined) out.skillStateNotesPerProject = skillStateNotesPerProject;
|
|
172
174
|
// Root Σ integration (WS-2..WS-4) — paradigm flags ENFORCED (R0); the window/byte knobs
|
|
@@ -181,6 +183,10 @@ export function validateConfig(raw: unknown): Partial<RlmConfig> {
|
|
|
181
183
|
if (rootContextKeepTurns !== undefined) out.rootContextKeepTurns = rootContextKeepTurns;
|
|
182
184
|
const rootContextElideChars = validateNumber(r.rootContextElideChars, 100);
|
|
183
185
|
if (rootContextElideChars !== undefined) out.rootContextElideChars = rootContextElideChars;
|
|
186
|
+
// Recall W1 archive calibration: 0 legitimately disables the archive (plain stubs), so the
|
|
187
|
+
// floor differs from the other char knobs.
|
|
188
|
+
const rootArchiveMaxChars = validateNumber(r.rootArchiveMaxChars, 0);
|
|
189
|
+
if (rootArchiveMaxChars !== undefined) out.rootArchiveMaxChars = rootArchiveMaxChars;
|
|
184
190
|
const rootContextSnapshot = validateBoolean(r.rootContextSnapshot);
|
|
185
191
|
if (rootContextSnapshot !== undefined) out.rootContextSnapshot = rootContextSnapshot;
|
|
186
192
|
out.enableRootStateFences = validateEnforcedOn(r.enableRootStateFences, "enableRootStateFences");
|
package/src/config/skillstate.ts
CHANGED
|
@@ -22,10 +22,15 @@ import { bm25Rank } from "../util/bm25.ts";
|
|
|
22
22
|
import { deepMergeWithNullDeletion } from "../util/state-merge.ts";
|
|
23
23
|
import { formatError } from "../util/errors.ts";
|
|
24
24
|
import { isRecord } from "../util/type-guards.ts";
|
|
25
|
-
import type { RunState } from "../core/run-state.ts";
|
|
25
|
+
import type { ApproachOutcome, RunState } from "../core/run-state.ts";
|
|
26
26
|
import type { RlmConfig } from "../core/types.ts";
|
|
27
27
|
import { skillStateLines } from "../prompts/glossary.ts";
|
|
28
28
|
|
|
29
|
+
/** The outcome detail text per union member (filter can't narrow; this keeps it one switch). */
|
|
30
|
+
function outcomeDetail(outcome: ApproachOutcome): string {
|
|
31
|
+
return outcome.status === "failed" ? outcome.reason : outcome.status === "partial" ? outcome.note : outcome.evidence;
|
|
32
|
+
}
|
|
33
|
+
|
|
29
34
|
export const SKILL_STATE_FILE = "rlm-skillstate.json";
|
|
30
35
|
|
|
31
36
|
/** A-Mem-derived note (no embedding): write-time annotation + reinforcement counter. */
|
|
@@ -43,6 +48,10 @@ export interface SkillNote {
|
|
|
43
48
|
/** Reinforcement count: a duplicate write bumps this instead of duplicating. */
|
|
44
49
|
readonly hits: number;
|
|
45
50
|
readonly ts: number;
|
|
51
|
+
/** A-Mem links — note ids of BM25-nearest neighbors at merge time (bidirectional, ≤LINK_MAX). */
|
|
52
|
+
readonly links?: readonly string[];
|
|
53
|
+
/** Recursion depth of the run that harvested this note (0 = root; child notes crowd less). */
|
|
54
|
+
readonly depth?: number;
|
|
46
55
|
}
|
|
47
56
|
|
|
48
57
|
export interface SkillStateFile {
|
|
@@ -61,6 +70,21 @@ const NOTE_MAX_CHARS = 240;
|
|
|
61
70
|
const CONTEXT_MAX_CHARS = 120;
|
|
62
71
|
const KEYWORDS_MAX = 6;
|
|
63
72
|
|
|
73
|
+
/**
|
|
74
|
+
* A-Mem link/retrieval constants (frozen; cited by tests). Link generation at merge time is
|
|
75
|
+
* A-Mem §3.2 with BM25 in place of embeddings; retrieval expansion (§Fig. 2 "box") pulls a
|
|
76
|
+
* hit's linked notes back at a discounted score. The floor ramp fixes the cold-start no-op:
|
|
77
|
+
* absolute BM25 floors barely clear on a young store, so they halve below COLD_STORE_NOTES,
|
|
78
|
+
* and the relative factor trims the long tail against the corpus-independent top score.
|
|
79
|
+
*/
|
|
80
|
+
export const LINK_TOP_K = 3;
|
|
81
|
+
export const LINK_MAX = 4;
|
|
82
|
+
export const LINK_REL_FLOOR = 0.3;
|
|
83
|
+
export const LINK_EXPANSION_FACTOR = 0.6;
|
|
84
|
+
export const REL_FLOOR_FACTOR = 0.25;
|
|
85
|
+
export const COLD_STORE_NOTES = 8;
|
|
86
|
+
export const GOTCHA_NOTES_MAX = 6;
|
|
87
|
+
|
|
64
88
|
export function skillStatePath(dir?: string): string {
|
|
65
89
|
return join(dir ?? getAgentDir(), SKILL_STATE_FILE);
|
|
66
90
|
}
|
|
@@ -106,15 +130,21 @@ function normalizeTags(tags: readonly string[] | undefined): readonly string[] {
|
|
|
106
130
|
|
|
107
131
|
export function isSkillNote(value: unknown): value is SkillNote {
|
|
108
132
|
if (!isRecord(value)) return false;
|
|
109
|
-
|
|
110
|
-
typeof value.id
|
|
111
|
-
typeof value.text
|
|
112
|
-
isStringArray(value.keywords)
|
|
113
|
-
isStringArray(value.tags)
|
|
114
|
-
typeof value.context
|
|
115
|
-
typeof value.hits
|
|
116
|
-
typeof value.ts
|
|
117
|
-
)
|
|
133
|
+
if (
|
|
134
|
+
typeof value.id !== "string" ||
|
|
135
|
+
typeof value.text !== "string" ||
|
|
136
|
+
!isStringArray(value.keywords) ||
|
|
137
|
+
!isStringArray(value.tags) ||
|
|
138
|
+
typeof value.context !== "string" ||
|
|
139
|
+
typeof value.hits !== "number" ||
|
|
140
|
+
typeof value.ts !== "number"
|
|
141
|
+
) {
|
|
142
|
+
return false;
|
|
143
|
+
}
|
|
144
|
+
// Optional A-Mem fields: absent on pre-link notes (fail-soft read — old files stay valid).
|
|
145
|
+
if (value.links !== undefined && !isStringArray(value.links)) return false;
|
|
146
|
+
if (value.depth !== undefined && typeof value.depth !== "number") return false;
|
|
147
|
+
return true;
|
|
118
148
|
}
|
|
119
149
|
|
|
120
150
|
export function isSkillStateFile(value: unknown): value is SkillStateFile {
|
|
@@ -162,6 +192,8 @@ export interface SkillNoteInput {
|
|
|
162
192
|
readonly keywords?: readonly string[];
|
|
163
193
|
readonly tags?: readonly string[];
|
|
164
194
|
readonly context?: string;
|
|
195
|
+
/** Recursion depth of the harvesting run (0 = root). Recorded, not yet eviction-weighted. */
|
|
196
|
+
readonly depth?: number;
|
|
165
197
|
}
|
|
166
198
|
|
|
167
199
|
const PATH_TOKEN = /(?:[\w@.-]+\/)+[\w@.-]+/g;
|
|
@@ -186,9 +218,11 @@ function keywordsOf(text: string): readonly string[] {
|
|
|
186
218
|
|
|
187
219
|
/**
|
|
188
220
|
* Workstream B Hook 1 (deterministic half): harvest Σ into note inputs — zero extra tokens.
|
|
189
|
-
* verifiedFacts → "symbol" notes; successful approaches → "recipe" notes
|
|
221
|
+
* verifiedFacts → "symbol" notes; successful approaches → "recipe" notes; the most recent
|
|
222
|
+
* failed approaches → "gotcha" notes (the reusable don't-do-this-again material, capped so a
|
|
223
|
+
* failure-heavy run cannot flood the store).
|
|
190
224
|
*/
|
|
191
|
-
export function notesFromRunState(state: RunState): readonly SkillNoteInput[] {
|
|
225
|
+
export function notesFromRunState(state: RunState, depth = 0): readonly SkillNoteInput[] {
|
|
192
226
|
const notes: SkillNoteInput[] = [];
|
|
193
227
|
for (const fact of state.verifiedFacts) {
|
|
194
228
|
if (fact.trim().length < 8) continue;
|
|
@@ -197,6 +231,7 @@ export function notesFromRunState(state: RunState): readonly SkillNoteInput[] {
|
|
|
197
231
|
keywords: keywordsOf(fact),
|
|
198
232
|
tags: ["symbol"],
|
|
199
233
|
context: state.task.slice(0, CONTEXT_MAX_CHARS),
|
|
234
|
+
depth,
|
|
200
235
|
});
|
|
201
236
|
}
|
|
202
237
|
for (const [key, outcome] of Object.entries(state.testedApproaches)) {
|
|
@@ -207,14 +242,37 @@ export function notesFromRunState(state: RunState): readonly SkillNoteInput[] {
|
|
|
207
242
|
keywords: keywordsOf(text),
|
|
208
243
|
tags: ["recipe"],
|
|
209
244
|
context: state.task.slice(0, CONTEXT_MAX_CHARS),
|
|
245
|
+
depth,
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
const failed = Object.entries(state.testedApproaches)
|
|
249
|
+
.filter(([, outcome]) => outcome.status !== "succeeded")
|
|
250
|
+
.slice(-GOTCHA_NOTES_MAX);
|
|
251
|
+
for (const [key, outcome] of failed) {
|
|
252
|
+
const text = `${key} — ${outcomeDetail(outcome)}`.slice(0, NOTE_MAX_CHARS);
|
|
253
|
+
notes.push({
|
|
254
|
+
text,
|
|
255
|
+
keywords: keywordsOf(text),
|
|
256
|
+
tags: ["gotcha"],
|
|
257
|
+
context: state.task.slice(0, CONTEXT_MAX_CHARS),
|
|
258
|
+
depth,
|
|
210
259
|
});
|
|
211
260
|
}
|
|
212
261
|
return notes;
|
|
213
262
|
}
|
|
214
263
|
|
|
215
|
-
/**
|
|
264
|
+
/**
|
|
265
|
+
* A-Mem phrasing prompt (Hook 1, LLM half — enableSkillStateDistill, default ON). Failed and
|
|
266
|
+
* partial approaches ride along: they are usually the most reusable gotcha/recipe material,
|
|
267
|
+
* and feeding only verifiedFacts starved the store of exactly that.
|
|
268
|
+
*/
|
|
216
269
|
export function distillPromptFor(state: RunState): string {
|
|
217
|
-
|
|
270
|
+
const approaches = Object.entries(state.testedApproaches)
|
|
271
|
+
.filter(([, outcome]) => outcome.status !== "succeeded")
|
|
272
|
+
.slice(-8)
|
|
273
|
+
.map(([key, outcome]) =>
|
|
274
|
+
`- ${key} — ${outcomeDetail(outcome)} (${outcome.status})`);
|
|
275
|
+
const lines = [
|
|
218
276
|
"Distill AT MOST 6 durable, reusable project facts from this run. One per line, exactly:",
|
|
219
277
|
"text | kw1, kw2 | tag",
|
|
220
278
|
'tag ∈ {config, gotcha, symbol, recipe}. text ≤240 chars, factual, path-anchored. No preamble.',
|
|
@@ -222,12 +280,24 @@ export function distillPromptFor(state: RunState): string {
|
|
|
222
280
|
`Run task: ${state.task}`,
|
|
223
281
|
"Verified facts:",
|
|
224
282
|
...state.verifiedFacts.slice(-12).map((f) => `- ${f}`),
|
|
225
|
-
]
|
|
283
|
+
];
|
|
284
|
+
if (approaches.length > 0) {
|
|
285
|
+
lines.push("Failed or partial approaches (gotcha candidates — what NOT to retry):", ...approaches);
|
|
286
|
+
}
|
|
287
|
+
if (state.nextStep.trim() !== "") {
|
|
288
|
+
lines.push(`Next step at run end: ${state.nextStep}`);
|
|
289
|
+
}
|
|
290
|
+
return lines.join("\n");
|
|
226
291
|
}
|
|
227
292
|
|
|
228
|
-
/**
|
|
229
|
-
|
|
293
|
+
/**
|
|
294
|
+
* Defensive parser for the distill leaf's output — anything malformed is dropped. `context`
|
|
295
|
+
* (the run task) is supplied by the caller so distilled notes carry a non-empty A-Mem X_i;
|
|
296
|
+
* an empty context weakened BM25 ranking for every LLM-distilled note.
|
|
297
|
+
*/
|
|
298
|
+
export function parseDistilledNotes(raw: string, context = ""): readonly SkillNoteInput[] {
|
|
230
299
|
const out: SkillNoteInput[] = [];
|
|
300
|
+
const contextSlice = context.slice(0, CONTEXT_MAX_CHARS);
|
|
231
301
|
for (const line of raw.split("\n")) {
|
|
232
302
|
const trimmed = line.trim().replace(/^-\s*/, "");
|
|
233
303
|
if (trimmed === "") continue;
|
|
@@ -244,7 +314,12 @@ export function parseDistilledNotes(raw: string): readonly SkillNoteInput[] {
|
|
|
244
314
|
.split(/[\s,]+/)
|
|
245
315
|
.map((t) => t.trim())
|
|
246
316
|
.filter((t) => TAGS.has(t));
|
|
247
|
-
out.push({
|
|
317
|
+
out.push({
|
|
318
|
+
text: text.slice(0, NOTE_MAX_CHARS),
|
|
319
|
+
keywords,
|
|
320
|
+
tags: normalizeTags(tags),
|
|
321
|
+
context: contextSlice,
|
|
322
|
+
});
|
|
248
323
|
if (out.length >= 6) break;
|
|
249
324
|
}
|
|
250
325
|
return out;
|
|
@@ -256,6 +331,53 @@ function noteCorpus(note: SkillNote): string {
|
|
|
256
331
|
return `${note.text} ${note.keywords.join(" ")} ${note.tags.join(" ")} ${note.context}`;
|
|
257
332
|
}
|
|
258
333
|
|
|
334
|
+
/**
|
|
335
|
+
* A-Mem §3.2/§3.3, deterministic (no embeddings, no LLM): each NEW note links its
|
|
336
|
+
* BM25-nearest neighbors above a relative floor (bidirectional, LINK_MAX cap), and each
|
|
337
|
+
* formed link co-reinforces the neighbor (hits bump + ts touch — the store-side analog of
|
|
338
|
+
* A-Mem memory evolution). Pure over its inputs; `now` is the merge timestamp.
|
|
339
|
+
*/
|
|
340
|
+
function linkNewNotes(
|
|
341
|
+
notes: readonly SkillNote[],
|
|
342
|
+
newIds: ReadonlySet<string>,
|
|
343
|
+
now: number,
|
|
344
|
+
): readonly SkillNote[] {
|
|
345
|
+
if (newIds.size === 0 || notes.length < 2) return notes;
|
|
346
|
+
const byId = new Map(notes.map((note) => [note.id, note]));
|
|
347
|
+
const updated = new Map<string, SkillNote>();
|
|
348
|
+
const current = (id: string): SkillNote | undefined => updated.get(id) ?? byId.get(id);
|
|
349
|
+
for (const id of newIds) {
|
|
350
|
+
const note = current(id);
|
|
351
|
+
if (note === undefined) continue;
|
|
352
|
+
const candidates = notes.filter((other) => other.id !== id);
|
|
353
|
+
const ranked = bm25Rank(
|
|
354
|
+
noteCorpus(note),
|
|
355
|
+
candidates.map((other) => ({ item: other, text: noteCorpus(other) })),
|
|
356
|
+
LINK_TOP_K,
|
|
357
|
+
);
|
|
358
|
+
if (ranked.length === 0) continue;
|
|
359
|
+
const top = ranked[0].score;
|
|
360
|
+
const links = [...(note.links ?? [])];
|
|
361
|
+
for (const { item, score } of ranked) {
|
|
362
|
+
if (links.length >= LINK_MAX) break;
|
|
363
|
+
if (score <= 0 || score < LINK_REL_FLOOR * top) continue;
|
|
364
|
+
if (links.includes(item.id)) continue;
|
|
365
|
+
links.push(item.id);
|
|
366
|
+
const neighbor = current(item.id);
|
|
367
|
+
if (neighbor === undefined) continue;
|
|
368
|
+
const neighborLinks = [...(neighbor.links ?? [])];
|
|
369
|
+
if (!neighborLinks.includes(id) && neighborLinks.length < LINK_MAX) {
|
|
370
|
+
neighborLinks.push(id);
|
|
371
|
+
}
|
|
372
|
+
// Co-reinforcement: the neighbor was just confirmed related — evolution, not rewrite.
|
|
373
|
+
updated.set(neighbor.id, { ...neighbor, links: neighborLinks, hits: neighbor.hits + 1, ts: now });
|
|
374
|
+
}
|
|
375
|
+
updated.set(id, { ...note, links });
|
|
376
|
+
}
|
|
377
|
+
if (updated.size === 0) return notes;
|
|
378
|
+
return notes.map((note) => updated.get(note.id) ?? note);
|
|
379
|
+
}
|
|
380
|
+
|
|
259
381
|
export interface SkillSearchHit {
|
|
260
382
|
readonly id: string;
|
|
261
383
|
readonly text: string;
|
|
@@ -293,26 +415,79 @@ export class SkillStore {
|
|
|
293
415
|
return this.file.projects[this.project]?.length ?? 0;
|
|
294
416
|
}
|
|
295
417
|
|
|
418
|
+
/** Tag histogram over this project's notes — ONE summary source (distill card, telemetry). */
|
|
419
|
+
stats(): { readonly notes: number; readonly byTag: Readonly<Record<string, number>> } {
|
|
420
|
+
const notes = this.file.projects[this.project] ?? [];
|
|
421
|
+
const byTag: Record<string, number> = {};
|
|
422
|
+
for (const note of notes) {
|
|
423
|
+
for (const tag of note.tags) byTag[tag] = (byTag[tag] ?? 0) + 1;
|
|
424
|
+
}
|
|
425
|
+
return { notes: notes.length, byTag };
|
|
426
|
+
}
|
|
427
|
+
|
|
296
428
|
/** Test/telemetry seam — the exact on-disk shape a flush would write. */
|
|
297
429
|
snapshot(): SkillStateFile {
|
|
298
430
|
return this.file;
|
|
299
431
|
}
|
|
300
432
|
|
|
301
|
-
/**
|
|
302
|
-
|
|
433
|
+
/**
|
|
434
|
+
* BM25 rank + A-Mem "box" expansion (§Fig. 2): a hit's linked notes ride along at
|
|
435
|
+
* LINK_EXPANSION_FACTOR × the hit's score, deduped — base hits always outrank expansions.
|
|
436
|
+
* Sorted desc; callers slice. The one recall rank for search/pack (DRY).
|
|
437
|
+
*/
|
|
438
|
+
private rankWithLinks(
|
|
439
|
+
query: string,
|
|
440
|
+
k: number,
|
|
441
|
+
): readonly { readonly note: SkillNote; readonly score: number }[] {
|
|
303
442
|
const notes = this.file.projects[this.project] ?? [];
|
|
304
443
|
if (notes.length === 0 || query.trim() === "") return [];
|
|
444
|
+
const byId = new Map(notes.map((note) => [note.id, note]));
|
|
305
445
|
const ranked = bm25Rank(
|
|
306
446
|
query,
|
|
307
447
|
notes.map((note) => ({ item: note, text: noteCorpus(note) })),
|
|
308
|
-
Math.max(1,
|
|
448
|
+
Math.max(1, k),
|
|
309
449
|
);
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
450
|
+
const merged = new Map<string, { note: SkillNote; score: number }>();
|
|
451
|
+
for (const { item, score } of ranked) {
|
|
452
|
+
merged.set(item.id, { note: item, score });
|
|
453
|
+
for (const link of item.links ?? []) {
|
|
454
|
+
if (merged.has(link)) continue;
|
|
455
|
+
const neighbor = byId.get(link);
|
|
456
|
+
if (neighbor !== undefined) {
|
|
457
|
+
merged.set(link, { note: neighbor, score: score * LINK_EXPANSION_FACTOR });
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
const out = [...merged.values()];
|
|
462
|
+
out.sort((a, b) => b.score - a.score);
|
|
463
|
+
return out;
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* Effective acceptance floor: the configured absolute floor (halved on a cold store —
|
|
468
|
+
* absolute BM25 floors barely clear when the corpus is small, so grounding silently no-ops
|
|
469
|
+
* exactly when the store is young) lifted by the relative long-tail cut (REL_FLOOR_FACTOR ×
|
|
470
|
+
* top score). `minScore <= 0` keeps the old inject-anything behavior unchanged.
|
|
471
|
+
*/
|
|
472
|
+
private effectiveFloor(ranked: readonly { readonly score: number }[], minScore: number): number {
|
|
473
|
+
if (minScore <= 0) return 0;
|
|
474
|
+
const ramped = this.noteCount < COLD_STORE_NOTES ? minScore * 0.5 : minScore;
|
|
475
|
+
const top = ranked[0]?.score ?? 0;
|
|
476
|
+
return Math.max(ramped, top * REL_FLOOR_FACTOR);
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
/** Workstream E: the sandbox skill_search surface. Score > 0 hits only, best first. */
|
|
480
|
+
search(query: string, k = 8): readonly SkillSearchHit[] {
|
|
481
|
+
const cap = Math.max(1, Math.min(32, k));
|
|
482
|
+
const ranked = this.rankWithLinks(query, cap);
|
|
483
|
+
return ranked
|
|
484
|
+
.slice(0, cap + LINK_TOP_K) // base hits + their expansions, bounded
|
|
485
|
+
.map(({ note, score }) => ({
|
|
486
|
+
id: note.id,
|
|
487
|
+
text: note.text,
|
|
488
|
+
tags: note.tags,
|
|
489
|
+
score: Math.round(score * 100) / 100,
|
|
490
|
+
}));
|
|
316
491
|
}
|
|
317
492
|
|
|
318
493
|
/** Ξ body lines shared by blockFor/sliceForPrompt — packed greedily under the char budget. */
|
|
@@ -322,18 +497,14 @@ export class SkillStore {
|
|
|
322
497
|
budgetChars: number,
|
|
323
498
|
minScore: number,
|
|
324
499
|
): readonly string[] {
|
|
325
|
-
const
|
|
326
|
-
if (
|
|
327
|
-
const
|
|
328
|
-
query,
|
|
329
|
-
notes.map((note) => ({ item: note, text: noteCorpus(note) })),
|
|
330
|
-
Math.min(k, notes.length),
|
|
331
|
-
);
|
|
500
|
+
const ranked = this.rankWithLinks(query, k);
|
|
501
|
+
if (ranked.length === 0) return [];
|
|
502
|
+
const floor = this.effectiveFloor(ranked, minScore);
|
|
332
503
|
const lines: string[] = [];
|
|
333
504
|
let used = 0;
|
|
334
|
-
for (const {
|
|
335
|
-
if (score <
|
|
336
|
-
const line = `- (${
|
|
505
|
+
for (const { note, score } of ranked) {
|
|
506
|
+
if (score < floor) break; // ranked desc — the first miss ends the window
|
|
507
|
+
const line = `- (${note.tags[0] ?? "symbol"}) ${note.text}`;
|
|
337
508
|
if (used + line.length + 1 > budgetChars) continue; // too fat — try the next, smaller
|
|
338
509
|
lines.push(line);
|
|
339
510
|
used += line.length + 1;
|
|
@@ -343,10 +514,12 @@ export class SkillStore {
|
|
|
343
514
|
|
|
344
515
|
/**
|
|
345
516
|
* Workstream C: the full Ξ block for a root prompt (header via skillStateLines — the single
|
|
346
|
-
* wording source). "" when nothing is relevant or the store is empty.
|
|
517
|
+
* wording source). "" when nothing is relevant or the store is empty. Recall W3: `minScore`
|
|
518
|
+
* gates the block (was `Number.MIN_VALUE` — stale cross-session notes rode every prompt);
|
|
519
|
+
* 0 keeps the old inject-anything behavior.
|
|
347
520
|
*/
|
|
348
|
-
blockFor(query: string, budgetTokens: number): string {
|
|
349
|
-
const lines = this.pack(query, 24, Math.max(0, budgetTokens) * 4,
|
|
521
|
+
blockFor(query: string, budgetTokens: number, minScore: number): string {
|
|
522
|
+
const lines = this.pack(query, 24, Math.max(0, budgetTokens) * 4, minScore);
|
|
350
523
|
if (lines.length === 0) return "";
|
|
351
524
|
return skillStateLines(lines.length, lines.join("\n"));
|
|
352
525
|
}
|
|
@@ -362,13 +535,16 @@ export class SkillStore {
|
|
|
362
535
|
|
|
363
536
|
/**
|
|
364
537
|
* Dedup write: duplicate (by normalized text) bumps `hits` and refreshes `ts`; new notes
|
|
365
|
-
* insert. Per-project cap with LRU-by-ts eviction, top-hits quartile pinned.
|
|
538
|
+
* insert. Per-project cap with LRU-by-ts eviction, top-hits quartile pinned. New notes then
|
|
539
|
+
* get A-Mem §3.2 link generation (BM25-nearest neighbors, bidirectional) with §3.3-style
|
|
540
|
+
* deterministic evolution: linked neighbors co-reinforce (hits bump + ts touch).
|
|
366
541
|
*/
|
|
367
542
|
merge(inputs: readonly SkillNoteInput[]): void {
|
|
368
543
|
if (inputs.length === 0) return;
|
|
369
544
|
const existing = this.file.projects[this.project] ?? [];
|
|
370
545
|
const byId = new Map<string, SkillNote>(existing.map((note) => [note.id, note]));
|
|
371
546
|
const now = Date.now();
|
|
547
|
+
const newIds = new Set<string>();
|
|
372
548
|
for (const input of inputs) {
|
|
373
549
|
const text = input.text.trim();
|
|
374
550
|
if (text === "") continue;
|
|
@@ -398,11 +574,13 @@ export class SkillStore {
|
|
|
398
574
|
context: (input.context ?? "").slice(0, CONTEXT_MAX_CHARS),
|
|
399
575
|
hits: 1,
|
|
400
576
|
ts: now,
|
|
577
|
+
...(input.depth === undefined ? {} : { depth: input.depth }),
|
|
401
578
|
};
|
|
402
579
|
byId.set(id, note);
|
|
580
|
+
newIds.add(id);
|
|
403
581
|
}
|
|
404
582
|
}
|
|
405
|
-
let notes = [...byId.values()];
|
|
583
|
+
let notes: readonly SkillNote[] = [...byId.values()];
|
|
406
584
|
if (notes.length > this.notesPerProject) {
|
|
407
585
|
// Pinned: top quartile by hits (ceil) — frequently-reinforced facts survive eviction.
|
|
408
586
|
const byHits = [...notes].sort((a, b) => b.hits - a.hits || b.ts - a.ts);
|
|
@@ -415,6 +593,7 @@ export class SkillStore {
|
|
|
415
593
|
const evict = new Set(evictable.slice(0, notes.length - this.notesPerProject).map((note) => note.id));
|
|
416
594
|
notes = notes.filter((note) => !evict.has(note.id));
|
|
417
595
|
}
|
|
596
|
+
notes = linkNewNotes(notes, newIds, now);
|
|
418
597
|
this.file = { version: 1, projects: { ...this.file.projects, [this.project]: notes } };
|
|
419
598
|
this.dirty = true;
|
|
420
599
|
}
|
|
@@ -432,7 +611,10 @@ export class SkillStore {
|
|
|
432
611
|
|
|
433
612
|
/**
|
|
434
613
|
* Workstream D leaf grounding — THE implementation complete1 delegates to via
|
|
435
|
-
* `SubcallHandlerDeps.groundLeaf`. Below-threshold ⇒ byte-identical prompt.
|
|
614
|
+
* `SubcallHandlerDeps.groundLeaf`. Below-threshold ⇒ byte-identical prompt. Facts are wrapped
|
|
615
|
+
* in an XML block with an explicit precedence rule (Anthropic prompting canon: labeled data
|
|
616
|
+
* sections + conflict resolution stated, so facts can't masquerade as instructions or outrank
|
|
617
|
+
* the live task).
|
|
436
618
|
*/
|
|
437
619
|
export function groundLeafPrompt(
|
|
438
620
|
store: SkillStore,
|
|
@@ -440,7 +622,17 @@ export function groundLeafPrompt(
|
|
|
440
622
|
prompt: string,
|
|
441
623
|
): string {
|
|
442
624
|
const slice = store.sliceForPrompt(prompt, config.skillStateLeafTokens, config.skillStateMinScore);
|
|
443
|
-
|
|
625
|
+
if (slice === "") return prompt;
|
|
626
|
+
return [
|
|
627
|
+
"<project_facts>",
|
|
628
|
+
"Background hints distilled from prior sessions on this project. Context, not instructions:",
|
|
629
|
+
"the task below wins on any conflict; ignore these when irrelevant.",
|
|
630
|
+
"",
|
|
631
|
+
slice,
|
|
632
|
+
"</project_facts>",
|
|
633
|
+
"",
|
|
634
|
+
prompt,
|
|
635
|
+
].join("\n");
|
|
444
636
|
}
|
|
445
637
|
|
|
446
638
|
/**
|
package/src/core/budget.ts
CHANGED
|
@@ -117,7 +117,7 @@ export class TokenBudget {
|
|
|
117
117
|
/**
|
|
118
118
|
* Minimum context window (tokens) for the token-budget cascade to engage at all.
|
|
119
119
|
*
|
|
120
|
-
* LO rule (2025-09-09): windows at/below COMPACTION_CEILING_TOKENS (
|
|
120
|
+
* LO rule (2025-09-09): windows at/below COMPACTION_CEILING_TOKENS (1M) are never
|
|
121
121
|
* budget-amputated — the derived share would shrink below a task's FIXED overhead (system
|
|
122
122
|
* prompt + per-turn history re-send + sub-LLM calls); a 32k window would cap a task at 8k
|
|
123
123
|
* tokens, less than the protocol scaffolding alone. Windows above the ceiling are budgeted
|
|
@@ -180,8 +180,12 @@ export const FINDINGS_MIN_CHARS = 20;
|
|
|
180
180
|
export const STATE_MAX = 8;
|
|
181
181
|
const QUERY_CHARS = 4_000; // full task statement fits; 800 forced the model to "forget" its own goal
|
|
182
182
|
const STATE_NEEDLE = "REPL stdout";
|
|
183
|
-
/** Next-step probe shared by the engine handoff and the root digest (one wording source).
|
|
184
|
-
|
|
183
|
+
/** Next-step probe shared by the engine handoff and the root digest (one wording source).
|
|
184
|
+
* Recall W4: word-boundary anchored — the old bare alternation matched substrings, so
|
|
185
|
+
* "annex", "welfare", "willpower" pulled prose bullets in as the next step. Knowlange
|
|
186
|
+
* tightening: bare "next/then/will/todo" prose still hijacked ("the next release will…"),
|
|
187
|
+
* so only explicit step shapes match now — colon labels, "next step", or commitments. */
|
|
188
|
+
export const NEXT_STEP_RE = /\b(?:(?:next|then|todo)\s*:|next step\b|(?:i|we)\s+(?:will|'ll|’ll)\b)/i;
|
|
185
189
|
|
|
186
190
|
/**
|
|
187
191
|
* Deterministic trajectory → handoff (v5 `distill_trajectory`). No LLM call: the model was
|
package/src/core/compaction.ts
CHANGED
|
@@ -31,8 +31,8 @@ interface CompactionDeps {
|
|
|
31
31
|
|
|
32
32
|
/**
|
|
33
33
|
* True if the history is at/over the compaction threshold — the ABSOLUTE
|
|
34
|
-
* COMPACTION_CEILING_TOKENS (LO rule 2025-09-09): windows ≤
|
|
35
|
-
* windows compact exactly at
|
|
34
|
+
* COMPACTION_CEILING_TOKENS (LO rule 2025-09-09): windows ≤ 1M never compact; larger
|
|
35
|
+
* windows compact exactly at 1M. `contextWindow`/`thresholdPct` percentage math is gone.
|
|
36
36
|
*/
|
|
37
37
|
export function shouldCompact(history: ChatMsg[]): boolean {
|
|
38
38
|
return estimateMessageTokens(history) >= COMPACTION_CEILING_TOKENS;
|