@hicaru/pi-rlm 0.3.19 → 0.3.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +8 -5
  2. package/package.json +1 -1
  3. package/src/bridge/handlers/completion.ts +3 -0
  4. package/src/bridge/handlers/emitting.ts +0 -4
  5. package/src/bridge/handlers/rlm-query.ts +3 -3
  6. package/src/bridge/handlers/task-registry.ts +46 -19
  7. package/src/bridge/handlers/types.ts +6 -3
  8. package/src/bridge/model.ts +4 -0
  9. package/src/commands/rlm.ts +14 -7
  10. package/src/config/defaults.ts +41 -13
  11. package/src/config/settings.ts +7 -3
  12. package/src/config/skillstate.ts +236 -44
  13. package/src/context/merge.ts +10 -3
  14. package/src/context/namespace.ts +6 -2
  15. package/src/context/refresh.ts +32 -11
  16. package/src/core/answer.ts +15 -0
  17. package/src/core/budget.ts +39 -17
  18. package/src/core/compaction.ts +85 -9
  19. package/src/core/engine.ts +117 -32
  20. package/src/core/iteration.ts +4 -0
  21. package/src/core/limits.ts +10 -14
  22. package/src/core/root-context.ts +83 -19
  23. package/src/core/root-digest.ts +48 -11
  24. package/src/core/root-state.ts +39 -12
  25. package/src/core/run-state.ts +86 -14
  26. package/src/core/session-archive.ts +174 -0
  27. package/src/core/types.ts +13 -2
  28. package/src/index.ts +142 -12
  29. package/src/mode/rlm-mode.ts +2 -2
  30. package/src/prompts/glossary.ts +36 -5
  31. package/src/prompts/native.ts +8 -2
  32. package/src/prompts/user.ts +6 -4
  33. package/src/sandbox/py/__pycache__/retrieval.cpython-314.pyc +0 -0
  34. package/src/sandbox/py/__pycache__/scaffold.cpython-314.pyc +0 -0
  35. package/src/sandbox/py/retrieval.py +202 -36
  36. package/src/sandbox/py/scaffold.py +20 -5
  37. package/src/sandbox/py/worker.py +1 -1
  38. package/src/sandbox/sandbox-manager.ts +19 -0
  39. package/src/sandbox/sandbox.ts +13 -1
  40. package/src/text/parsing.ts +133 -2
  41. package/src/text/tokens.ts +39 -4
  42. package/src/tool/repl-details.ts +2 -2
  43. package/src/tool/repl-render.ts +38 -2
  44. package/src/tool/repl-tool.ts +37 -23
  45. package/src/tool/rlm-aggregator.ts +1 -1
  46. package/src/tool/rlm-details.ts +1 -2
  47. package/src/tool/rlm-events.ts +3 -6
  48. package/src/tool/rlm-tool.ts +1 -1
  49. package/src/tool/subcall-render.ts +7 -4
  50. package/src/tool/subcall-store.ts +5 -18
  51. package/src/ui/config-panel.ts +4 -19
  52. package/src/ui/intro.ts +1 -1
  53. package/src/ui/panel/run-registry.ts +2 -2
  54. package/src/ui/python-highlight.ts +49 -0
  55. package/src/ui/stage-cards.ts +192 -0
  56. package/src/ui/tree/tree-model.ts +69 -19
  57. package/src/ui/tree/tree-rows.ts +2 -1
  58. package/src/util/abort.ts +34 -0
  59. package/src/util/bm25.ts +170 -21
  60. package/src/util/errors.ts +1 -1
@@ -3,19 +3,20 @@
3
3
  * (ported from rlm/core/rlm.py `_check_timeout` / `_check_iteration_limits`). Any breach throws
4
4
  * a LimitError; the engine catches it and returns the best partial answer it has.
5
5
  *
6
- * Cost is tracked for reporting only — there is no USD spend ceiling.
6
+ * Cost is NOT tracked: there is no USD spend ceiling and no cost reporting.
7
7
  */
8
8
 
9
9
  import type { Usage } from "@earendil-works/pi-ai";
10
10
 
11
11
  /**
12
- * Absolute working ceiling (LO rule 2025-09-09): model windows AT/BELOW this value are never
13
- * compacted or budget-amputated — the agent runs its full window. Windows ABOVE it are
14
- * compacted/budgeted exactly AT the ceiling (e.g. a 1M-context model works up to ~256k tokens
15
- * of history, then rebases/compacts). Single source of truth for budget.ts (resolveBudget)
16
- * and compaction.ts (shouldCompact + rebaseWithState token bound).
12
+ * Absolute working ceiling (LO rule 2025-09-09, raised 2025-09-10): model windows AT/BELOW
13
+ * this value are never compacted or budget-amputated — the agent runs its full window, and
14
+ * tree-wide spend (root turns + sub-LLM calls) is NOT metered against it. Windows ABOVE it
15
+ * are budgeted exactly AT the ceiling an outlier context-fit guard, not a cost meter.
16
+ * Single source of truth for budget.ts (resolveBudget) and compaction.ts (shouldCompact +
17
+ * rebaseWithState token bound).
17
18
  */
18
- export const COMPACTION_CEILING_TOKENS = 256_000;
19
+ export const COMPACTION_CEILING_TOKENS = 1_000_000;
19
20
 
20
21
  export interface Limits {
21
22
  readonly maxTimeoutMs?: number;
@@ -36,7 +37,6 @@ export function limitsFromConfig(config: Limits): Limits {
36
37
  interface UsageSnapshot {
37
38
  readonly inputTokens: number;
38
39
  readonly outputTokens: number;
39
- readonly costUsd: number;
40
40
  readonly durationMs: number;
41
41
  }
42
42
 
@@ -54,7 +54,6 @@ export class LimitGuard {
54
54
  private start: number;
55
55
  private inputTokens = 0;
56
56
  private outputTokens = 0;
57
- private costUsd = 0;
58
57
  private consecutiveErrors = 0;
59
58
 
60
59
  constructor(private readonly limits: Limits = {}, seedElapsedMs = 0) {
@@ -73,12 +72,10 @@ export class LimitGuard {
73
72
  addUsage(usage: Usage): void {
74
73
  this.inputTokens += usage.input;
75
74
  this.outputTokens += usage.output;
76
- this.costUsd += usage.cost.total;
77
75
  }
78
76
 
79
- /** Fold a recursive child run's total cost/tokens into this guard. */
80
- addRaw(costUsd: number, inputTokens: number, outputTokens: number): void {
81
- this.costUsd += costUsd;
77
+ /** Fold a recursive child run's total tokens into this guard. */
78
+ addRaw(inputTokens: number, outputTokens: number): void {
82
79
  this.inputTokens += inputTokens;
83
80
  this.outputTokens += outputTokens;
84
81
  }
@@ -99,7 +96,6 @@ export class LimitGuard {
99
96
  return {
100
97
  inputTokens: this.inputTokens,
101
98
  outputTokens: this.outputTokens,
102
- costUsd: this.costUsd,
103
99
  durationMs: Date.now() - this.start,
104
100
  };
105
101
  }
@@ -9,7 +9,9 @@
9
9
  * array) to honor the zero-extra-allocations rule.
10
10
  *
11
11
  * - `elideStalePayloads` — discard semantics (paper §5.3): tool payloads older than the
12
- * keep window become head+tail previews; the full bytes remain in the session log.
12
+ * keep window become head+tail previews; the full bytes ride the optional `onElide`
13
+ * sink into the SessionArchive (recall W1) so elision stays dereferenceable — search()
14
+ * over `ctx/session-log/*` recalls what the stub replaced.
13
15
  * - `spliceSigmaSnapshot` — the fresh Σ snapshot rides immediately before the last user
14
16
  * message, exactly one instance (previous ones are removed — idempotent per call).
15
17
  *
@@ -19,7 +21,13 @@
19
21
  import type { ContextEvent } from "@earendil-works/pi-coding-agent";
20
22
  import type { RunState } from "./run-state.ts";
21
23
  import { runStateRootBlock } from "./run-state.ts";
22
- import { ROOT_TURN_ELIDED_LINE } from "../prompts/glossary.ts";
24
+ import {
25
+ ROOT_ELIDE_PREVIEW_MARK,
26
+ ROOT_ELIDE_PREVIEW_MARK_REPL,
27
+ ROOT_TURN_ELIDED_ARCHIVE_LINE,
28
+ ROOT_TURN_ELIDED_LINE,
29
+ ROOT_TURN_ELIDED_PLAIN_LINE,
30
+ } from "../prompts/glossary.ts";
23
31
  import { truncateOutput } from "../text/parsing.ts";
24
32
  import { textContentOf } from "../text/agent-text.ts";
25
33
 
@@ -29,9 +37,43 @@ export type RootMessage = ContextEvent["messages"][number];
29
37
  export interface ElideOptions {
30
38
  readonly keepTurns: number;
31
39
  readonly elideChars: number;
40
+ /** When true (SessionArchive wired + enabled), stubs point at the searchable archive;
41
+ * when false they stay honest by promising less (no phantom recovery channel). */
42
+ readonly archiveActive?: boolean;
32
43
  }
33
44
 
34
- const ELIDE_MARK = "chars elidedfull result in session log";
45
+ /** One message the elision is about to destroy the SessionArchive record shape. */
46
+ export interface ElidedEntry {
47
+ readonly role: "assistant" | "toolResult";
48
+ readonly toolName: string | undefined;
49
+ /** The FULL original text (pre-preview, pre-stub) — the archive exists to hold it. */
50
+ readonly text: string;
51
+ }
52
+ export type ElideSink = (entry: ElidedEntry) => void;
53
+
54
+ const REPL_TOOL = "repl";
55
+
56
+ function elidedLineFor(role: string, toolName: string | undefined, archiveActive: boolean): string {
57
+ if (role === "toolResult" && toolName === REPL_TOOL) return ROOT_TURN_ELIDED_LINE;
58
+ return archiveActive ? ROOT_TURN_ELIDED_ARCHIVE_LINE : ROOT_TURN_ELIDED_PLAIN_LINE;
59
+ }
60
+
61
+ /** Normalized `archiveActive` — optional knob, resolved once per elide walk. */
62
+ function archiveFlag(opts: ElideOptions): boolean {
63
+ return opts.archiveActive === true;
64
+ }
65
+
66
+ function previewMarkFor(toolName: string | undefined, archiveActive: boolean): string {
67
+ if (toolName === REPL_TOOL) return ROOT_ELIDE_PREVIEW_MARK_REPL;
68
+ // Recall W1: the archive mark may only promise what the archive actually holds.
69
+ return archiveActive ? ROOT_ELIDE_PREVIEW_MARK : "chars elided";
70
+ }
71
+
72
+ /** Narrow the toolName field off a toolResult message (indexed — host evolution safe). */
73
+ function toolNameOf(message: RootMessage): string | undefined {
74
+ const name: unknown = (message as { toolName?: unknown }).toolName;
75
+ return typeof name === "string" ? name : undefined;
76
+ }
35
77
 
36
78
  /**
37
79
  * WS-3a: elide stale turns. The newest `keepTurns` assistant turns and the final user message
@@ -40,12 +82,18 @@ const ELIDE_MARK = "chars elided — full result in session log";
40
82
  * - recent stale ring (the last `max(keepTurns, 1)` stale turns): toolResult payloads over
41
83
  * `elideChars` become head+tail previews (same truncation shape as repl stdout); assistant
42
84
  * prose always collapses to the one-line stub.
43
- * - older still: EVERYTHING (payloads included) becomes the one-line session-log stub —
44
- * a stale preview per turn would itself accumulate linearly and re-create O(T).
85
+ * - older still: EVERYTHING (payloads included) becomes the one-line stub — a stale preview
86
+ * per turn would itself accumulate linearly and re-create O(T).
87
+ * When `onElide` is provided, every destroyed message's FULL text is handed to the sink
88
+ * BEFORE the stub replaces it (recall W1: the archive keeps elision dereferenceable).
45
89
  * `role:"custom"` messages of the Σ/intro kinds are immune (WS-3b owns them). Mutates the
46
90
  * array in place; returns the number of messages elided (telemetry), for zero-cost counters.
47
91
  */
48
- export function elideStalePayloads(messages: RootMessage[], opts: ElideOptions): number {
92
+ export function elideStalePayloads(
93
+ messages: RootMessage[],
94
+ opts: ElideOptions,
95
+ onElide?: ElideSink,
96
+ ): number {
49
97
  const keepTurns = Math.max(0, Math.floor(opts.keepTurns));
50
98
  if (messages.length === 0) return 0;
51
99
  // Preview ring: stale turns recent enough to deserve the §5.3 head+tail preview. Scales with
@@ -55,7 +103,7 @@ export function elideStalePayloads(messages: RootMessage[], opts: ElideOptions):
55
103
  if (keepTurns === 0) {
56
104
  // R5 strict-0: nothing inside the window — Σ + immune customs + the final user message
57
105
  // are all that survive verbatim.
58
- return elideRange(messages, 0, messages.length, opts, previewRing);
106
+ return elideRange(messages, 0, messages.length, opts, previewRing, onElide);
59
107
  }
60
108
 
61
109
  // Index of the assistant message that opens the keepTurns-th-from-last turn — everything
@@ -72,7 +120,7 @@ export function elideStalePayloads(messages: RootMessage[], opts: ElideOptions):
72
120
  }
73
121
  }
74
122
  if (tailStart <= 0) return 0; // fewer turns than the window — nothing to elide
75
- return elideRange(messages, 0, tailStart, opts, previewRing);
123
+ return elideRange(messages, 0, tailStart, opts, previewRing, onElide);
76
124
  }
77
125
 
78
126
  /** Custom messages the elision never touches — the Σ snapshot/observation and the intro. */
@@ -94,6 +142,7 @@ function elideRange(
94
142
  to: number,
95
143
  opts: ElideOptions,
96
144
  previewRing: number,
145
+ onElide: ElideSink | undefined,
97
146
  ): number {
98
147
  const lastUser = lastIndexOfRole(messages, "user");
99
148
  // Pre-pass: stale assistant indices in [from, to) — a payload's recency is measured by the
@@ -109,21 +158,40 @@ function elideRange(
109
158
  if (i === lastUser) continue; // paranoia: the final user message is never touched
110
159
  if (m.role === "assistant") {
111
160
  staleAssistants -= 1; // turns AFTER this one = count minus itself
112
- messages[i] = { ...m, content: [{ type: "text", text: ROOT_TURN_ELIDED_LINE }] } as RootMessage;
161
+ const prose = textContentOf(m.content);
162
+ onElide?.({ role: "assistant", toolName: undefined, text: prose });
163
+ // Provider pairing invariant: Anthropic requires every tool_result to follow the
164
+ // assistant message carrying its tool_use; OpenAI requires each tool/function_call_output
165
+ // to pair with its tool_call/function_call. Eliding the toolCall blocks here orphaned the
166
+ // surviving toolResults and broke the request. Elide only the prose — keep toolCall blocks.
167
+ const toolCalls = Array.isArray(m.content)
168
+ ? (m.content as Array<{ type?: string }>).filter((b) => b?.type === "toolCall")
169
+ : [];
170
+ const line = elidedLineFor("assistant", undefined, archiveFlag(opts));
171
+ const content: unknown[] =
172
+ toolCalls.length > 0 ? [...toolCalls, { type: "text", text: line }] : [{ type: "text", text: line }];
173
+ messages[i] = { ...m, content } as RootMessage;
113
174
  elided += 1;
114
175
  continue;
115
176
  }
116
177
  if (m.role !== "toolResult") continue;
178
+ const toolName = toolNameOf(m);
179
+ const fullText = textContentOf(m.content);
117
180
  if (staleAssistants >= previewRing) {
118
181
  // Deep-stale payload: even the preview would accumulate — collapse to the stub.
119
- messages[i] = { ...m, content: [{ type: "text", text: ROOT_TURN_ELIDED_LINE }] } as RootMessage;
182
+ onElide?.({ role: "toolResult", toolName, text: fullText });
183
+ messages[i] = {
184
+ ...m,
185
+ content: [{ type: "text", text: elidedLineFor("toolResult", toolName, archiveFlag(opts)) }],
186
+ } as RootMessage;
120
187
  elided += 1;
121
188
  continue;
122
189
  }
123
- if (totalTextLength(m) <= opts.elideChars) continue;
190
+ if (fullText.length <= opts.elideChars) continue;
191
+ onElide?.({ role: "toolResult", toolName, text: fullText });
124
192
  messages[i] = {
125
193
  ...m,
126
- content: [{ type: "text", text: previewToolText(m, opts.elideChars) }],
194
+ content: [{ type: "text", text: previewToolText(fullText, opts.elideChars, toolName, archiveFlag(opts)) }],
127
195
  } as RootMessage;
128
196
  elided += 1;
129
197
  }
@@ -173,12 +241,8 @@ function lastIndexOfRole(messages: readonly RootMessage[], role: "user"): number
173
241
  return -1;
174
242
  }
175
243
 
176
- function totalTextLength(message: RootMessage): number {
177
- return textContentOf((message as { content?: unknown }).content).length;
178
- }
179
-
180
- function previewToolText(message: RootMessage, elideChars: number): string {
181
- const text = textContentOf((message as { content?: unknown }).content);
244
+ /** Preview truncation moved the FULL text out to the caller first — this shapes what stays. */
245
+ function previewToolText(fullText: string, elideChars: number, toolName: string | undefined, archiveActive: boolean): string {
182
246
  // The preview REPLACES the payload, so `elideChars` budgets the WHOLE head+tail result.
183
- return truncateOutput(text, Math.max(100, elideChars), ELIDE_MARK);
247
+ return truncateOutput(fullText, Math.max(100, elideChars), previewMarkFor(toolName, archiveActive));
184
248
  }
@@ -25,9 +25,11 @@ import type { CompactionResult, SessionEntry } from "@earendil-works/pi-coding-a
25
25
  import type { RlmConfig } from "./types.ts";
26
26
  import { DEFAULT_NEXT_STEP, FINDINGS_MAX, FINDINGS_MIN_CHARS, NEXT_STEP_RE, STATE_MAX, truncateMid } from "./budget.ts";
27
27
  import { estimateMessageTokens } from "../text/tokens.ts";
28
+ import { bm25Rank } from "../util/bm25.ts";
28
29
  import { agentMessageText } from "../text/agent-text.ts";
29
30
  import { isRecord } from "../util/type-guards.ts";
30
31
  import { ROOT_DIGEST_HEADER, ROOT_DIGEST_SECTIONS } from "../prompts/glossary.ts";
32
+ import { trace, traceEnabled } from "../util/trace.ts";
31
33
 
32
34
  /** Marker persisted on the session's compaction entry — tests and soaks assert on it. */
33
35
  export interface RootDigestDetails {
@@ -106,17 +108,47 @@ function taskSection(messages: readonly unknown[]): string {
106
108
  return "";
107
109
  }
108
110
 
109
- /** [Findings] — newest-first substantive assistant blobs, capped, then chronological. */
110
- function findingsSection(messages: readonly unknown[]): readonly string[] {
111
- const findings: string[] = [];
112
- for (let i = messages.length - 1; i >= 0 && findings.length < FINDINGS_MAX; i--) {
111
+ /** [Findings] — hybrid selection, then chronological render. Candidates keep the dedup
112
+ * discipline (recall W4: exact + 80-char prefix), but the final pick is no longer pure
113
+ * recency: each candidate scores `BM25(task) normalized + recency share`, so load-bearing
114
+ * findings outrank newer rambles while recency breaks ties (and keeps no-overlap corpora
115
+ * byte-compatible with the old newest-first behavior). */
116
+ const FINDING_CANDIDATES_MAX = 40;
117
+
118
+ function findingsSection(messages: readonly unknown[], task: string): readonly string[] {
119
+ const candidates: string[] = [];
120
+ const seenExact = new Set<string>();
121
+ const seenPrefix = new Set<string>();
122
+ for (let i = messages.length - 1; i >= 0 && candidates.length < FINDING_CANDIDATES_MAX; i--) {
113
123
  const m = messages[i];
114
124
  if (!isRecord(m) || m.role !== "assistant") continue;
115
125
  const text = agentMessageText(m).trim();
116
- if (text.length > FINDINGS_MIN_CHARS) findings.push(text);
126
+ if (text.length <= FINDINGS_MIN_CHARS) continue;
127
+ const exactKey = text.toLowerCase().replace(/\s+/g, " ");
128
+ const prefixKey = exactKey.slice(0, 80);
129
+ if (seenExact.has(exactKey) || seenPrefix.has(prefixKey)) continue;
130
+ seenExact.add(exactKey);
131
+ seenPrefix.add(prefixKey);
132
+ candidates.push(text);
117
133
  }
118
- findings.reverse();
119
- return findings;
134
+ if (candidates.length <= FINDINGS_MAX) {
135
+ candidates.reverse();
136
+ return candidates;
137
+ }
138
+ // Hybrid rank: BM25 relevance to the live task + recency share (newest ≈ 1, oldest ≈ 0).
139
+ // `candidates` is collected newest-first, so recency decays with the index.
140
+ const ranked = bm25Rank(task, candidates.map((text) => ({ item: text, text })), candidates.length);
141
+ const relevance = new Map<string, number>();
142
+ const maxRel = ranked[0]?.score ?? 0;
143
+ for (const { item, score } of ranked) relevance.set(item, maxRel > 0 ? score / maxRel : 0);
144
+ const n = candidates.length;
145
+ const scored = candidates.map((text, i) => ({
146
+ text,
147
+ hybrid: (relevance.get(text) ?? 0) + (n - i) / n,
148
+ }));
149
+ scored.sort((a, b) => b.hybrid - a.hybrid);
150
+ const picked = new Set(scored.slice(0, FINDINGS_MAX).map((s) => s.text));
151
+ return candidates.filter((text) => picked.has(text));
120
152
  }
121
153
 
122
154
  /** [State] — newest-first toolResult first-lines (the last observed machinery state). */
@@ -163,10 +195,10 @@ export function buildRootDigestCompaction(
163
195
 
164
196
  const max = Math.max(200, args.config.rootDigestMaxChars);
165
197
  const task = truncateMid(taskSection(messages), Math.floor(max * TASK_FRACTION));
166
- const findings = findingsSection(messages).map((f) => truncateMid(f, BULLET_CHARS));
198
+ const findings = findingsSection(messages, task).map((f) => truncateMid(f, BULLET_CHARS));
167
199
  const states = stateSection(messages);
168
200
  // Next-step probe mirrors distillTrajectory: newest-first scan, chronological render.
169
- const next = findings.find((f) => NEXT_STEP_RE.test(f)) ?? DEFAULT_NEXT_STEP;
201
+ const next = [...findings].reverse().find((f) => NEXT_STEP_RE.test(f)) ?? DEFAULT_NEXT_STEP;
170
202
  const facts = args.store === undefined
171
203
  ? ""
172
204
  : args.store.sliceForPrompt(
@@ -190,12 +222,17 @@ export function buildRootDigestCompaction(
190
222
 
191
223
  // Over-cap drop order: [Project facts] → [State] → [Findings]; [Task]/[Next] survive to a
192
224
  // final truncate — task continuity beats trivia (paper §3.1 sufficient statistic).
225
+ // Recall W4: drops are traced (they were invisible before — a fat facts section could
226
+ // silently eat the entire [State] continuity floor).
227
+ let dropped = "";
193
228
  let summary = render(facts, states, findings);
194
- if (summary.length > max) summary = render("", states, findings);
195
- if (summary.length > max) summary = render("", [], findings);
229
+ if (summary.length > max) { summary = render("", states, findings); dropped = "facts"; }
230
+ if (summary.length > max) { summary = render("", [], findings); dropped += " state"; }
196
231
  if (summary.length > max) {
197
232
  summary = render("", [], findings.slice(0, Math.max(1, Math.floor(findings.length / 2))));
233
+ dropped += " findings-half";
198
234
  }
235
+ if (dropped !== "" && traceEnabled) trace("root-digest.drop", { dropped, max, finalChars: summary.length });
199
236
  summary = truncateMid(summary, max);
200
237
 
201
238
  const boundary = cut >= 0 ? args.branchEntries[cut] : undefined;
@@ -124,14 +124,17 @@ export class RootStateTracker {
124
124
  }
125
125
 
126
126
  noteFinding(text: string): void {
127
- const trimmed = text.trim();
127
+ // Runtime-harvested findings are clamped at the source — fence-sourced ones arrive ≤120
128
+ // via the patch clamp; this is the defense for direct feeds (engine mirrors arrive capped).
129
+ const trimmed = text.trim().slice(0, 300);
128
130
  if (trimmed === "") return;
129
131
  this.draft.findings = dedupStrings([...this.draft.findings, trimmed]).slice(-RUN_STATE_LIMITS.findings);
130
132
  this.touch();
131
133
  }
132
134
 
133
135
  noteFact(text: string): void {
134
- const trimmed = text.trim();
136
+ // Clamp before dedup (recall W2): a verbose runtime feed must not eat the κ_Σ budget.
137
+ const trimmed = text.trim().slice(0, 200);
135
138
  if (trimmed === "") return;
136
139
  this.draft.verifiedFacts = dedupStrings([...this.draft.verifiedFacts, trimmed])
137
140
  .slice(-RUN_STATE_LIMITS.verifiedFacts);
@@ -150,17 +153,27 @@ export class RootStateTracker {
150
153
  this.touch();
151
154
  }
152
155
 
153
- /** Tool-outcome feed (WS-4 v1 source): failures become approach outcomes, verbatim. */
156
+ /** Tool-outcome feed (WS-4 v1 source; recall W2 pollution fix): a SINGLE transient failure
157
+ * (binary read, typo'd path) no longer writes a `testedApproaches` record — that record
158
+ * evicted real approach entries and fired spurious [rectify] hints. Failures always grow
159
+ * the rectify counter, but only PROMOTE to Σ once the failure repeats (threshold parity)
160
+ * or a record already exists (then the record refreshes, keeping recency truthful). */
154
161
  observeToolResult(toolName: string, isError: boolean, reasonFirstLine: string): void {
162
+ const key = `tool:${toolName}`;
155
163
  if (isError) {
156
- this.noteOutcome(`tool:${toolName}`, { status: "failed", reason: reasonFirstLine });
164
+ const streak = (this.failures.get(key) ?? 0) + 1;
165
+ this.failures.set(key, streak);
166
+ if (streak >= RECTIFY_FAILURE_THRESHOLD || this.draft.testedApproaches[key] !== undefined) {
167
+ this.noteOutcome(key, { status: "failed", reason: reasonFirstLine });
168
+ }
157
169
  } else {
158
170
  // A success clears the tool's failure streak — the state reflects the newest truth.
159
- if (this.draft.testedApproaches[`tool:${toolName}`] !== undefined) {
160
- this.noteOutcome(`tool:${toolName}`, { status: "succeeded", evidence: reasonFirstLine });
161
- } else {
162
- this.failures.delete(`tool:${toolName}`);
171
+ if (this.draft.testedApproaches[key] !== undefined) {
172
+ const failures = this.failures.get(key) ?? 0;
173
+ const evidence = failures > 0 ? `recovered after ${failures} failure(s)` : reasonFirstLine;
174
+ this.noteOutcome(key, { status: "succeeded", evidence });
163
175
  }
176
+ this.failures.delete(key);
164
177
  }
165
178
  }
166
179
 
@@ -204,14 +217,27 @@ export class RootStateTracker {
204
217
  * see the const's soak citation). Any accepted delta resets the streak. In
205
218
  * degraded mode fences stop applying and the context transform stops splicing (`isActive`),
206
219
  * while runtime `observeToolResult` remains the Σ floor (degrade, never crash).
220
+ *
221
+ * Recall W2 amendments (root-only, engine ladder untouched):
222
+ * - `productiveTurn` — a turn that RAN TOOLS did work; growing the idle streak over it
223
+ * degraded file-editing sessions whose turns legitimately need no fence. Productive
224
+ * turns neither grow nor reset the streak (neutral).
225
+ * - Retry accounting — a batch that ACCEPTED deltas does not accumulate its problems into
226
+ * the session-wide retry counter (real work is never punished for a sibling's malformed
227
+ * fence — the observation still reports them); the counter only grows over zero-progress
228
+ * batches, so the storm cap stays a storm cap, not a session-lifetime budget.
207
229
  */
208
- applyFences(fences: readonly StateFenceResult[]): FenceOutcome {
230
+ applyFences(fences: readonly StateFenceResult[], opts?: { readonly productiveTurn?: boolean }): FenceOutcome {
209
231
  // R7-fix (recoverable degrade): a DEGRADED tracker no longer drops fences on the floor.
210
232
  // The old early-return turned idle degrade into a one-way amnesia valve — every later
211
233
  // fence vanished silently while the context transform kept eliding turns. Now the same
212
234
  // validation ladder runs in degraded mode, and a CLEAN batch re-activates compensation.
213
235
  // Degrade stays sticky only against zero-progress storms (all-malformed batches).
214
236
  if (fences.length === 0) {
237
+ if (opts?.productiveTurn === true) {
238
+ // Recall W2: tool-running turns are work, not fence idleness — neutral, no degrade.
239
+ return { fences: 0, accepted: 0, problems: 0 };
240
+ }
215
241
  // R4 (G6): a fence-free turn on a conditioned loop is IDLE — the contract rode the
216
242
  // prompt for nothing. Grow the streak; degrade at the engine's threshold.
217
243
  this.idleFenceTurns += 1;
@@ -247,9 +273,10 @@ export class RootStateTracker {
247
273
  this.touch();
248
274
  return { fences: fences.length, accepted, problems: 0 };
249
275
  }
250
- // Degraded variant carries `reason`, not `retries` recovery is clean-batch-only, so a
251
- // degraded tracker neither accumulates retries nor re-activates on a partial batch.
252
- const retries = (this.mode.kind === "active" ? this.mode.retries : 0) + problems.length;
276
+ // Recall W2: a productive batch (accepted > 0) starts its retry count at zero instead of
277
+ // accumulating the session's history the storm cap then measures CONSECUTIVE failure
278
+ // storms, not session age. Zero-progress batches accumulate exactly as before.
279
+ const retries = accepted > 0 ? problems.length : (this.mode.kind === "active" ? this.mode.retries : 0) + problems.length;
253
280
  this.pendingObservation = statePatchObservation(problems);
254
281
  // Accepted deltas in a partially-failing batch still land — engine parity: real work is
255
282
  // never rolled back just because a sibling fence was malformed.
@@ -109,9 +109,32 @@ const RUN_STATE_FIELDS: ReadonlySet<string> = new Set([
109
109
  const ARRAY_FIELDS: ReadonlySet<string> = new Set(["findings", "verifiedFacts", "openQuestions"]);
110
110
  const TASK_MAX_CHARS = 200;
111
111
  const NEXT_STEP_MAX_CHARS = 300;
112
+ // Recall W2 contract alignment: STATE_FENCE_INSTRUCTION promises "≤ 5 keys per patch, every
113
+ // string value ≤ 120 chars" — the validator now MEANS it. Oversized values are CLAMPED
114
+ // fail-soft (the prose carries the story; Σ carries pointers), while the key count is
115
+ // rejected with an explicit error (a restatement-shaped patch must come back as feedback —
116
+ // §5.7: consistent validator feedback is the small-model bottleneck).
117
+ const STATE_PATCH_MAX_KEYS = 5;
118
+ const STATE_VALUE_MAX_CHARS = 120;
112
119
  // `findings` | `findings[+]` | `findings[2]` | `testedApproaches.h1`
113
120
  const PATCH_KEY = /^([a-zA-Z_][a-zA-Z0-9_]*)((?:\.[a-zA-Z_][a-zA-Z0-9_]*)*)(\[\+\]|\[\d+\])?$/;
114
121
 
122
+ /** Fail-soft contract clamp — every string Σ value honors the promised ≤120 chars. */
123
+ function clampStateValue(value: string): string {
124
+ return value.length > STATE_VALUE_MAX_CHARS ? value.slice(0, STATE_VALUE_MAX_CHARS) : value;
125
+ }
126
+
127
+ /** Clamp the string fields of a validated outcome record IN PLACE (string→clamped, everything
128
+ * else — explicit `null` deletes included — passes through untouched; the strict-merge null
129
+ * semantics live in strictMergeRecord and must never be eaten by the clamp). */
130
+ function clampOutcomeRecord(value: Readonly<Record<string, unknown>>): Record<string, unknown> {
131
+ const out: Record<string, unknown> = {};
132
+ for (const [key, field] of Object.entries(value)) {
133
+ out[key] = typeof field === "string" ? clampStateValue(field) : field;
134
+ }
135
+ return out;
136
+ }
137
+
115
138
  export function freshRunState(task: string): RunState {
116
139
  return {
117
140
  task,
@@ -256,20 +279,23 @@ export function enforceCaps(draft: MutableState): Result<RunState, PatchError> {
256
279
  artifacts: { ...draft.artifacts },
257
280
  };
258
281
  // bytesTotal: |Σ_t| must never exceed κ_Σ — deterministic eviction order keeps runs flat.
282
+ // Order (recall W2): diary entries (findings) go first, SPECULATIVE entries (openQuestions)
283
+ // next, FOUNDATION (verifiedFacts: paths/symbols/configs) survives longest — losing a
284
+ // foundational fact is the irreversible-recall failure the paper warns about (§7 case 2).
259
285
  let guard = 0;
260
286
  while (JSON.stringify(capped).length > RUN_STATE_LIMITS.bytesTotal && guard++ < 10_000) {
261
287
  if (capped.findings.length > 0) {
262
288
  capped.findings = capped.findings.slice(1);
263
289
  continue;
264
290
  }
265
- if (capped.verifiedFacts.length > 0) {
266
- capped.verifiedFacts = capped.verifiedFacts.slice(1);
267
- continue;
268
- }
269
291
  if (capped.openQuestions.length > 0) {
270
292
  capped.openQuestions = capped.openQuestions.slice(1);
271
293
  continue;
272
294
  }
295
+ if (capped.verifiedFacts.length > 0) {
296
+ capped.verifiedFacts = capped.verifiedFacts.slice(1);
297
+ continue;
298
+ }
273
299
  const approach = firstKey(capped.testedApproaches);
274
300
  if (approach !== undefined) {
275
301
  const rest = { ...capped.testedApproaches };
@@ -331,7 +357,7 @@ function applyKey(draft: MutableState, rawKey: string, value: unknown): Result<n
331
357
  }
332
358
  if (op === "[+]") {
333
359
  if (typeof value !== "string") return err({ kind: "type", path: rawKey, expected: "string" });
334
- list.push(value);
360
+ list.push(clampStateValue(value));
335
361
  } else if (op !== undefined) {
336
362
  const index = Number.parseInt(op.slice(1, -1), 10);
337
363
  if (value === null) {
@@ -346,11 +372,11 @@ function applyKey(draft: MutableState, rawKey: string, value: unknown): Result<n
346
372
  if (index > list.length) {
347
373
  return err({ kind: "schema", detail: `${rawKey} would leave a hole (len=${list.length})` });
348
374
  }
349
- list[index] = value;
375
+ list[index] = clampStateValue(value);
350
376
  } else {
351
377
  if (!isStringArray(value)) return err({ kind: "type", path: rawKey, expected: "string[]" });
352
378
  list.length = 0;
353
- list.push(...dedupStrings(value));
379
+ list.push(...dedupStrings(value.map(clampStateValue)));
354
380
  }
355
381
  return ok(null);
356
382
  }
@@ -409,15 +435,17 @@ function applyKey(draft: MutableState, rawKey: string, value: unknown): Result<n
409
435
  if (!isPlainObject(value) || !isApproachOutcome(value)) {
410
436
  return err({ kind: "type", path: rawKey, expected: "ApproachOutcome" });
411
437
  }
438
+ // Contract clamp: outcome strings honor the promised ≤120 chars (nulls stay nulls).
439
+ const outcome = clampOutcomeRecord(value);
412
440
  const prev: unknown = cursor[leaf];
413
441
  const merged = isPlainObject(prev)
414
- ? strictMergeRecord(prev, value, rawKey)
415
- : ok<Record<string, unknown>, PatchError>({ ...value });
442
+ ? strictMergeRecord(prev, outcome, rawKey)
443
+ : ok<Record<string, unknown>, PatchError>(outcome);
416
444
  if (!merged.ok) return merged;
417
445
  refreshOrder(cursor, leaf, merged.value); // last-mention ordering for eviction
418
446
  } else {
419
447
  if (typeof value !== "string") return err({ kind: "type", path: rawKey, expected: "string" });
420
- refreshOrder(cursor, leaf, value);
448
+ refreshOrder(cursor, leaf, clampStateValue(value));
421
449
  }
422
450
  commitRecord(draft, root, record);
423
451
  return ok(null);
@@ -454,6 +482,14 @@ export function applyPatch(prev: RunState, patch: unknown, t: number): Result<Ru
454
482
  }
455
483
  const ops = Object.entries(patch.state_patch);
456
484
  if (ops.length === 0) return err({ kind: "schema", detail: "empty state_patch" });
485
+ // Contract alignment (recall W2): the instruction promises ≤ 5 keys — a bigger patch is a
486
+ // restatement, and restatements must come back as feedback, not silently inflate Σ.
487
+ if (ops.length > STATE_PATCH_MAX_KEYS) {
488
+ return err({
489
+ kind: "schema",
490
+ detail: `${ops.length} keys — ≤ ${STATE_PATCH_MAX_KEYS} per patch; commit deltas only`,
491
+ });
492
+ }
457
493
  // Per-turn byte cap (bench rec #1): reject BEFORE the draft clone — a verbose restatement
458
494
  // must come back as an error observation, never silently eat output tokens.
459
495
  if (JSON.stringify(patch).length > RUN_STATE_LIMITS.patchBytes) {
@@ -469,6 +505,21 @@ export function applyPatch(prev: RunState, patch: unknown, t: number): Result<Ru
469
505
  return enforceCaps(draft);
470
506
  }
471
507
 
508
+ /** The actual limits, spelled out at rejection time — a small model told only "cap exceeded"
509
+ * thrashes blind retries into the degrade threshold. Field name → human limit (one source:
510
+ * RUN_STATE_LIMITS + the scalar caps). */
511
+ const CAP_LIMITS: Readonly<Record<string, string>> = Object.freeze({
512
+ task: `≤ ${TASK_MAX_CHARS} chars`,
513
+ nextStep: `≤ ${NEXT_STEP_MAX_CHARS} chars`,
514
+ findings: `≤ ${RUN_STATE_LIMITS.findings} entries`,
515
+ verifiedFacts: `≤ ${RUN_STATE_LIMITS.verifiedFacts} entries`,
516
+ testedApproaches: `≤ ${RUN_STATE_LIMITS.testedApproaches} entries`,
517
+ openQuestions: `≤ ${RUN_STATE_LIMITS.openQuestions} entries`,
518
+ artifacts: `≤ ${RUN_STATE_LIMITS.artifacts} entries`,
519
+ patchBytes: `≤ ${RUN_STATE_LIMITS.patchBytes} bytes per patch`,
520
+ bytesTotal: `≤ ${RUN_STATE_LIMITS.bytesTotal} bytes total`,
521
+ });
522
+
472
523
  /** Human-facing patch rejection — becomes the next O_t prefix (error-as-observation). */
473
524
  export function patchErrorText(error: PatchError): string {
474
525
  switch (error.kind) {
@@ -478,8 +529,10 @@ export function patchErrorText(error: PatchError): string {
478
529
  return `type mismatch at "${error.path}" (expected ${error.expected})`;
479
530
  case "implicit-drop":
480
531
  return `implicit key drop at "${error.path}" — restate the key or delete it with null`;
481
- case "cap":
482
- return `cap exceeded on "${error.field}"`;
532
+ case "cap": {
533
+ const limit = CAP_LIMITS[error.field];
534
+ return `cap exceeded on "${error.field}"${limit === undefined ? "" : ` — the limit is ${limit}`}`;
535
+ }
483
536
  }
484
537
  }
485
538
 
@@ -491,6 +544,9 @@ export function patchErrorText(error: PatchError): string {
491
544
  * Idle degrade (bench rec #2): when `fenceRequested` is true (the turn conditioned on Σ)
492
545
  * and zero patches were accepted, the turn is IDLE — Σ inflated the prompt for nothing.
493
546
  * `RUN_STATE_IDLE_DEGRADE_TURNS` consecutive idle turns degrade, same as-built outcome.
547
+ * `productive` (root parity, recall W2): a turn that executed real repl work without
548
+ * raising RESETS the idle streak — heavy execution is progress even without a delta, and
549
+ * a mid-run amputation of Σ costs more than the prompt it saves.
494
550
  */
495
551
  export function applyStatePatches(
496
552
  mode: Extract<RunStateMode, { kind: "active" }>,
@@ -498,10 +554,11 @@ export function applyStatePatches(
498
554
  iteration: number,
499
555
  config: Pick<RlmConfig, "runStateRetryMax">,
500
556
  fenceRequested = false,
557
+ productive = false,
501
558
  ): { readonly mode: RunStateMode; readonly observation: string | undefined } {
502
559
  // Identity fast-path: a fence-free turn on a run that never asked for fences changes
503
560
  // nothing — keep the same mode object (callers may compare identity).
504
- if (parsed.length === 0 && !fenceRequested) return { mode, observation: undefined };
561
+ if (parsed.length === 0 && !fenceRequested && !productive) return { mode, observation: undefined };
505
562
  let state = mode.state;
506
563
  let retries = mode.retries;
507
564
  let accepted = 0;
@@ -522,7 +579,8 @@ export function applyStatePatches(
522
579
  }
523
580
  }
524
581
  // Bench rec #2: accepted deltas reset the idle streak; a requested-but-empty turn grows it.
525
- const idle = accepted > 0 || !fenceRequested ? 0 : mode.idle + 1;
582
+ // Productive-turn parity (root tracker): executed work resets the streak too.
583
+ const idle = accepted > 0 || !fenceRequested || productive ? 0 : mode.idle + 1;
526
584
  let nextMode: RunStateMode = { kind: "active", state, retries, idle };
527
585
  if (retries > config.runStateRetryMax) {
528
586
  nextMode = {
@@ -557,6 +615,9 @@ export const STATE_FENCE_INSTRUCTION: string =
557
615
  "Σ is an index of pointers, not a report: ≤ 5 keys per patch, every string value ≤ 120 chars, " +
558
616
  "telegraphic style (`path — fact`, `verdict — numbers`). NEVER paste findings, tables, JSON " +
559
617
  "blobs, or long excerpts into Σ — the prose carries the story, Σ carries only the pointers.\n" +
618
+ `Caps: findings ≤ ${RUN_STATE_LIMITS.findings}, verifiedFacts ≤ ${RUN_STATE_LIMITS.verifiedFacts}, ` +
619
+ `testedApproaches ≤ ${RUN_STATE_LIMITS.testedApproaches}, openQuestions/artifacts ≤ ${RUN_STATE_LIMITS.openQuestions}/${RUN_STATE_LIMITS.artifacts} — ` +
620
+ "oldest entries are evicted automatically, so push new facts and let Σ prune itself.\n" +
560
621
  "Keys: dotted paths write record leaves; [+] appends; [N] sets an array slot; null deletes.\n" +
561
622
  "Commit DELTAS only — never restate unchanged records or arrays; touch single dotted keys " +
562
623
  `or append with [+]. Whole-record restatements must keep EVERY key (implicit drops are ` +
@@ -578,6 +639,17 @@ export function runStateTurnBlock(state: RunState): string {
578
639
  return sigmaBlock(state, true);
579
640
  }
580
641
 
642
+ /**
643
+ * Σ economics: re-sending a byte-identical Σ every turn is up to ~3K tokens of attention tax
644
+ * that carries zero new information — the model saw it one turn ago. The marker keeps the
645
+ * patch grammar within reach (a small model without it stops committing at all) while cutting
646
+ * the block ~10x. Callers must re-send the FULL block after compaction/rebase or degrade.
647
+ */
648
+ export const SIGMA_UNCHANGED_LINE =
649
+ "[Σ] unchanged since last turn — deltas apply against it. Grammar: dotted keys, [+] append, " +
650
+ "[N] set slot, null delete; ≤ 5 keys per patch, every value ≤ 120 chars. Fence only NEW " +
651
+ "durable facts; an absent fence is free.";
652
+
581
653
  /**
582
654
  * Root Σ (WS-3b): the ROOT's A_t block. R2 (G2): with `withContract` the splice carries the
583
655
  * SAME fence contract (delegating to the one composer above — no re-wording), making the