@hicaru/pi-rlm 0.3.16 → 0.3.17

package/src/core/root-digest.ts

@@ -0,0 +1,213 @@
+/**
+ * Root Σ (WS-2) — deterministic root compaction for the Pi harness orchestrator.
+ *
+ * When the native session compacts, this replaces Pi's LLM prose summarizer with a
+ * structural digest: the same no-LLM distillation shape the engine proven-uses at its
+ * budget hard-state (core/budget.ts:distillTrajectory), adapted to host AgentMessages.
+ * Zero summary tokens; compaction becomes reproducible. Paper framing: the digest is a
+ * Σ-style sufficient statistic (§3.1) — sections are ordered by continuity value, and the
+ * verbatim tail after `firstKeptEntryId` stays byte-identical (§5.3 observation override:
+ * fresh tool results outrank the digest).
+ *
+ * Cut-point semantics: Pi's preparation already picked `firstKeptEntryId` via its
+ * keepRecentTokens walk. We only ever TIGHTEN (move the cut later, shrinking the verbatim
+ * tail toward `rootDigestKeepRecentChars`) — never extend — and fold every message the
+ * tighter cut displaces into the digest inputs, so nothing is dropped unaccounted. Cuts
+ * land on entry boundaries Pi itself treats as safe (the walk is char-budgeted over whole
+ * entries, mirroring pi-docs/compaction.md's accumulate-and-cut).
+ *
+ * Failure semantics: pure function, no model call, no throw paths — the index.ts handler
+ * still wraps it fail-soft (host contract: handlers must not throw) and returns undefined
+ * so Pi falls back to its own summarizer.
+ */
+
+import type { CompactionResult, SessionEntry } from "@earendil-works/pi-coding-agent";
+import type { RlmConfig } from "./types.ts";
+import { DEFAULT_NEXT_STEP, FINDINGS_MAX, FINDINGS_MIN_CHARS, NEXT_STEP_RE, STATE_MAX, truncateMid } from "./budget.ts";
+import { estimateMessageTokens } from "../text/tokens.ts";
+import { agentMessageText } from "../text/agent-text.ts";
+import { isRecord } from "../util/type-guards.ts";
+import { ROOT_DIGEST_HEADER, ROOT_DIGEST_SECTIONS } from "../prompts/glossary.ts";
+
+/** Marker persisted on the session's compaction entry — tests and soaks assert on it. */
+export interface RootDigestDetails {
+  readonly kind: "root-digest";
+  readonly version: 1;
+}
+
+/** Structural slice of SkillStore the digest needs — tests inject stand-ins without a store. */
+export interface DigestFactSource {
+  sliceForPrompt(query: string, budgetTokens: number, minScore: number): string;
+}
+
+/** `SessionBeforeCompactEvent["preparation"]` slice — indexed on the event shape upstream. */
+export interface RootDigestPreparation {
+  readonly firstKeptEntryId: string;
+  readonly messagesToSummarize: readonly unknown[];
+  readonly isSplitTurn: boolean;
+  readonly tokensBefore: number;
+}
+
+export interface RootDigestArgs {
+  readonly preparation: RootDigestPreparation;
+  /** Chronological session entries (event.branchEntries) — the tighten walk reads these. */
+  readonly branchEntries: readonly SessionEntry[];
+  readonly config: Pick<RlmConfig, "rootDigestKeepRecentChars" | "rootDigestMaxChars" | "skillStateMinScore">;
+  readonly store: DigestFactSource | undefined;
+}
+
+/** Section caps come from budget.ts (ONE source — N2); only digest-local shapes live here. */
+const BULLET_CHARS = 400;
+const TASK_FRACTION = 0.3;
+const FACTS_FRACTION = 4; // facts budget = maxChars / FACTS_FRACTION, tokens at 4 chars/token
+
+/** AgentMessage-shaped projection of one session entry (host sessionEntryToContextMessages shape). */
+function entryMessages(entry: SessionEntry): readonly unknown[] {
+  if (!isRecord(entry)) return [];
+  if (entry.type === "message" && isRecord(entry.message)) return [entry.message];
+  if (entry.type === "custom_message") return [{ role: "custom", content: entry.content }];
+  if (entry.type === "compaction" && typeof entry.summary === "string") {
+    return [{ role: "compactionSummary", summary: entry.summary }];
+  }
+  if (entry.type === "branch_summary" && typeof entry.summary === "string") {
+    return [{ role: "branchSummary", summary: entry.summary }];
+  }
+  return [];
+}
+
+function entryChars(entry: SessionEntry): number {
+  let total = 0;
+  for (const message of entryMessages(entry)) total += agentMessageText(message).length + 8;
+  return total;
+}
+/**
+ * The ONE tighten walk: index of the first entry kept verbatim (≥ piCut). Walks backward
+ * accumulating entry chars until the budget is met; when the whole tail fits under the
+ * budget, Pi's boundary stands — we never EXTEND the digest span past Pi's cut.
+ */
+function tightenedCut(entries: readonly SessionEntry[], piCut: number, keepChars: number): number {
+  let acc = 0;
+  for (let i = entries.length - 1; i >= piCut; i--) {
+    acc += entryChars(entries[i]);
+    if (acc >= keepChars) return i;
+  }
+  return piCut;
+}
+
+/** [Task] — the LAST user prompt in the span (the ask still in force when compaction hits). */
+function taskSection(messages: readonly unknown[]): string {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const m = messages[i];
+    if (isRecord(m) && m.role === "user") {
+      const text = agentMessageText(m).trim();
+      if (text !== "") return text;
+    }
+  }
+  return "";
+}
+
+/** [Findings] — newest-first substantive assistant blobs, capped, then chronological. */
+function findingsSection(messages: readonly unknown[]): readonly string[] {
+  const findings: string[] = [];
+  for (let i = messages.length - 1; i >= 0 && findings.length < FINDINGS_MAX; i--) {
+    const m = messages[i];
+    if (!isRecord(m) || m.role !== "assistant") continue;
+    const text = agentMessageText(m).trim();
+    if (text.length > FINDINGS_MIN_CHARS) findings.push(text);
+  }
+  findings.reverse();
+  return findings;
+}
+
+/** [State] — newest-first toolResult first-lines (the last observed machinery state). */
+function stateSection(messages: readonly unknown[]): readonly string[] {
+  const states: string[] = [];
+  for (let i = messages.length - 1; i >= 0 && states.length < STATE_MAX; i--) {
+    const m = messages[i];
+    if (!isRecord(m) || m.role !== "toolResult") continue;
+    const text = agentMessageText(m).trim();
+    if (text === "") continue;
+    const tool = typeof m.toolName === "string" ? m.toolName : "tool";
+    states.push(`${tool}: ${text.split("\n", 1)[0] ?? text}`);
+  }
+  states.reverse();
+  return states;
+}
+
+/**
+ * Build the digest compaction. Returns undefined when Pi should keep its own path:
+ * split turns (our flat summary would double-count the turn prefix) or an empty span.
+ *
+ * `tokensBeforeRecomputed` is our own estimateMessageTokens pass over the same span —
+ * V1 soak probe: the host consumes `compaction.tokensBefore` (Pi's own preparation number)
+ * as-is; one trace line carries both so the first real /compact confirms whether the host
+ * ever diverges (status-line truth check) without touching the persisted CompactionResult.
+ */
+export function buildRootDigestCompaction(
+  args: RootDigestArgs,
+): {
+  readonly compaction: CompactionResult<RootDigestDetails>;
+  readonly tokensBeforeRecomputed: number;
+} | undefined {
+  const prep = args.preparation;
+  if (prep.isSplitTurn) return undefined;
+
+  const piCut = args.branchEntries.findIndex((entry) => entry.id === prep.firstKeptEntryId);
+  const cut = piCut < 0 ? -1 : tightenedCut(args.branchEntries, piCut, args.config.rootDigestKeepRecentChars);
+
+  const messages: unknown[] = [...prep.messagesToSummarize];
+  if (cut > piCut && piCut >= 0) {
+    for (let i = piCut; i < cut; i++) messages.push(...entryMessages(args.branchEntries[i]));
+  }
+  if (messages.length === 0) return undefined;
+
+  const max = Math.max(200, args.config.rootDigestMaxChars);
+  const task = truncateMid(taskSection(messages), Math.floor(max * TASK_FRACTION));
+  const findings = findingsSection(messages).map((f) => truncateMid(f, BULLET_CHARS));
+  const states = stateSection(messages);
+  // Next-step probe mirrors distillTrajectory: newest-first scan, chronological render.
+  const next = findings.find((f) => NEXT_STEP_RE.test(f)) ?? DEFAULT_NEXT_STEP;
+  const facts = args.store === undefined
+    ? ""
+    : args.store.sliceForPrompt(
+        task,
+        Math.max(1, Math.floor(max / FACTS_FRACTION / 4)),
+        args.config.skillStateMinScore,
+      );
+
+  const render = (factsText: string, stateLines: readonly string[], findingLines: readonly string[]): string => {
+    const blocks: string[] = [ROOT_DIGEST_HEADER];
+    const section = (label: string, body: string): void => {
+      if (body.trim() !== "") blocks.push(`[${label}] ${body}`);
+    };
+    section(ROOT_DIGEST_SECTIONS.task, task);
+    section(ROOT_DIGEST_SECTIONS.findings, findingLines.map((f) => `- ${f}`).join("\n"));
+    section(ROOT_DIGEST_SECTIONS.state, stateLines.map((s) => `- ${s}`).join("\n"));
+    section(ROOT_DIGEST_SECTIONS.next, next);
+    section(ROOT_DIGEST_SECTIONS.facts, factsText);
+    return blocks.join("\n\n");
+  };
+
+  // Over-cap drop order: [Project facts] → [State] → [Findings]; [Task]/[Next] survive to a
+  // final truncate — task continuity beats trivia (paper §3.1 sufficient statistic).
+  let summary = render(facts, states, findings);
+  if (summary.length > max) summary = render("", states, findings);
+  if (summary.length > max) summary = render("", [], findings);
+  if (summary.length > max) {
+    summary = render("", [], findings.slice(0, Math.max(1, Math.floor(findings.length / 2))));
+  }
+  summary = truncateMid(summary, max);
+
+  const boundary = cut >= 0 ? args.branchEntries[cut] : undefined;
+  return {
+    compaction: {
+      summary,
+      firstKeptEntryId: (isRecord(boundary) && typeof boundary.id === "string" ? boundary.id : undefined)
+        ?? prep.firstKeptEntryId,
+      tokensBefore: prep.tokensBefore,
+      details: { kind: "root-digest", version: 1 },
+    },
+    // Same CHAR_PER_TOKEN=4 (+8/message) math as the engine budgets — comparable numbers.
+    tokensBeforeRecomputed: estimateMessageTokens(messages.map((m) => ({ content: agentMessageText(m) }))),
+  };
+}

package/src/core/root-state.ts

@@ -0,0 +1,240 @@
+/**
+ * Root Σ (WS-3b/WS-4) — RootStateTracker: a digest-level Σ_t for the NATIVE Pi session.
+ *
+ * This is NOT the engine's RunStateTracker (core/engine turn loop); it is the root
+ * orchestrator's sufficient statistic, living in the extension closure next to
+ * controller/sandboxManager. It is runtime-derived — tool outcomes, engine-run mirrors,
+ * the user's latest prompt — zero model cooperation required (paper §5.3: observation
+ * override; §5.7: small models must not be the state's author by default). The optional
+ * fence protocol (enableRootStateFences, default OFF) is the only model-proposed input and
+ * rides the SAME V(ΔΣ_t,Σ_t) validator + retry/degrade ladder as engine runs.
+ *
+ * Caps/dedup/serialization are the engine's own machinery: RUN_STATE_LIMITS, dedupStrings,
+ * enforceCaps, applyPatch — reused, never re-implemented here.
+ */
+
+import {
+  applyPatch,
+  dedupStrings,
+  enforceCaps,
+  freshRunState,
+  malformedFenceProblem,
+  patchErrorText,
+  RUN_STATE_LIMITS,
+  statePatchObservation,
+  type ApproachOutcome,
+  type MutableState,
+  type PatchError,
+  type RunState,
+} from "./run-state.ts";
+import type { Result } from "../util/errors.ts";
+import type { StateFenceResult } from "../text/parsing.ts";
+
+/** Consecutive failed outcomes on one key before the rectify hint fires (MAS2 Eq. 5 parity). */
+const RECTIFY_FAILURE_THRESHOLD = 2;
+/** Task restatement cap — mirrors run-state.ts TASK_MAX_CHARS (kept in sync by comment). */
+const ROOT_TASK_MAX_CHARS = 200;
+
+/** Root Σ mode — same discriminated union shape as the engine's (active | degraded). */
+type RootStateMode =
+  | { readonly kind: "active"; readonly retries: number }
+  | { readonly kind: "degraded"; readonly reason: string };
+
+export class RootStateTracker {
+  private draft: MutableState;
+  private mode: RootStateMode = { kind: "active", retries: 0 };
+  private opCounter = 0;
+  private dirtyFlag = false;
+  private readonly failures = new Map<string, number>();
+  private pendingObservation: string | undefined;
+  private readonly retryMax: number;
+
+  private constructor(task: string, retryMax: number) {
+    const fresh = freshRunState(task);
+    this.draft = {
+      task: fresh.task,
+      nextStep: fresh.nextStep,
+      updatedAt: fresh.updatedAt,
+      findings: [...fresh.findings],
+      verifiedFacts: [...fresh.verifiedFacts],
+      openQuestions: [...fresh.openQuestions],
+      testedApproaches: { ...fresh.testedApproaches },
+      artifacts: { ...fresh.artifacts },
+    };
+    this.retryMax = Math.max(0, Math.floor(retryMax));
+  }
+
+  static fresh(task: string, retryMax = 2): RootStateTracker {
+    return new RootStateTracker(task.trim().slice(0, ROOT_TASK_MAX_CHARS), retryMax);
+  }
+
+  /** True when nothing worth splicing has accumulated — the context transform no-ops. */
+  get isEmpty(): boolean {
+    return (
+      this.draft.findings.length === 0 &&
+      this.draft.verifiedFacts.length === 0 &&
+      Object.keys(this.draft.testedApproaches).length === 0 &&
+      this.draft.nextStep === ""
+    );
+  }
+
+  get dirty(): boolean {
+    return this.dirtyFlag;
+  }
+
+  snapshot(): RunState {
+    const capped = enforceCaps(this.draft);
+    // enforceCaps never fails in practice; the fallback keeps the tracker fail-soft anyway.
+    return capped.ok ? capped.value : freshRunState(this.draft.task);
+  }
+
+  noteFinding(text: string): void {
+    const trimmed = text.trim();
+    if (trimmed === "") return;
+    this.draft.findings = dedupStrings([...this.draft.findings, trimmed]).slice(-RUN_STATE_LIMITS.findings);
+    this.touch();
+  }
+
+  noteFact(text: string): void {
+    const trimmed = text.trim();
+    if (trimmed === "") return;
+    this.draft.verifiedFacts = dedupStrings([...this.draft.verifiedFacts, trimmed])
+      .slice(-RUN_STATE_LIMITS.verifiedFacts);
+    this.touch();
+  }
+
+  noteOutcome(key: string, outcome: ApproachOutcome): void {
+    const k = key.trim();
+    if (k === "") return;
+    this.draft.testedApproaches[k] = outcome;
+    if (outcome.status === "failed") {
+      this.failures.set(k, (this.failures.get(k) ?? 0) + 1);
+    } else {
+      this.failures.delete(k);
+    }
+    this.touch();
+  }
+
+  /** Tool-outcome feed (WS-4 v1 source): failures become approach outcomes, verbatim. */
+  observeToolResult(toolName: string, isError: boolean, reasonFirstLine: string): void {
+    if (isError) {
+      this.noteOutcome(`tool:${toolName}`, { status: "failed", reason: reasonFirstLine });
+    } else {
+      // A success clears the tool's failure streak — the state reflects the newest truth.
+      if (this.draft.testedApproaches[`tool:${toolName}`] !== undefined) {
+        this.noteOutcome(`tool:${toolName}`, { status: "succeeded", evidence: reasonFirstLine });
+      } else {
+        this.failures.delete(`tool:${toolName}`);
+      }
+    }
+  }
+
+  /** The user's latest ask IS the next step by definition (WS-4 v1 source). Empty is a no-op. */
+  setNextStep(text: string): void {
+    const trimmed = text.trim();
+    if (trimmed === "") return;
+    this.draft.nextStep = trimmed.slice(0, 300);
+    this.touch();
+  }
+
+  /**
+   * WS-4 mirror: absorb a finished engine run's Σ (same object the SkillState harvest
+   * consumes — one source, two sinks). Findings/facts dedup through the shared key; the
+   * engine's approach outcomes ride along under their own keys.
+   */
+  absorbEngineState(engine: RunState): void {
+    this.draft.findings = dedupStrings([...this.draft.findings, ...engine.findings])
+      .slice(-RUN_STATE_LIMITS.findings);
+    this.draft.verifiedFacts = dedupStrings([...this.draft.verifiedFacts, ...engine.verifiedFacts])
+      .slice(-RUN_STATE_LIMITS.verifiedFacts);
+    for (const [key, outcome] of Object.entries(engine.testedApproaches)) {
+      this.draft.testedApproaches[key] = outcome;
+    }
+    if (this.draft.task === "") this.draft.task = engine.task;
+    this.touch();
+  }
+
+  /**
+   * WS-4.2 (default OFF): model-proposed ΔΣ_t fences through the ONE validator. EXACT ladder
+   * parity with the engine's `applyStatePatches` (N3): every fence is processed — accepted
+   * deltas land sequentially, ALL problems accumulate into ONE observation (error-as-
+   * observation), and only past `runStateRetryMax` total rejections the tracker degrades and
+   * fences stop being applied. Wording delegates to run-state.ts (N1) — one source.
+   */
+  applyFences(fences: readonly StateFenceResult[]): void {
+    if (this.mode.kind !== "active" || fences.length === 0) return;
+    let state = this.snapshot();
+    const problems: string[] = [];
+    for (const fence of fences) {
+      if (!fence.ok) {
+        problems.push(malformedFenceProblem(fence.error));
+        continue;
+      }
+      const next: Result<RunState, PatchError> = applyPatch(state, fence.value, ++this.opCounter);
+      if (next.ok) {
+        state = next.value;
+      } else {
+        problems.push(patchErrorText(next.error));
+      }
+    }
+    if (problems.length === 0) {
+      this.mode = { kind: "active", retries: 0 };
+      this.pendingObservation = undefined;
+      this.draft = this.toMutable(state);
+      this.touch();
+      return;
+    }
+    const retries = this.mode.retries + problems.length;
+    this.pendingObservation = statePatchObservation(problems);
+    // Accepted deltas in a partially-failing batch still land — engine parity: real work is
+    // never rolled back just because a sibling fence was malformed.
+    this.draft = this.toMutable(state);
+    this.touch();
+    if (retries > this.retryMax) {
+      this.mode = { kind: "degraded", reason: `state-patch retry cap exceeded (${retries} rejected)` };
+    } else {
+      this.mode = { kind: "active", retries };
+    }
+  }
+
+  /** Consume (and clear) the pending fence-rejection observation, if any. */
+  takePendingObservation(): string | undefined {
+    const observation = this.pendingObservation;
+    this.pendingObservation = undefined;
+    return observation;
+  }
+
+  /**
+   * WS-4 rectify parity (budget.ts:rectify ideas, deterministic): after N consecutive
+   * failed outcomes on one key, suggest the local fix — never a model switch.
+   */
+  rectifyHint(): string | undefined {
+    if (this.mode.kind !== "active") return undefined;
+    for (const [key, count] of this.failures) {
+      if (count >= RECTIFY_FAILURE_THRESHOLD) {
+        return `[rectify] '${key}' failed ${count}× consecutively — narrow the approach ` +
+          "(different path/tool/slice) instead of retrying blind (MAS2 Eq. 5).";
+      }
+    }
+    return undefined;
+  }
+
+  private toMutable(state: RunState): MutableState {
+    return {
+      task: state.task,
+      nextStep: state.nextStep,
+      updatedAt: state.updatedAt,
+      findings: [...state.findings],
+      verifiedFacts: [...state.verifiedFacts],
+      openQuestions: [...state.openQuestions],
+      testedApproaches: { ...state.testedApproaches },
+      artifacts: { ...state.artifacts },
+    };
+  }
+
+  private touch(): void {
+    this.opCounter += 1;
+    this.draft.updatedAt = this.opCounter;
+    this.dirtyFlag = true;
+  }
+}