@hicaru/pi-rlm 0.1.7 → 0.1.8

package/README.md

@@ -117,11 +117,41 @@ These functions are injected into the model's Python namespace inside the REPL:
 | `rlm_query_batched` | `(prompts, model=None) -> list[str]` | Concurrent recursive child RLMs |
 | `todo` | `(action, **kwargs) -> str` | Task list: `create`/`update`/`list`/`get`/`delete`/`clear` |
 | `ask_user_question` | `(questions) -> list[dict]` | Ask the user structured questions (depth 0 only) |
+| `load_library` | `(source) -> dict \| str` | Load an external dir, file, or git URL into a new `context_N` slot |
 | `stage_edit` | `(path, old_text, new_text) -> str` | Stage a file edit; relayed to the host's native edit flow |
-| `advance_phase` | `(phase, summary=None) -> str` | Move the root pipeline to a new phase |
+| `save_artifact` | `(kind, content) -> str` | Persist a stage artifact (`clarification` / `research` / `plan` / `validation`) under `.rlm/artifacts/` (root depth only) |
+| `advance_phase` | `(phase, summary=None) -> str` | Advance one step in order `clarify → research → blueprint → implement → validate` (clarify skipped when `askUserQuestion` is off). **Engine-gated** on the latest artifact + interview rounds. Rejected transitions return the gate error. |
 | `SHOW_VARS` | `() -> str` | List currently defined variables & their types |
 | `answer` | `dict` | Set `answer["content"]=...; answer["ready"]=True` to finalize |
 
+### Loading external libraries
+
+When the task needs an **external library, another source tree, or standalone docs** that are
+not in the packed repo `context`, the model calls `load_library(source)` mid-run:
+
+```python
+info = load_library("../some-lib")           # local directory → repomix-packed list[dict]
+info = load_library("docs/api.md")           # single file → plain str
+info = load_library("https://github.com/x/y.git")  # shallow clone, then pack
+# info == {"index": 1, "var": "context_1", "files": …, "chars": …}
+# then chunk context_1 exactly like context
+```
+
+Slots start at `context_1` (`context` / `context_0` remains the repo). Toggle via
+`/rlm-config` → **Library loader** (`libraryLoader`, default on). On headless runs with
+persistence, each loaded slot is written as a resume sidecar (`context.<N>.json`).
+
+### Artifact-gated pipeline (opt-in via `pipeline: true`)
+
+When enabled at root depth:
+
+1. **Goal capture** — the brief is written verbatim to `.rlm/artifacts/goal/goal-<ts>.md` with a pre-run dirty-tree baseline.
+2. **Stages** — `clarify → research → blueprint → implement → validate`. Each produces a durable markdown artifact with frontmatter contracts; chat history is **reset** at every phase boundary (artifacts are the only channel; REPL vars persist).
+3. **Clarify (intake)** — interviews the user via `ask_user_question` (intent first, then evidence-confirmed decisions). Writes `.rlm/artifacts/clarifications/*` with `decisions_count` / `open_questions_count`. Engine gate: **≥1 serviced ask round** + artifact contract. When **`askUserQuestion` is off**, clarify is skipped and the run starts at research.
+4. **Gates (TypeScript, never LLM judgment)** — `status: ready`; clarify structure; plan `phases:` ≡ fence-aware `## Phase N:` headings; every `file:line` citation resolves; validate carries `blockers_count` + `verdict`.
+5. **Implement fanout** — on `advance_phase("implement")` the engine runs one **serial** child RLM per plan phase and applies that child's edits before the next phase starts.
+6. **Corrective loop** — `blockers_count > 0` re-enters blueprint, bounded by `maxBackwardJumps` (default 2).
+
 ## Settings (`/rlm-config`)
 
 | Setting | Default | Meaning |
@@ -137,11 +167,15 @@ These functions are injected into the model's Python namespace inside the REPL:
 | Token ceiling | none | total input+output token cap for the whole recursive tree |
 | Max consecutive errors | `5` | stop after N consecutive failing turns (none = off) |
 | Orchestrator addendum | on | divide-and-conquer guidance in the root system prompt |
+| Phase pipeline | off | artifact-gated clarify→research→blueprint→implement fanout→validate |
+| Max validate→blueprint loops | `2` | bounded corrective re-entries when validation reports blockers |
+| Ask user question | on | when pipeline is on, enables clarify intake; when off, pipeline starts at research |
 | Trajectory compaction | on (0.65) | summarize old turns when history nears the context window |
 | Root model output cap (tok) | `16384` | max output tokens per root-model turn |
 | Sandbox init timeout | `30000` ms | how long to wait for the Python worker to start |
 | `askUserQuestion` | on | expose `ask_user_question()` to the model |
 | `todo` | on | expose `todo()` to the model |
+| Library loader | on | expose `load_library()` for external dirs/files/git repos |
 
 > **Concurrency note:** each `rlm_query` child spawns its own `python3` worker (~50–150 ms
 > cold start). Worst-case concurrent interpreters ≈ `maxConcurrentSubcalls`^(depth−1); at

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hicaru/pi-rlm",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "type": "module",
   "description": "Save 99% tokens, Recursive Language Model (RLM) for the Pi",
   "license": "MIT",

package/src/bridge/library.ts

@@ -0,0 +1,77 @@
+/**
+ * Shared load_library handler for headless engine and native repl() mode.
+ *
+ * Host assigns context slot indices (slot 0 = repo); packs the source via
+ * resolveLibrarySource; optional onLoaded writes resume sidecars.
+ *
+ * Late-bound deps (getCwd / getEmitter) keep a single handler closure correct
+ * across native repl() calls — getOrCreate only installs handlers at spawn.
+ */
+
+import type { RlmEmitter } from "../tool/rlm-events.ts";
+import type { SubLlmHandlers } from "../sandbox/sandbox.ts";
+import { resolveLibrarySource } from "../context/library-context.ts";
+import { previewText } from "../text/preview.ts";
+
+export interface LibraryBridgeOpts {
+  /** Fixed cwd (headless). Prefer getCwd when the sandbox outlives a single invocation. */
+  readonly cwd?: string;
+  /** Late-bound cwd (native mode — sandbox handlers outlive a single repl()). */
+  readonly getCwd?: () => string;
+  readonly emitter?: RlmEmitter;
+  /** Native mode: read the live emitter each call. */
+  readonly getEmitter?: () => RlmEmitter | null | undefined;
+  readonly parentId?: string;
+  readonly signal?: AbortSignal;
+  /** First slot to assign (slot 0 is the repo context). Resume passes 1 + restored slots. */
+  readonly startIndex: number;
+  /** Post-load hook — the engine writes the resume sidecar here; native mode omits it. */
+  readonly onLoaded?: (index: number, payload: unknown) => void | Promise<void>;
+}
+
+export interface LibraryHandlerBundle {
+  readonly handlers: Pick<SubLlmHandlers, "loadLibrary">;
+  /** Reset the slot counter (call when the sandbox is discarded and will re-spawn). */
+  readonly reset: () => void;
+}
+
+export function buildLibraryHandler(opts: LibraryBridgeOpts): LibraryHandlerBundle {
+  let nextIndex = opts.startIndex;
+  return {
+    reset: () => { nextIndex = opts.startIndex; },
+    handlers: {
+      async loadLibrary(source, depth) {
+        const emitter = opts.getEmitter?.() ?? opts.emitter;
+        const cwd = opts.getCwd?.() ?? opts.cwd;
+        if (cwd === undefined || cwd === "") {
+          throw new Error("load_library: no cwd configured");
+        }
+        const id = emitter?.emitSubcallCreated({
+          kind: "tool", parentId: opts.parentId,
+          label: "load_library",
+          args: previewText(source, 80),
+          depth,
+        });
+        try {
+          const resolved = await resolveLibrarySource(source, cwd, opts.signal);
+          if (!resolved.ok) throw new Error(resolved.error);
+          const index = nextIndex++;
+          await opts.onLoaded?.(index, resolved.value.payload);
+          if (id) emitter?.emitSubcallUpdated({
+            id, status: "done",
+            resultPreview: `context_${index}: ${resolved.value.files ?? 1} file(s), ${resolved.value.chars.toLocaleString()} chars`,
+          });
+          return {
+            payload: resolved.value.payload,
+            index,
+            files: resolved.value.files,
+            chars: resolved.value.chars,
+          };
+        } catch (err) {
+          if (id) emitter?.emitSubcallUpdated({ id, status: "error", detail: String(err) });
+          throw err;   // serviceInterrupt catch → {error} reply → "Error: …" in the REPL
+        }
+      },
+    },
+  };
+}

package/src/bridge/rlm-query.ts

@@ -4,18 +4,31 @@
  * A child RLM gets its own sandbox and iterates over the prompt as its context. At/over the
  * depth cap it degrades to a plain `llm_query` (ported from rlm/core/rlm.py `_subcall`). The
  * concurrency pool bounds parallel children for `rlm_query_batched`.
+ *
+ * `childRun` is the single spawn path: rlmQuery returns `.answer`; implement fanout also
+ * needs `.edits` — both share this implementation (DRY).
  */
 
-import type { RunRlm } from "../core/types.ts";
+import type { RlmResult, RunRlm } from "../core/types.ts";
 import type { LlmBridge } from "./llm-query.ts";
 import type { RlmEmitter } from "../tool/rlm-events.ts";
 import { checkResourceLimits } from "../core/resource-limits.ts";
 import { formatError } from "../util/errors.ts";
 import { mapPool } from "../util/concurrency.ts";
 
+export interface ChildRunInput {
+  readonly rootPrompt: string;
+  readonly context: unknown;
+  readonly depth: number;
+  readonly label?: string;
+  readonly model?: string | null;
+}
+
 export interface RlmHandlers {
   rlmQuery(prompt: string, model: string | null, depth: number): Promise<string>;
   rlmQueryBatched(prompts: string[], model: string | null, depth: number): Promise<string[]>;
+  /** Full child-run result (answer + edits + usage) for engine-driven fanout. */
+  childRun(input: ChildRunInput): Promise<RlmResult>;
 }
 
 export interface RlmBridgeOptions {
@@ -33,29 +46,51 @@ export interface RlmBridgeOptions {
   readonly onChildUsage?: (costUsd: number, inputTokens: number, outputTokens: number) => void;
 }
 
+function emptyResult(answer: string): RlmResult {
+  return {
+    answer,
+    edits: [],
+    iterations: 0,
+    costUsd: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    durationMs: 0,
+  };
+}
+
 export function createRlmHandlers(opts: RlmBridgeOptions): RlmHandlers {
-  async function child(prompt: string, model: string | null, depth: number): Promise<string> {
-    const childDepth = depth + 1;
+  async function childRun(input: ChildRunInput): Promise<RlmResult> {
+    const childDepth = input.depth;
     // At the cap, a child RLM would just be an LM — short-circuit to a one-shot llm_query.
-    if (childDepth >= opts.maxDepth) return opts.llm.llmQuery(prompt, model, depth);
+    // (Callers pass the absolute child depth; rlmQuery wraps with depth+1.)
+    if (childDepth >= opts.maxDepth) {
+      const answer = await opts.llm.llmQuery(
+        input.rootPrompt || String(input.context),
+        input.model ?? null,
+        childDepth - 1,
+      );
+      return emptyResult(answer);
+    }
     let subId: string | undefined;
     try {
       const rem = opts.remainingBudget?.() ?? {};
       // Pre-spawn guard: refuse if the parent's budget or timeout is already exhausted
       // (reference: _subcall checks remaining_budget/timeout before spawning).
       const limitError = checkResourceLimits(rem);
-      if (limitError) return limitError;
+      if (limitError) return emptyResult(limitError);
+      const label = input.label ?? "rlm_query";
+      const detailSource = input.rootPrompt || String(input.context);
       subId = opts.emitter.emitSubcallCreated({
-        kind: "rlm", parentId: opts.parentNodeId, label: "rlm_query",
-        model: model ?? undefined, detail: prompt.slice(0, 60),
+        kind: "rlm", parentId: opts.parentNodeId, label,
+        model: input.model ?? undefined, detail: detailSource.slice(0, 60),
         depth: childDepth,
       });
       const res = await opts.run({
-        rootPrompt: "",
-        context: prompt,
+        rootPrompt: input.rootPrompt,
+        context: input.context,
         depth: childDepth,
         parentNodeId: subId,
-        modelOverride: model ?? undefined,
+        modelOverride: input.model ?? undefined,
         remainingBudgetUsd: rem.budgetUsd,
         remainingTimeoutMs: rem.timeoutMs,
       });
@@ -63,16 +98,27 @@ export function createRlmHandlers(opts: RlmBridgeOptions): RlmHandlers {
       opts.emitter.emitSubcallUpdated({ id: subId,
         status: "done", resultPreview: res.answer.slice(0, 200),
       });
-      return res.answer;
+      return res;
     } catch (err) {
       const msg = err instanceof Error ? err.message : String(err);
       if (subId) opts.emitter.emitSubcallUpdated({ id: subId, status: "error", detail: msg });
-      return formatError(`child RLM failed - ${msg}`);
+      return emptyResult(formatError(`child RLM failed - ${msg}`));
     }
   }
 
+  async function child(prompt: string, model: string | null, depth: number): Promise<string> {
+    const res = await childRun({
+      rootPrompt: "",
+      context: prompt,
+      depth: depth + 1,
+      model,
+    });
+    return res.answer;
+  }
+
   return {
     rlmQuery: (prompt, model, depth) => child(prompt, model, depth),
     rlmQueryBatched: (prompts, model, depth) => mapPool(prompts, opts.maxConcurrent, (p) => child(p, model, depth)),
+    childRun,
   };
 }

package/src/config/defaults.ts

@@ -21,12 +21,14 @@ export const DEFAULT_CONFIG: Readonly<RlmConfig> = Object.freeze({
   maxErrors: 5,
   orchestrator: true,
   pipeline: false,
+  maxBackwardJumps: 2,
   compaction: true,
   compactionThresholdPct: 0.65,
   python: "python3",
   sandboxInitTimeoutMs: 30_000,
   askUserQuestion: true,
   todo: true,
+  libraryLoader: true,
   rootSampling: Object.freeze({ maxTokens: 16_384 }),
   subSystemPrompt: DEFAULT_SUB_SYSTEM_PROMPT,
   subSampling: Object.freeze({ maxTokens: 8192 }),

package/src/config/settings.ts

@@ -75,6 +75,8 @@ function validateConfig(raw: unknown): Partial<RlmConfig> {
   if (orchestrator !== undefined) out.orchestrator = orchestrator;
   const pipeline = validateBoolean(r.pipeline);
   if (pipeline !== undefined) out.pipeline = pipeline;
+  const maxBackwardJumps = validateNumber(r.maxBackwardJumps, 0);
+  if (maxBackwardJumps !== undefined) out.maxBackwardJumps = maxBackwardJumps;
   const compaction = validateBoolean(r.compaction);
   if (compaction !== undefined) out.compaction = compaction;
   const compactionThresholdPct = validateNumber(r.compactionThresholdPct, 0);
@@ -92,6 +94,8 @@ function validateConfig(raw: unknown): Partial<RlmConfig> {
   if (askUserQuestion !== undefined) out.askUserQuestion = askUserQuestion;
   const todo = validateBoolean(r.todo);
   if (todo !== undefined) out.todo = todo;
+  const libraryLoader = validateBoolean(r.libraryLoader);
+  if (libraryLoader !== undefined) out.libraryLoader = libraryLoader;
   if (typeof r.subSampling === "object" && r.subSampling !== null) {
     const ss = r.subSampling as Record<string, unknown>;
     const sampling: { maxTokens?: number; temperature?: number; reasoning?: ThinkingLevel } = {};

package/src/context/library-context.ts

@@ -0,0 +1,79 @@
+/**
+ * Resolve load_library(source) into a sandbox-ready payload.
+ *
+ * Sources: local directory (repomix-packed), single file (utf-8 string), or
+ * remote git URL (shallow clone then pack). Host-side only — never runs in the sandbox.
+ */
+
+import { execFile } from "node:child_process";
+import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { isAbsolute, join, resolve } from "node:path";
+import { promisify } from "node:util";
+import { packRepository, serializeForSandbox, type ContextBundle } from "./repomix-context.ts";
+import type { Result } from "../util/errors.ts";
+import { errorMessage } from "../util/errors.ts";
+
+const execFileP = promisify(execFile);
+
+export interface LibrarySource {
+  readonly payload: unknown;     // str for a single file; ContextFile[] for a packed dir/repo
+  readonly files?: number;       // undefined for single-file payloads
+  readonly chars: number;
+}
+
+/** https://host/… or git@host:… — option-injection safe (never starts with "-"). */
+const GIT_URL = /^(https:\/\/|git@)[\w.-]+[:/]\S+$/;
+
+export async function resolveLibrarySource(
+  source: string,
+  cwd: string,
+  signal?: AbortSignal,
+): Promise<Result<LibrarySource, string>> {
+  const trimmed = source.trim();
+  if (trimmed === "") return { ok: false, error: "load_library: empty source" };
+  if (GIT_URL.test(trimmed)) return await cloneAndPack(trimmed, signal);
+  if (/^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed)) {
+    return { ok: false, error: `unsupported URL scheme (only https:// and git@ are allowed): ${trimmed}` };
+  }
+  const path = isAbsolute(trimmed) ? trimmed : resolve(cwd, trimmed);
+  let s: Awaited<ReturnType<typeof stat>>;
+  try {
+    s = await stat(path);
+  } catch {
+    return { ok: false, error: `load_library: path not found: ${path}` };
+  }
+  if (s.isDirectory()) return await packDir(path, signal);
+  const text = await readFile(path, "utf-8");
+  return { ok: true, value: { payload: text, chars: text.length } };
+}
+
+async function packDir(dir: string, signal?: AbortSignal): Promise<Result<LibrarySource, string>> {
+  const packed = await packRepository(dir, signal);       // repomix + existing per-path cache
+  if (!packed.ok) return { ok: false, error: `pack failed for ${dir} — ${packed.error}` };
+  return { ok: true, value: bundleToSource(packed.value) };
+}
+
+function bundleToSource(bundle: ContextBundle): LibrarySource {
+  const payload = serializeForSandbox(bundle);
+  // Match slot-0 contextLength for object/array payloads: JSON-serialized size,
+  // not sum-of-content-lengths (bundle.totalChars), so the model sees one ruler.
+  return {
+    payload,
+    files: bundle.totalFiles,
+    chars: JSON.stringify(payload).length,
+  };
+}
+
+async function cloneAndPack(url: string, signal?: AbortSignal): Promise<Result<LibrarySource, string>> {
+  const dir = await mkdtemp(join(tmpdir(), "rlm-lib-"));
+  try {
+    // "--" terminates options; URL regex already forbids a leading dash.
+    await execFileP("git", ["clone", "--depth", "1", "--", url, dir], { signal, timeout: 120_000 });
+    return await packDir(dir, signal);
+  } catch (err: unknown) {
+    return { ok: false, error: `git clone failed for ${url} — ${errorMessage(err)}` };
+  } finally {
+    await rm(dir, { recursive: true, force: true }).catch(() => {});
+  }
+}

package/src/core/artifacts.ts

@@ -0,0 +1,88 @@
+/**
+ * Artifact plumbing for the gated RLM pipeline: goal capture, baseline dirty-tree
+ * snapshot, and timestamped stage artifact writes under `.rlm/artifacts/`.
+ */
+import { execFileSync } from "node:child_process";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { Result } from "../util/errors.ts";
+
+export const ARTIFACTS_DIR = ".rlm/artifacts";
+
+const stamp = (): string => new Date().toISOString().replace(/[:.]/g, "-");
+
+export interface GoalCapture {
+  readonly goalPath: string;      // repo-relative
+  readonly baselinePath: string;  // repo-relative
+}
+
+export type SaveOutcome =
+  | { readonly ok: true; readonly path: string }
+  | { readonly ok: false; readonly error: string };
+
+export type GoalCaptureResult =
+  | { readonly ok: true; readonly value: GoalCapture }
+  | { readonly ok: false; readonly error: string };
+
+/**
+ * Capture the user's brief VERBATIM: no frontmatter, no headers — the raw file
+ * is the only artifact that preserves explicit user constraints unrefracted. The
+ * baseline snapshot records paths ALREADY dirty before the run, so validate
+ * judges only the run's own delta. Best-effort: git failure ⇒ empty baseline.
+ * Failures never throw (unwritable cwd etc.) — returns error for the engine to surface.
+ */
+export function captureGoal(cwd: string, brief: string): GoalCaptureResult {
+  try {
+    const ts = stamp();
+    const dir = join(ARTIFACTS_DIR, "goal");
+    mkdirSync(join(cwd, dir), { recursive: true });
+    const goalPath = join(dir, `goal-${ts}.md`);
+    writeFileSync(join(cwd, goalPath), brief, "utf-8");
+    let paths: readonly string[] = [];
+    try {
+      paths = execFileSync("git", ["status", "--short"], {
+        cwd,
+        encoding: "utf-8",
+        stdio: ["ignore", "pipe", "ignore"],
+      })
+        .split("\n")
+        .filter((l) => l.trim() !== "")
+        .map((l) => {
+          const rest = l.slice(3).trim();
+          const arrow = rest.indexOf(" -> ");
+          return arrow >= 0 ? rest.slice(arrow + 4).trim() : rest;
+        });
+    } catch {
+      paths = [];
+    }
+    const baselinePath = join(dir, `baseline-${ts}.json`);
+    writeFileSync(join(cwd, baselinePath), JSON.stringify({ paths }, null, 2), "utf-8");
+    return { ok: true, value: { goalPath, baselinePath } };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    return { ok: false, error: message };
+  }
+}
+
+/** Write a stage artifact under its dir; timestamped so runs never collide. */
+export function saveArtifact(cwd: string, dir: string, slug: string, content: string): SaveOutcome {
+  try {
+    const rel = join(ARTIFACTS_DIR, dir, `${stamp()}_${slug}.md`);
+    mkdirSync(join(cwd, ARTIFACTS_DIR, dir), { recursive: true });
+    writeFileSync(join(cwd, rel), content, "utf-8");
+    return { ok: true, path: rel };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    return { ok: false, error: message };
+  }
+}
+
+/** Read a previously saved artifact (repo-relative path). Failures never throw. */
+export function readArtifact(cwd: string, relPath: string): Result<string, string> {
+  try {
+    return { ok: true, value: readFileSync(join(cwd, relPath), "utf-8") };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    return { ok: false, error: `could not read artifact ${relPath}: ${message}` };
+  }
+}