@agentsfleet/orly 0.10.14 → 0.11.0

package/src/cli_help.ts

@@ -0,0 +1,69 @@
+const TELEMETRY_CONFIG_KEY = "telemetry";
+const ANONYMOUS_TELEMETRY_CONFIG = JSON.stringify({ [TELEMETRY_CONFIG_KEY]: "anonymous" });
+const OFF_TELEMETRY_CONFIG = JSON.stringify({ [TELEMETRY_CONFIG_KEY]: "off" });
+
+const HELP_TEXT = `orly — prove the boundary; carry the rules
+
+Gates (read-only; no PR without every criterion green or a recorded override):
+  orly gate [--accept-dirty]        run work → verify → pr; stop at first red
+  orly gate <work|verify|pr>        run one gate
+      work    does this commit conform? the declared conform command; no git
+              state, so a commit hook's own dirty tree never blocks it
+      verify  does the work hold up? configuration, docs language, and the
+              non-test verify.* set; unfinished Sections may be pushed
+      pr      can this ship? branch, tree, pushed, every spec criterion, and
+              every declared verify.* command
+  orly override <CRITERION> --reason <REASON>
+                                    empty commit with an Orly-Override trailer
+  Commands default to 30 minutes and 64 mebibytes of combined output each.
+  Set limits.<command>.timeout_ms and output_bytes in .orly/orly.json to override.
+  Exceeding either limit fails the check and prints the saved output path.
+
+Advice (explicit; hooks stay offline):
+  orly judge <plan|verify|review|document> --input <MANIFEST>
+                                    exact replay of bounded Jev advice
+  orly judge --help                 input selectors, questions and refresh
+  orly lifecycle [--json]           inspect stages, declared local checks and
+                                    resource ownership; remote adapter disabled
+
+Install (the repository is the unit — no checkout of this package required):
+  orly init [--force] [--no-hooks] [--with <PACK>] [--dry-run] [--json]
+                                    materialise rules, gates, hooks, and a
+                                    seeded .orly/orly.json. A repository that
+                                    already has an AGENTS.md keeps it: orly's
+                                    rules land as .orly/AGENTS.md, reached by a
+                                    pointer block in the file you own. One
+                                    import line lands per runtime that needs
+                                    one (CLAUDE.md, opencode.json), so the
+                                    rules load rather than wait to be noticed;
+                                    a loader you wrote yourself is left alone.
+  orly update [--force] [--with <PACK>] [--dry-run] [--json]
+                                    re-materialise at the installed engine version
+
+  --with <PACK>                     record an opt-in pack (repeatable) in
+                                    .orly/orly.json, so every clone selects it
+  --dry-run                         show what would be written; change nothing
+
+  orly doctor                       check this repository's installed rules
+                                    against what orly would write today
+  orly --version                    the installed package version
+
+Usage telemetry (off by default):
+  Consent file:                     ~/.config/agentsfleet/orly/.orly.json
+  State-root override:              AGENTSFLEET_STATE_DIR
+  Set ${ANONYMOUS_TELEMETRY_CONFIG} to send command, gate, and packaged Orly skill
+  names, outcome, failed criterion, duration, Orly version, operating system, architecture,
+  invocation type, timestamps, and random event, session, and installation IDs.
+  Set ${OFF_TELEMETRY_CONFIG} to write and send nothing.
+  ORLY_TELEMETRY=off|anonymous changes one process; ORLY_TELEMETRY_OFF=1 wins.
+  Orly never collects source, paths, repositories, branches, arguments, prompts,
+  output, environment values, raw errors, or personal and account details.
+
+Ruleset authoring (an orly checkout only — refused from an installed package):
+  orly verify                       the rules render the same twice, and the
+                                    committed copy matches that render`;
+
+export function printHelp(): void {
+  // logging: command help is the program interface.
+  console.log(HELP_TEXT);
+}

package/src/command_limits.ts

@@ -0,0 +1,20 @@
+import { z } from "zod";
+
+export const DEFAULT_COMMAND_TIMEOUT_MS = 30 * 60 * 1_000;
+export const DEFAULT_COMMAND_OUTPUT_BYTES = 64 * 1024 * 1024;
+export const SUPERVISOR_GRACE_MS = 5000;
+const MAX_COMMAND_TIMEOUT_MS = 2 ** 31 - 1 - SUPERVISOR_GRACE_MS;
+export const DEFAULT_COMMAND_LIMITS = Object.freeze({ timeout_ms: DEFAULT_COMMAND_TIMEOUT_MS, output_bytes: DEFAULT_COMMAND_OUTPUT_BYTES });
+export const commandLimitsSchema = z.object({
+  timeout_ms: z.number().int().positive().max(MAX_COMMAND_TIMEOUT_MS).optional(),
+  output_bytes: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).optional(),
+}).strict();
+export type CommandLimits = typeof DEFAULT_COMMAND_LIMITS;
+export type LaneLimits = Record<string, z.infer<typeof commandLimitsSchema>>;
+
+export function limitsFor(limits: LaneLimits | undefined, lane: string): CommandLimits {
+  return {
+    timeout_ms: limits?.[lane]?.timeout_ms ?? DEFAULT_COMMAND_TIMEOUT_MS,
+    output_bytes: limits?.[lane]?.output_bytes ?? DEFAULT_COMMAND_OUTPUT_BYTES,
+  };
+}

package/src/command_process.test.ts

@@ -0,0 +1,109 @@
+import { afterEach, expect, test } from "bun:test";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { cleanupTemporaryDirectories, ROOT, temporaryDirectory } from "./gates_test_support";
+import { UNSCOPED_ENVIRONMENT } from "./git_env";
+
+const PROCESS_SOURCE = "src/command_process.ts";
+const RESULT_FILE = "result.json";
+const ENCODING = "utf8";
+const TEST_TIMEOUT_MS = 5_000;
+const WORKLOAD_TIMEOUT_MS = 10_000;
+const CHILD_START_POLL_MS = 10;
+const CHILD_START_ATTEMPTS = 200;
+const REPEATED_STOP_REFUSAL = `const initialKill = process.kill;
+const stoppedGroups = new Set<number>();
+process.kill = ((pid: number, signal?: NodeJS.Signals | number) => {
+  if (pid < 0 && signal === "SIGKILL") {
+    if (stoppedGroups.has(pid)) throw Object.assign(new Error("group already stopped"), { code: "EPERM" });
+    stoppedGroups.add(pid);
+  }
+  return initialKill(pid, signal);
+}) as typeof process.kill;
+`;
+
+afterEach(cleanupTemporaryDirectories);
+
+test("a failed pid receipt write stops the already started workload", () => {
+  const root = temporaryDirectory();
+  const pidFile = join(root, "owned.pid");
+  const script = join(root, "command_process.ts");
+  const source = readFileSync(join(ROOT, PROCESS_SOURCE), ENCODING);
+  const waitForChild = `for (let attempt = 0; attempt < ${CHILD_START_ATTEMPTS} && !await Bun.file(${JSON.stringify(pidFile)}).exists(); attempt++) await Bun.sleep(${CHILD_START_POLL_MS});`;
+  writeFileSync(script, source.replace("writeFileSync(join(request.receipt, COMMAND_FILES.pid),", `${waitForChild}\n  writeFileSync(join(request.receipt, COMMAND_FILES.pid),`));
+  mkdirSync(join(root, "child.pid"));
+  const request = join(root, "request.json");
+  writeFileSync(request, JSON.stringify({ command: ["sh", "-c", `echo $$ > '${pidFile}'; exec sleep 10`], limits: { timeout_ms: WORKLOAD_TIMEOUT_MS, output_bytes: 4096 }, receipt: root, parent_pid: process.pid }));
+  let pid: number | undefined;
+  try {
+    const supervisor = Bun.spawnSync([process.execPath, script, request], { env: UNSCOPED_ENVIRONMENT, stdout: "pipe", stderr: "pipe", timeout: TEST_TIMEOUT_MS });
+    expect(supervisor.exitCode).not.toBe(0);
+    expect(supervisor.stderr.toString()).toContain("EISDIR");
+    expect(existsSync(pidFile)).toBeTrue();
+    const workloadPid = Number(readFileSync(pidFile, ENCODING));
+    pid = workloadPid;
+    expect(() => process.kill(workloadPid, 0)).toThrow();
+  } finally {
+    if (pid) { try { process.kill(-pid, "SIGKILL"); } catch { /* owned group already stopped */ } }
+  }
+});
+
+test("a successful deadline stop is not repeated during supervisor cleanup", () => {
+  const root = temporaryDirectory();
+  const script = join(root, "command_process.ts");
+  writeFileSync(script, REPEATED_STOP_REFUSAL + readFileSync(join(ROOT, PROCESS_SOURCE), ENCODING));
+  const request = join(root, "request.json");
+  writeFileSync(request, JSON.stringify({ command: ["sh", "-c", "echo measured; sleep 3"], limits: { timeout_ms: 100, output_bytes: 4096 }, receipt: root, parent_pid: process.pid }));
+  const child = Bun.spawnSync([process.execPath, script, request], { env: UNSCOPED_ENVIRONMENT, stdout: "pipe", stderr: "pipe", timeout: TEST_TIMEOUT_MS });
+  expect(child.exitCode, child.stderr.toString()).toBe(0);
+  expect(child.stderr.toString()).toBe("");
+  expect(JSON.parse(readFileSync(join(root, RESULT_FILE), ENCODING)).failure).toBe("deadline exceeded");
+  expect(readFileSync(join(root, "stdout.log"), ENCODING)).toBe("measured\n");
+});
+
+test("killing the gate owner stops its detached workload", async () => {
+  const root = temporaryDirectory();
+  const pidFile = join(root, "owned.pid");
+  const wrapper = join(root, "gate.ts");
+  writeFileSync(wrapper, `import { runCommand } from ${JSON.stringify(join(ROOT, "src/command_runner.ts"))}; runCommand(${JSON.stringify(root)}, ["sh", "-c", "echo $$ > owned.pid; sleep 10"], {timeout_ms: ${WORKLOAD_TIMEOUT_MS}, output_bytes: 4096});`);
+  const gate = Bun.spawn([process.execPath, wrapper], { cwd: root, env: { ...UNSCOPED_ENVIRONMENT, TMPDIR: root }, stdout: "pipe", stderr: "pipe" });
+  let pid: number | undefined;
+  try {
+    for (let attempt = 0; attempt < 200 && !existsSync(pidFile); attempt++) await Bun.sleep(10);
+    expect(existsSync(pidFile)).toBeTrue();
+    pid = Number(readFileSync(pidFile, ENCODING));
+    gate.kill("SIGKILL");
+    await gate.exited;
+    let alive = true;
+    for (let attempt = 0; attempt < 100 && alive; attempt++) {
+      await Bun.sleep(10);
+      try { process.kill(pid, 0); } catch { alive = false; }
+    }
+    expect(alive).toBeFalse();
+  } finally {
+    gate.kill("SIGKILL");
+    await gate.exited;
+    if (pid) { try { process.kill(-pid, "SIGKILL"); } catch { /* owned group already stopped */ } }
+  }
+}, TEST_TIMEOUT_MS);
+
+test.each(["SIGTERM", "SIGINT", "SIGHUP"] as const)("supervisor %s stops its owned workload", async (signal) => {
+  const root = temporaryDirectory();
+  const request = join(root, "request.json");
+  writeFileSync(request, JSON.stringify({ command: ["sh", "-c", "echo ready; sleep 10"], limits: { timeout_ms: WORKLOAD_TIMEOUT_MS, output_bytes: 4096 }, receipt: root, parent_pid: process.pid }));
+  const supervisor = Bun.spawn([process.execPath, join(ROOT, PROCESS_SOURCE), request], { env: UNSCOPED_ENVIRONMENT, stdout: "pipe", stderr: "pipe" });
+  try {
+    for (let attempt = 0; attempt < 200 && !existsSync(join(root, "child.pid")); attempt++) await Bun.sleep(10);
+    expect(existsSync(join(root, "child.pid"))).toBeTrue();
+    await Bun.sleep(20);
+    supervisor.kill(signal);
+    expect(await supervisor.exited).toBe(0);
+    expect(JSON.parse(readFileSync(join(root, RESULT_FILE), ENCODING)).failure).toBe(`supervisor received ${signal}`);
+    const pid = Number(readFileSync(join(root, "child.pid"), ENCODING));
+    expect(() => process.kill(pid, 0)).toThrow();
+  } finally {
+    supervisor.kill("SIGKILL");
+    await supervisor.exited;
+  }
+}, TEST_TIMEOUT_MS);

package/src/command_process.ts

@@ -0,0 +1,88 @@
+import { appendFileSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import type { CommandLimits } from "./command_limits";
+
+const PRIVATE_FILE_MODE = 0o600;
+const TEXT_ENCODING = "utf8";
+const KILL_SIGNAL = "SIGKILL";
+const MISSING_PROCESS = "ESRCH";
+const PIPE_OUTPUT = "pipe";
+const OWNER_POLL_MS = 100;
+const TERMINATION_SIGNALS = ["SIGTERM", "SIGINT", "SIGHUP"] as const;
+export const COMMAND_FILES = {
+  result: "result.json", pid: "child.pid", stdout: "stdout.log", stderr: "stderr.log",
+} as const;
+
+type Request = { command: string[]; limits: CommandLimits; receipt: string; parent_pid: number };
+export type CommandResult = { exit_code: number | null; elapsed_ms: number; output_bytes: number; failure?: string };
+
+export function stopOwnedGroup(pid: number): void {
+  if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error("invalid owned process group");
+  try { process.kill(-pid, KILL_SIGNAL); }
+  catch (error) { if ((error as NodeJS.ErrnoException).code !== MISSING_PROCESS) throw error; }
+}
+
+async function capture(stream: ReadableStream<Uint8Array>, file: string, onBytes: (size: number) => void): Promise<void> {
+  for await (const chunk of stream) {
+    appendFileSync(file, chunk);
+    onBytes(chunk.byteLength);
+  }
+}
+
+async function run(request: Request): Promise<CommandResult> {
+  const started = performance.now();
+  const stdout = join(request.receipt, COMMAND_FILES.stdout);
+  const stderr = join(request.receipt, COMMAND_FILES.stderr);
+  for (const file of [stdout, stderr]) writeFileSync(file, "", { mode: PRIVATE_FILE_MODE });
+  let child;
+  try {
+    child = Bun.spawn(request.command, { detached: true, stdin: "ignore", stdout: PIPE_OUTPUT, stderr: PIPE_OUTPUT });
+  } catch (error) {
+    return { exit_code: null, elapsed_ms: Math.ceil(performance.now() - started), output_bytes: 0, failure: `${request.command[0] ?? "command"} could not be run: ${error instanceof Error ? error.message : String(error)}` };
+  }
+  let stopped = false;
+  try {
+    writeFileSync(join(request.receipt, COMMAND_FILES.pid), String(child.pid), { mode: PRIVATE_FILE_MODE });
+    let outputBytes = 0;
+    let failure: string | undefined;
+    const stop = (reason: string) => {
+      failure ??= reason;
+      if (stopped) return;
+      stopOwnedGroup(child.pid);
+      stopped = true;
+    };
+    const timer = setTimeout(() => stop("deadline exceeded"), request.limits.timeout_ms);
+    const owner = setInterval(() => {
+      try { process.kill(request.parent_pid, 0); }
+      catch (error) { if ((error as NodeJS.ErrnoException).code === MISSING_PROCESS) stop("gate owner exited"); }
+    }, OWNER_POLL_MS);
+    const signals = TERMINATION_SIGNALS.map((signal) => {
+      const handler = () => stop(`supervisor received ${signal}`);
+      process.on(signal, handler);
+      return { signal, handler };
+    });
+    try {
+      const onBytes = (size: number) => {
+        outputBytes += size;
+        if (outputBytes > request.limits.output_bytes) stop("output limit exceeded");
+      };
+      await Promise.all([capture(child.stdout, stdout, onBytes), capture(child.stderr, stderr, onBytes), child.exited]);
+      if (child.signalCode) failure ??= `stopped by ${child.signalCode}`;
+      return { exit_code: child.exitCode, elapsed_ms: Math.ceil(performance.now() - started), output_bytes: outputBytes, ...(failure ? { failure } : {}) };
+    } finally {
+      clearTimeout(timer);
+      clearInterval(owner);
+      for (const { signal, handler } of signals) process.off(signal, handler);
+    }
+  } finally {
+    if (!stopped) stopOwnedGroup(child.pid);
+    await child.exited;
+  }
+}
+
+if (import.meta.main) {
+  const request: Request = JSON.parse(readFileSync(Bun.argv[2]!, TEXT_ENCODING));
+  const result = await run(request);
+  writeFileSync(join(request.receipt, COMMAND_FILES.result), JSON.stringify(result), { mode: PRIVATE_FILE_MODE });
+}

package/src/command_runner.test.ts

@@ -0,0 +1,88 @@
+import { afterEach, expect, test } from "bun:test";
+import { existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { runCommand } from "./command_runner";
+import { criteriaFor } from "./criteria";
+import { CONFIG_PATH } from "./config";
+import { cleanupTemporaryDirectories, modelFor, newRepository } from "./gates_test_support";
+
+const PRIVATE_DIRECTORY_MODE = 0o700;
+const PRIVATE_FILE_MODE = 0o600;
+const PERMISSION_MASK = 0o777;
+const CHILD_FILE = "child.pid";
+const UNIT_LANE = "verify.unit";
+const roots: string[] = [];
+
+afterEach(() => {
+  cleanupTemporaryDirectories();
+  for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
+});
+
+function scratch(): string {
+  const root = mkdtempSync(join(tmpdir(), "orly-command-test-"));
+  roots.push(root);
+  return root;
+}
+
+function receipt(detail: string): string {
+  const path = detail.split("complete captured output: ")[1];
+  expect(path).toBeDefined();
+  roots.push(path!);
+  expect(statSync(path!).mode & PERMISSION_MASK).toBe(PRIVATE_DIRECTORY_MODE);
+  for (const file of ["stdout.log", "stderr.log"]) expect(statSync(join(path!, file)).mode & PERMISSION_MASK).toBe(PRIVATE_FILE_MODE);
+  return path!;
+}
+
+async function expectChildStopped(root: string): Promise<void> {
+  expect(existsSync(join(root, CHILD_FILE))).toBeTrue();
+  const pid = Number(readFileSync(join(root, CHILD_FILE), "utf8").trim());
+  for (let attempt = 0; attempt < 100; attempt++) {
+    try { process.kill(pid, 0); } catch { return; }
+    await Bun.sleep(10);
+  }
+  throw new Error(`owned child ${pid} survived the command limit`);
+}
+
+test("a deadline stops the owned child group and keeps captured output", async () => {
+  const root = scratch();
+  const result = runCommand(root, ["sh", "-c", `sleep 30 & echo $! > ${CHILD_FILE}; echo started; echo detail >&2; wait`], { timeout_ms: 300, output_bytes: 4096 });
+  expect(result.ok).toBeFalse();
+  expect(result.detail).toContain("deadline exceeded");
+  expect(result.detail).toContain("300 ms deadline");
+  const path = receipt(result.detail);
+  expect(readFileSync(join(path, "stdout.log"), "utf8")).toBe("started\n");
+  expect(readFileSync(join(path, "stderr.log"), "utf8")).toBe("detail\n");
+  await expectChildStopped(root);
+});
+
+test("the output budget covers both streams and stops the owned child group", async () => {
+  const root = scratch();
+  const chunk = "123456789012345678901234";
+  const started = performance.now();
+  const result = runCommand(root, ["sh", "-c", `sleep 30 & echo $! > ${CHILD_FILE}; printf ${chunk}; printf ${chunk} >&2; wait`], { timeout_ms: 3000, output_bytes: 32 });
+  expect(performance.now() - started).toBeLessThan(1500);
+  expect(result.ok).toBeFalse();
+  expect(result.detail).toContain("output limit exceeded");
+  expect(result.detail).toContain("48 bytes / 32 bytes");
+  const path = receipt(result.detail);
+  expect(readFileSync(join(path, "stdout.log"), "utf8")).toBe(chunk);
+  expect(readFileSync(join(path, "stderr.log"), "utf8")).toBe(chunk);
+  await expectChildStopped(root);
+});
+
+test("an explicit lane override applies through the public gate criteria", async () => {
+  const root = newRepository();
+  const model = await modelFor(root);
+  writeFileSync(join(root, CONFIG_PATH), JSON.stringify({
+    schema_version: 1, commands: { conform: [["true"]], [UNIT_LANE]: [["sh", "-c", "echo measured; sleep 3"]] },
+    limits: { [UNIT_LANE]: { timeout_ms: 200, output_bytes: 4096 } },
+  }));
+  const context = { root, model, acceptDirty: false };
+  const check = criteriaFor("pr", context).find((entry) => entry.name === `cmd.${UNIT_LANE}`)!;
+  const result = check.evaluate(context);
+  expect(result.ok).toBeFalse();
+  expect(result.detail).toContain("200 ms deadline");
+  expect(readFileSync(join(receipt(result.detail), "stdout.log"), "utf8")).toBe("measured\n");
+});

package/src/command_runner.ts

@@ -0,0 +1,33 @@
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { DEFAULT_COMMAND_LIMITS, SUPERVISOR_GRACE_MS, type CommandLimits } from "./command_limits";
+import { UNSCOPED_ENVIRONMENT } from "./git_env";
+import { COMMAND_FILES, stopOwnedGroup, type CommandResult } from "./command_process";
+import type { Verdict } from "./criteria_support";
+
+const PRIVATE_FILE_MODE = 0o600;
+const TEXT_ENCODING = "utf8";
+const PIPE_OUTPUT = "pipe";
+const RECEIPT_PREFIX = "orly-command-";
+
+export function runCommand(root: string, command: string[], limits: CommandLimits = DEFAULT_COMMAND_LIMITS): Verdict {
+  const receipt = mkdtempSync(join(tmpdir(), RECEIPT_PREFIX));
+  const request = join(receipt, "request.json");
+  writeFileSync(request, JSON.stringify({ command, limits, receipt, parent_pid: process.pid }), { mode: PRIVATE_FILE_MODE });
+  const supervisor = Bun.spawnSync([process.execPath, join(import.meta.dir, "command_process.ts"), request], {
+    cwd: root, env: UNSCOPED_ENVIRONMENT, stdout: PIPE_OUTPUT, stderr: PIPE_OUTPUT,
+    detached: true, timeout: limits.timeout_ms + SUPERVISOR_GRACE_MS, killSignal: "SIGKILL",
+  });
+  if (supervisor.exitCode !== 0) {
+    stopOwnedGroup(supervisor.pid);
+    try { stopOwnedGroup(Number(readFileSync(join(receipt, COMMAND_FILES.pid), TEXT_ENCODING))); } catch { /* child may not have started */ }
+    return { ok: false, detail: `command supervisor failed (${supervisor.signalCode ?? supervisor.exitCode}); complete captured output: ${receipt}; ${supervisor.stderr.toString().trim()}` };
+  }
+  const result: CommandResult = JSON.parse(readFileSync(join(receipt, COMMAND_FILES.result), TEXT_ENCODING));
+  if (result.failure) return { ok: false, detail: `${result.failure}; ${result.elapsed_ms} ms / ${limits.timeout_ms} ms deadline; ${result.output_bytes} bytes / ${limits.output_bytes} bytes output budget; complete captured output: ${receipt}` };
+  const output = [COMMAND_FILES.stdout, COMMAND_FILES.stderr].map((file) => readFileSync(join(receipt, file), TEXT_ENCODING)).filter(Boolean).join("\n").trim();
+  rmSync(receipt, { recursive: true, force: true });
+  return { ok: result.exit_code === 0, detail: output ? `exit ${result.exit_code}:\n${output}` : result.exit_code === 0 ? "exit 0" : `exit ${result.exit_code}: no output` };
+}

package/src/comprehension_cache.test.ts

@@ -0,0 +1,123 @@
+import { afterEach, expect, test } from "bun:test";
+import { copyFileSync, mkdirSync, readFileSync, readdirSync, symlinkSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { cleanupTemporaryDirectories, newRepository, ROOT } from "./gates_test_support";
+import { UNSCOPED_ENVIRONMENT } from "./git_env";
+
+const SCRIPT = "evals/llms/run.sh";
+const AGENT_NAMES = ["claude", "codex", "amp", "opencode"];
+const CALLS = "calls.txt";
+const AGENTS = "AGENTS.md";
+const FACADE = "dispatch/write_fixture.md";
+const PIPE_OUTPUT = "pipe";
+const TEXT_ENCODING = "utf8";
+const CASE_TIMEOUT_MS = 30_000;
+
+afterEach(cleanupTemporaryDirectories);
+
+function fixture(missingHost = false) {
+  const root = newRepository();
+  for (const file of [SCRIPT, "evals/llms/agents.sh", "evals/llms/fixtures.sh", "evals/llms/cache.ts", "skills/orly-write-unit-test/SKILL.md", "skills/orly-write-integration-test/SKILL.md"]) {
+    mkdirSync(join(root, file, ".."), { recursive: true });
+    copyFileSync(join(ROOT, file), join(root, file));
+  }
+  mkdirSync(join(root, "dispatch"));
+  writeFileSync(join(root, AGENTS), "# Deterministic rules\n");
+  writeFileSync(join(root, FACADE), "Require the declared check.\n");
+  writeFileSync(join(root, "evals/llms/fixtures.jsonl"), JSON.stringify({ id: "one", mode: "control", expect: "YES", q: "Does the rule require its check?", why: "Local adapter control.", ctx: [FACADE] }) + "\n");
+  const tools = join(root, "tools");
+  mkdirSync(tools);
+  for (const name of ["bash", "sh", "bun", "python3", "timeout", "dirname", "cat", "mktemp", "rm", "ln", "chmod", "cut", "grep", "tr", "sed", "awk", "mkdir", "basename", "git"]) {
+    const target = Bun.which(name);
+    if (target) symlinkSync(target, join(tools, name));
+  }
+  for (const host of AGENT_NAMES.filter((name) => !missingHost || name !== "opencode")) {
+    writeFileSync(join(tools, host), `#!/bin/sh
+printf '%s\n' '${host}' >> "$ORLY_STUB_CALLS"
+answer=YES
+if [ '${host}' = opencode ] && [ "\${ORLY_STUB_FAIL:-0}" = 1 ]; then answer=NO; fi
+if [ '${host}' = codex ]; then
+  cat >/dev/null
+  while [ "$#" -gt 0 ]; do
+    if [ "$1" = --output-last-message ]; then shift; printf 'VERDICT: %s\n' "$answer" > "$1"; break; fi
+    shift
+  done
+else
+  if [ '${host}' != opencode ]; then cat >/dev/null; fi
+  printf 'VERDICT: %s\n' "$answer"
+fi
+`, { mode: 0o755 });
+  }
+  const env = { ...UNSCOPED_ENVIRONMENT, PATH: tools, ORLY_STUB_CALLS: join(root, CALLS) };
+  return { root, env };
+}
+
+function run(f: ReturnType<typeof fixture>, args: string[] = [], fail = true) {
+  const result = Bun.spawnSync([Bun.which("bash")!, join(f.root, SCRIPT), ...args], {
+    cwd: f.root, env: { ...f.env, ORLY_STUB_FAIL: fail ? "1" : "0" }, stdout: PIPE_OUTPUT, stderr: PIPE_OUTPUT, timeout: 30_000,
+  });
+  return { code: result.exitCode, output: result.stdout.toString() + result.stderr.toString() };
+}
+
+function calls(f: ReturnType<typeof fixture>): number { return readFileSync(join(f.root, CALLS), TEXT_ENCODING).trim().split("\n").length; }
+
+test("changed prompt bytes invalidate saved results without a new commit", () => {
+  const f = fixture();
+  const first = run(f);
+  expect(first.code, first.output).toBe(1);
+  expect(calls(f)).toBe(4);
+  expect(run(f).output).toContain("resumed from journal");
+  expect(calls(f)).toBe(4);
+  writeFileSync(join(f.root, AGENTS), "# Changed deterministic rule\n");
+  const changed = run(f);
+  expect(changed.code).toBe(1);
+  expect(changed.output).not.toContain("resumed from journal");
+  expect(calls(f)).toBe(8);
+  writeFileSync(join(f.root, FACADE), "A changed check remains required.\n");
+  expect(run(f).output).not.toContain("resumed from journal");
+  expect(calls(f)).toBe(12);
+}, CASE_TIMEOUT_MS);
+
+test("host selection and adapter settings cannot reuse incompatible saved proof", () => {
+  const f = fixture();
+  run(f);
+  const partial = run(f, ["--agent", "claude"]);
+  expect(partial.code).toBe(0);
+  expect(partial.output).toContain("partial comprehension coverage");
+  expect(partial.output).not.toContain("live comprehension passed");
+  expect(calls(f)).toBe(5);
+  mkdirSync(join(f.root, ".claude"));
+  writeFileSync(join(f.root, ".claude/settings.json"), JSON.stringify({ model: "changed-local-control" }));
+  expect(run(f).output).not.toContain("resumed from journal");
+  expect(calls(f)).toBe(9);
+}, CASE_TIMEOUT_MS);
+
+test("malformed journal counts are discarded instead of becoming a pass", () => {
+  const f = fixture();
+  run(f);
+  const journalRoot = join(f.root, ".llmevals-journal");
+  const runDirectory = join(journalRoot, readdirSync(journalRoot)[0]!);
+  const path = join(runDirectory, "claude");
+  const saved = JSON.parse(readFileSync(path, TEXT_ENCODING));
+  writeFileSync(path, JSON.stringify({ ...saved, status: "PASS", correct: 0, total: 0 }));
+  expect(run(f).code).toBe(1);
+  expect(calls(f)).toBe(5);
+}, CASE_TIMEOUT_MS);
+
+test("a missing required host keeps full coverage incomplete", () => {
+  const f = fixture(true);
+  const result = run(f, [], false);
+  expect(result.code).toBe(1);
+  expect(result.output).toContain("requested agent absent: opencode");
+  expect(result.output).not.toContain("live comprehension passed");
+  expect(calls(f)).toBe(3);
+}, CASE_TIMEOUT_MS);
+
+test("full completion requires every selected fixture on all required local stub hosts", () => {
+  const f = fixture();
+  const result = run(f, [], false);
+  expect(result.code).toBe(0);
+  expect(result.output).toContain("graded=4/4");
+  expect(calls(f)).toBe(4);
+}, CASE_TIMEOUT_MS);

package/src/config.test.ts

@@ -3,7 +3,8 @@ import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 
-import { CONFIG_PATH, contentDigest, managedDrift, readConfig, readConfigSync, seedConfig, selectPacks, staleVersion, writeConfig } from "./config";
+import { selectPacks } from "./config_discovery";
+import { CONFIG_PATH, contentDigest, managedDrift, readConfig, readConfigSync, seedConfig, staleVersion, writeConfig } from "./config";
 import { RulesModel } from "./model";
 
 const ROOT = resolve(import.meta.dir, "..");
@@ -20,7 +21,7 @@ function scratch(): string {
 const temporary: string[] = [];
 
 function writeConfigText(root: string, body: unknown): void {
-  mkdirSync(join(root, ".oracle"), { recursive: true });
+  mkdirSync(join(root, ".orly"), { recursive: true });
   writeFileSync(join(root, CONFIG_PATH), JSON.stringify(body));
 }
 
@@ -236,7 +237,7 @@ describe("managedDrift", () => {
   });
 
   // The failure this pair exists for: agentsfleet ignored .claude/skills/ while
-  // .oracle/orly.json declared four files under it managed. Every local
+  // .orly/orly.json declared four files under it managed. Every local
   // checkout passed `orly doctor` because the files sat on disk, and CI failed
   // against a clone that never received them.
   test("a managed file git refuses to track is drift, even though it is present", async () => {
@@ -294,7 +295,7 @@ describe("writeConfig", () => {
     expect((await readConfig(root))?.managed).toEqual(["aaa.md", "zzz.md"]);
   });
 
-  test("writeConfig creates the .oracle/ parent directory", async () => {
+  test("writeConfig creates the .orly/ parent directory", async () => {
     const root = scratch();
     await writeConfig(root, await seedConfig(root));
     expect(existsSync(join(root, CONFIG_PATH))).toBeTrue();