@agentsfleet/orly 0.10.14 → 0.12.0

package/src/judgments/runner.integration.test.ts

@@ -0,0 +1,165 @@
+import { expect, test } from "bun:test";
+
+import { existsSync, mkdirSync, readdirSync, symlinkSync } from "node:fs";
+import { join } from "node:path";
+
+import { OrlyError } from "../model";
+import { REPLAY_DIRECTORY } from "./constants";
+import { prepareItem } from "./evidence";
+import { replayDirectory } from "./replay";
+import { judge } from "./runner";
+import { choiceReply, SOURCE, SOURCE_FILE, TestProject } from "./test_support";
+
+test("live evidence becomes exact offline replay without another request", async () => {
+  using project = new TestProject();
+  let requests = 0;
+  let scans = 0;
+  const dependencies = { credential: "test-credential", scan: async () => { scans++; }, fetch: async () => { requests++; return Response.json(choiceReply()); } };
+  const live = await judge(project.root, project.manifest(), true, dependencies);
+  expect(live.requests).toBe(1);
+  expect(live.results[0]).toMatchObject({ status: "complete", mode: "live", assessment: { concern: true, decision: "weak" } });
+  const replay = await judge(project.root, project.manifest(), false, dependencies);
+  expect(replay.requests).toBe(0);
+  expect(replay.results[0]).toMatchObject({ status: "complete", mode: "replay" });
+  expect(requests).toBe(1);
+  expect(scans).toBe(1);
+  expect(replay.results[0]?.status === "complete" && replay.results[0].answer).toEqual(live.results[0]?.status === "complete" && live.results[0].answer);
+});
+
+test("missing replay never falls through to live inference", async () => {
+  using project = new TestProject();
+  let calls = 0;
+  const result = await judge(project.root, project.manifest(), false, { credential: "unused", scan: async () => { calls++; }, fetch: async () => { calls++; return Response.json(choiceReply()); } });
+  expect(result.results[0]).toMatchObject({ status: "incomplete", requests: 0 });
+  expect(result.requests).toBe(0);
+  expect(calls).toBe(0);
+  expect(existsSync(join(project.root, REPLAY_DIRECTORY))).toBe(false);
+});
+
+test("missing credentials or failed scanning prevent all uploads", async () => {
+  using project = new TestProject();
+  let uploads = 0;
+  const send = async () => { uploads++; return Response.json(choiceReply()); };
+  const missing = await judge(project.root, project.manifest(), true, { credential: undefined, scan: async () => {}, fetch: send });
+  expect(missing.results[0]).toMatchObject({ status: "incomplete", requests: 0 });
+  const refused = await judge(project.root, project.manifest(), true, { credential: "test-value", scan: async () => { throw new OrlyError("Scanner refused input."); }, fetch: send });
+  expect(refused.results[0]).toMatchObject({ status: "incomplete", reason: "Scanner refused input.", requests: 0 });
+  expect(uploads).toBe(0);
+  expect(existsSync(join(project.root, REPLAY_DIRECTORY))).toBe(false);
+});
+
+test("an invalid later item stops the whole batch before upload", async () => {
+  using project = new TestProject();
+  let calls = 0;
+  const item = { ...project.item(), id: "missing", evidence: project.item().evidence.map((reference) => ({ ...reference, path: "missing.ts" })) };
+  const report = await judge(project.root, { stage: "verify", items: [project.item(), item] }, true, { credential: "unused", scan: async () => { calls++; }, fetch: async () => { calls++; return Response.json(choiceReply()); } });
+  expect(report.requests).toBe(0);
+  expect(report.results.every((result) => result.status === "incomplete")).toBe(true);
+  expect(calls).toBe(0);
+});
+
+test.each([true, false])("a replay symlink stops the whole batch before upload, target exists: %s", async (targetExists) => {
+  using project = new TestProject();
+  using outside = new TestProject();
+  const blocked = { ...project.item(), id: "blocked", requirement: `${project.item().requirement} Include the requested locale.` };
+  const prepared = await prepareItem(project.root, blocked);
+  const target = join(outside.root, targetExists ? SOURCE_FILE : "absent.json");
+  symlinkSync(target, join(replayDirectory(project.root, true), `${prepared.identity}.json`));
+  let uploads = 0;
+  const report = await judge(project.root, { stage: "verify", items: [project.item(), blocked] }, true, {
+    credential: "unused", scan: async () => {}, fetch: async () => { uploads++; return Response.json(choiceReply()); },
+  });
+  expect(report.requests).toBe(0);
+  expect(uploads).toBe(0);
+  expect(report.results).toHaveLength(2);
+  for (const result of report.results) expect(result).toMatchObject({ status: "incomplete", requests: 0, reason: "Replay file must not be a symlink." });
+});
+
+test("a directory at a later replay filename stops the whole batch before upload", async () => {
+  using project = new TestProject();
+  const blocked = { ...project.item(), id: "blocked", requirement: `${project.item().requirement} Include the requested locale.` };
+  const prepared = await prepareItem(project.root, blocked);
+  mkdirSync(join(replayDirectory(project.root, true), `${prepared.identity}.json`));
+  let uploads = 0;
+  const report = await judge(project.root, { stage: "verify", items: [project.item(), blocked] }, true, {
+    credential: "unused", scan: async () => {}, fetch: async () => { uploads++; return Response.json(choiceReply()); },
+  });
+  expect(report.requests).toBe(0);
+  expect(uploads).toBe(0);
+  expect(report.results).toHaveLength(2);
+  for (const result of report.results) expect(result).toMatchObject({ status: "incomplete", requests: 0, reason: "Replay file must be a regular file." });
+});
+
+test("invalid provider replies count the attempt and never create a valid replay", async () => {
+  using project = new TestProject();
+  const dependencies = { credential: "unused", scan: async () => {}, fetch: async () => Response.json({ model: "unexpected" }) };
+  const live = await judge(project.root, project.manifest(), true, dependencies);
+  expect(live.results[0]).toMatchObject({ status: "incomplete", requests: 1, usage: null });
+  expect(live.requests).toBe(1);
+  const replay = await judge(project.root, project.manifest(), false, dependencies);
+  expect(replay.results[0]).toMatchObject({ status: "incomplete", requests: 0 });
+  project.write(SOURCE_FILE, SOURCE.replace("DateTimeFormat(locale", 'DateTimeFormat("en-US"'));
+  expect((await judge(project.root, project.manifest(), false, dependencies)).results[0]?.status).toBe("incomplete");
+});
+
+test("a later scanner refusal leaves every item incomplete without upload", async () => {
+  using project = new TestProject();
+  const later = { ...project.item(), id: "later", requirement: `${project.item().requirement} Include the requested locale.` };
+  let scans = 0;
+  let uploads = 0;
+  const report = await judge(project.root, { stage: "verify", items: [project.item(), later] }, true, {
+    credential: "unused", scan: async () => { if (++scans === 2) throw new OrlyError("Scanner refused the later input."); },
+    fetch: async () => { uploads++; return Response.json(choiceReply()); },
+  });
+  expect(scans).toBe(2);
+  expect(uploads).toBe(0);
+  expect(report.requests).toBe(0);
+  for (const result of report.results) expect(result).toMatchObject({ status: "incomplete", requests: 0, reason: "Scanner refused the later input." });
+  expect(existsSync(join(project.root, REPLAY_DIRECTORY))).toBe(false);
+});
+
+test("a later provider refusal preserves earlier evidence and never retries", async () => {
+  using project = new TestProject();
+  const later = { ...project.item(), id: "later", requirement: `${project.item().requirement} Include the requested locale.` };
+  const manifest = { stage: "verify" as const, items: [project.item(), later] };
+  let uploads = 0;
+  const dependencies = {
+    credential: "unused", scan: async () => {},
+    fetch: async () => ++uploads === 2 ? new Response("private detail", { status: 503 }) : Response.json(choiceReply()),
+  };
+  const report = await judge(project.root, manifest, true, dependencies);
+  expect(uploads).toBe(2);
+  expect(report.requests).toBe(2);
+  expect(report.results[0]).toMatchObject({ status: "complete", requests: 1 });
+  expect(report.results[1]).toMatchObject({ status: "incomplete", requests: 1, usage: null });
+  const retry = await judge(project.root, manifest, false, dependencies);
+  expect(retry.requests).toBe(0);
+  expect(uploads).toBe(2);
+  expect(retry.results[0]?.status).toBe("complete");
+  expect(retry.results[1]?.status).toBe("incomplete");
+});
+
+test("a later replay failure after upload preserves usage and earlier evidence", async () => {
+  using project = new TestProject();
+  const later = { ...project.item(), id: "later", requirement: `${project.item().requirement} Include the requested locale.` };
+  const prepared = await prepareItem(project.root, later);
+  const manifest = { stage: "verify" as const, items: [project.item(), later] };
+  let uploads = 0;
+  const dependencies = {
+    credential: "unused", scan: async () => {}, fetch: async () => {
+      if (++uploads === 2) mkdirSync(join(project.root, REPLAY_DIRECTORY, `${prepared.identity}.json`));
+      return Response.json(choiceReply());
+    },
+  };
+  const report = await judge(project.root, manifest, true, dependencies);
+  expect(uploads).toBe(2);
+  expect(report.requests).toBe(2);
+  expect(report.results[0]).toMatchObject({ status: "complete", requests: 1 });
+  expect(report.results[1]).toMatchObject({ status: "incomplete", requests: 1, usage: choiceReply().usage, reason: "Replay file must be a regular file." });
+  expect(readdirSync(join(project.root, REPLAY_DIRECTORY)).filter((name) => name.startsWith(".pending-"))).toEqual([]);
+  const retry = await judge(project.root, manifest, false, dependencies);
+  expect(retry.requests).toBe(0);
+  expect(uploads).toBe(2);
+  expect(retry.results[0]?.status).toBe("complete");
+  expect(retry.results[1]?.status).toBe("incomplete");
+});

package/src/judgments/runner.ts

@@ -0,0 +1,97 @@
+import type { CompletedItem, IncompleteItem, Item, Manifest, Prepared, Report } from "./types";
+import type { Reply } from "./wire";
+
+import { OrlyError } from "../model";
+import { assess } from "./advice";
+import { JUDGMENT_MODE, MODEL, RESULT_STATUS } from "./constants";
+import { prepareItem } from "./evidence";
+import { validateManifest } from "./questions";
+import { readReplay, replayDestination, writeReplay } from "./replay";
+import { scanUpload } from "./scan";
+import { requestAdvice, type Fetch } from "./transport";
+
+type Dependencies = { fetch: Fetch; scan: (request: string) => Promise<void>; credential: string | undefined };
+type Preparation = { status: "prepared"; value: Prepared } | IncompleteItem;
+const PREPARED_STATUS = "prepared";
+const ENVIRONMENT = Object.freeze({ credential: Bun.env.TYPESAFE_API_KEY });
+const PREFLIGHT_REASON = "Input preflight is incomplete; no live requests were made.";
+
+export async function judge(root: string, manifest: Manifest, refresh: boolean, dependencies: Dependencies = {
+  fetch, scan: scanUpload, credential: ENVIRONMENT.credential,
+}): Promise<Report> {
+  validateManifest(manifest, manifest.stage);
+  const prepared: Preparation[] = [];
+  for (const item of manifest.items) {
+    try { prepared.push({ status: PREPARED_STATUS, value: await prepareItem(root, item) }); }
+    catch (error) { prepared.push(incomplete(item, error)); }
+  }
+  if (refresh) {
+    if (prepared.some((item) => item.status === RESULT_STATUS.incomplete)) return failedPreflight(manifest, prepared);
+    const failure = await livePreflight(root, prepared, dependencies);
+    if (failure) return { stage: manifest.stage, advisory: true, requests: 0, results: manifest.items.map((item) => incomplete(item, failure)) };
+  }
+  const results: Report["results"] = [];
+  // MAX_ITEMS bounds this sequential loop. Each item permits at most one request and no retry.
+  for (const item of prepared) {
+    if (item.status === RESULT_STATUS.incomplete) results.push(item);
+    else results.push(await runItem(root, item.value, refresh, dependencies));
+  }
+  return { stage: manifest.stage, advisory: true, requests: results.reduce((total, item) => total + item.requests, 0), results };
+}
+
+async function livePreflight(root: string, prepared: Preparation[], dependencies: Dependencies): Promise<OrlyError | undefined> {
+  try {
+    if (!dependencies.credential?.trim()) throw new OrlyError("TYPESAFE_API_KEY is missing; nothing was uploaded.");
+    for (const item of prepared) if (item.status === PREPARED_STATUS) await dependencies.scan(item.value.request);
+    for (const item of prepared) if (item.status === PREPARED_STATUS) replayDestination(root, item.value);
+    return undefined;
+  } catch (error) {
+    return new OrlyError(safeReason(error));
+  }
+}
+
+async function runItem(root: string, prepared: Prepared, refresh: boolean, dependencies: Dependencies): Promise<CompletedItem | IncompleteItem> {
+  const started = performance.now();
+  let requests = 0;
+  let reply: Reply | undefined;
+  try {
+    if (!refresh) {
+      const cached = await readReplay(root, prepared);
+      return complete(prepared, cached.reply, cached.elapsedMs, JUDGMENT_MODE.replay);
+    }
+    const credential = dependencies.credential;
+    if (!credential) throw new OrlyError("Provider credential is unavailable.");
+    requests = 1;
+    reply = await requestAdvice(prepared, credential, dependencies.fetch);
+    const elapsedMs = performance.now() - started;
+    await writeReplay(root, prepared, reply, elapsedMs);
+    return complete(prepared, reply, elapsedMs, JUDGMENT_MODE.live);
+  } catch (error) {
+    return { ...incomplete(prepared.item, error), requests, elapsedMs: performance.now() - started, usage: reply?.usage ?? null };
+  }
+}
+
+function complete(prepared: Prepared, reply: Parameters<typeof assess>[1], elapsedMs: number, mode: "live" | "replay"): CompletedItem {
+  const answer = reply.answers.decision;
+  if (!answer) throw new OrlyError("Validated reply has no requested answer.");
+  return {
+    status: RESULT_STATUS.complete, id: prepared.item.id, question: prepared.item.question,
+    mode, model: MODEL, identity: prepared.identity, sources: prepared.references,
+    assessment: assess(prepared, reply), answer, requests: mode === JUDGMENT_MODE.live ? 1 : 0, elapsedMs, usage: reply.usage,
+  };
+}
+
+function incomplete(item: Item, error: unknown): IncompleteItem {
+  return { status: RESULT_STATUS.incomplete, id: item.id, question: item.question, reason: safeReason(error), requests: 0, elapsedMs: 0, usage: null };
+}
+
+function safeReason(error: unknown): string {
+  return error instanceof OrlyError ? error.message : "Judgment unavailable; raw source and provider details were not logged.";
+}
+
+function failedPreflight(manifest: Manifest, prepared: Preparation[]): Report {
+  return {
+    stage: manifest.stage, advisory: true, requests: 0,
+    results: prepared.map((item) => item.status === RESULT_STATUS.incomplete ? item : incomplete(item.value.item, new OrlyError(PREFLIGHT_REASON))),
+  };
+}

package/src/judgments/scan.test.ts

@@ -0,0 +1,19 @@
+import { expect, test } from "bun:test";
+
+import { scanUpload } from "./scan";
+
+test("real secret scanning accepts ordinary evidence", async () => {
+  await scanUpload(JSON.stringify({ state: "expect(actual).toBe(expected)" }));
+});
+
+test("real secret scanning refuses a generated credential without logging it", async () => {
+  const fake = ["gh", "p_", Bun.randomUUIDv7().replaceAll("-", ""), "abcd"].join("");
+  const result = await scanUpload(JSON.stringify({ token: fake })).catch((error: unknown) => error);
+  expect(String(result)).toContain("found a credential");
+  expect(String(result)).not.toContain(fake);
+});
+
+test("the secret scanner deadline terminates its owned process", async () => {
+  const result = await scanUpload("ordinary evidence", 0).catch((error: unknown) => error);
+  expect(String(result)).toContain("timed out");
+});

package/src/judgments/scan.ts

@@ -0,0 +1,40 @@
+import { mkdtempSync, rmdirSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { OrlyError } from "../model";
+import { IGNORE_OUTPUT, PIPE_OUTPUT, SCAN_TIMEOUT_MS } from "./constants";
+
+const SCANNER = "gitleaks";
+const SCAN_ARGUMENTS = ["stdin", "--redact", "--no-banner"];
+const KILL_SIGNAL = "SIGKILL";
+
+export async function scanUpload(request: string, timeoutMs = SCAN_TIMEOUT_MS): Promise<void> {
+  const directory = mkdtempSync(join(tmpdir(), "orly-judgment-scan-"));
+  let child: ReturnType<typeof Bun.spawn> | undefined;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let timedOut = false;
+  try {
+    // Empty scratch directory and a minimal environment prevent project scanner settings from hiding a secret.
+    child = Bun.spawn([SCANNER, ...SCAN_ARGUMENTS], {
+      cwd: directory, env: { PATH: Bun.env.PATH ?? "", LANG: "C" },
+      stdin: PIPE_OUTPUT, stdout: IGNORE_OUTPUT, stderr: IGNORE_OUTPUT,
+    });
+    const running = child;
+    timer = setTimeout(() => { timedOut = true; running.kill(KILL_SIGNAL); }, timeoutMs);
+    if (typeof child.stdin !== "object" || !("write" in child.stdin)) throw new OrlyError("Secret scanner input is unavailable.");
+    child.stdin.write(request);
+    child.stdin.end();
+    const code = await child.exited;
+    if (timedOut) throw new OrlyError("Secret scanning timed out; nothing was uploaded.");
+    if (code === 1) throw new OrlyError("Secret scanning found a credential in selected evidence; nothing was uploaded.");
+    if (code !== 0) throw new OrlyError("Secret scanner failed; nothing was uploaded.");
+  } catch (error) {
+    if (error instanceof OrlyError) throw error;
+    throw new OrlyError("Secret scanner is unavailable; install gitleaks before explicit refresh.");
+  } finally {
+    clearTimeout(timer);
+    if (child && child.exitCode === null) { child.kill(KILL_SIGNAL); await child.exited; }
+    rmdirSync(directory);
+  }
+}

package/src/judgments/syntax.test.ts

@@ -0,0 +1,115 @@
+import { expect, test } from "bun:test";
+
+import { selectSyntax } from "./syntax";
+import { SOURCE, SOURCE_FILE, TEST_FILE, TEST_NAME, TEST_SOURCE, TestProject } from "./test_support";
+
+test("selects a whole function with nested blocks and template literals", async () => {
+  const text = `const unrelated = 1;\nexport function target(value: string) {\n if (value) { return \`x\${value}\`; }\n return "";\n}\nconst after = 2;`;
+  const span = await selectSyntax(SOURCE_FILE, text, { kind: "function", name: "target" });
+  expect(text.slice(span.start, span.end)).toBe('export function target(value: string) {\n if (value) { return `x${value}`; }\n return "";\n}');
+});
+
+test("selects complete named test calls", async () => {
+  const span = await selectSyntax(TEST_FILE, TEST_SOURCE, { kind: "test", name: TEST_NAME });
+  const selected = TEST_SOURCE.slice(span.start, span.end);
+  expect(selected.startsWith(`test("${TEST_NAME}"`)).toBe(true);
+  expect(selected.endsWith("})")).toBe(true);
+  expect(selected).toContain("toBeTruthy()");
+});
+
+test("refuses duplicate symbols and skipped ancestor tests", async () => {
+  const duplicate = await selectSyntax(SOURCE_FILE, `${SOURCE}\n${SOURCE}`, { kind: "function", name: "label" }).catch((error: unknown) => error);
+  expect(duplicate).toBeInstanceOf(Error);
+  expect(String(duplicate)).toContain("Evidence selector must match one complete, executable unit.");
+  const skipped = `describe.skip("group", () => { ${TEST_SOURCE.slice(TEST_SOURCE.indexOf("test("))} });`;
+  const blocked = await selectSyntax(TEST_FILE, skipped, { kind: "test", name: TEST_NAME }).catch((error: unknown) => error);
+  expect(String(blocked)).toContain("Evidence selector must match one complete, executable unit.");
+});
+
+test.each(['test["skip"]', 'test["todo"]'])("refuses a statically computed skipped test: %s", async (callee) => {
+  const text = TEST_SOURCE.replace("test(", `${callee}(`);
+  await expect(selectSyntax(TEST_FILE, text, { kind: "test", name: TEST_NAME })).rejects.toThrow("Evidence selector must match one complete, executable unit.");
+});
+
+test.each(['describe["skip"]', 'describe["todo"]'])("refuses a statically computed skipped ancestor: %s", async (callee) => {
+  const text = `${callee}("group", () => { ${TEST_SOURCE.slice(TEST_SOURCE.indexOf("test("))} });`;
+  await expect(selectSyntax(TEST_FILE, text, { kind: "test", name: TEST_NAME })).rejects.toThrow("Evidence selector must match one complete, executable unit.");
+});
+
+test("refuses syntax errors, absent symbols and unsupported file types", async () => {
+  const malformed = await selectSyntax(SOURCE_FILE, "function broken( {", { kind: "function", name: "broken" }).catch((error: unknown) => error);
+  expect(String(malformed)).toContain("Selected TypeScript has syntax errors.");
+  const absent = await selectSyntax(SOURCE_FILE, SOURCE, { kind: "function", name: "absent" }).catch((error: unknown) => error);
+  expect(String(absent)).toContain("Evidence selector must match one complete, executable unit.");
+  await expect(selectSyntax("source.rs", SOURCE, { kind: "function", name: "label" })).rejects.toThrow("Function and test selectors require TypeScript or JavaScript.");
+});
+
+test("selects a section including its child headings and ignores fenced headings", async () => {
+  const text = "# Root\n## Selected\nProof\n### Child\nMore proof\n```md\n## Fake\n```\n## Next\nOther\n";
+  const span = await selectSyntax("spec.md", text, { kind: "section", heading: "Selected" });
+  expect(text.slice(span.start, span.end)).toBe("## Selected\nProof\n### Child\nMore proof\n```md\n## Fake\n```\n");
+});
+
+test.each(["```", "~~~"])("a fenced example can repeat a real heading with fence %s", async (fence) => {
+  const selected = `## Selected\nProof\n${fence}md\n## Selected\n${fence}\n`;
+  const text = `${selected}## Next\nOther\n`;
+  const span = await selectSyntax("spec.md", text, { kind: "section", heading: "Selected" });
+  expect(text.slice(span.start, span.end)).toBe(selected);
+});
+
+test("shorter and unterminated fences cannot introduce section headings", async () => {
+  const text = "## Selected\nProof\n````md\n## Selected\n```\n## Selected\n";
+  const span = await selectSyntax("spec.md", text, { kind: "section", heading: "Selected" });
+  expect(text.slice(span.start, span.end)).toBe(text);
+});
+
+test("a list-item fence can repeat a real section heading", async () => {
+  const selected = "## Selected\nText\n- ```md\n  ## Selected\n  ```\n\n";
+  const text = `${selected}## Next\nOther\n`;
+  const span = await selectSyntax("spec.md", text, { kind: "section", heading: "Selected" });
+  expect(text.slice(span.start, span.end)).toBe(selected);
+});
+
+test("literal trailing hashes remain part of a Markdown heading", async () => {
+  const text = "## C#\nProof\n## Next\nOther\n";
+  const span = await selectSyntax("spec.md", text, { kind: "section", heading: "C#" });
+  expect(text.slice(span.start, span.end)).toBe("## C#\nProof\n");
+  await expect(selectSyntax("spec.md", text, { kind: "section", heading: "C" })).rejects.toThrow("Markdown section must match one complete heading.");
+});
+
+test("whitespace-separated closing hashes do not become part of the heading", async () => {
+  const text = "## C# ##\nProof\n## Next\nOther\n";
+  const span = await selectSyntax("spec.md", text, { kind: "section", heading: "C#" });
+  expect(text.slice(span.start, span.end)).toBe("## C# ##\nProof\n");
+});
+
+test("refuses duplicate or missing Markdown headings", async () => {
+  await expect(selectSyntax("spec.md", "## Repeat\none\n## Repeat\ntwo", { kind: "section", heading: "Repeat" })).rejects.toThrow("Markdown section requires an unambiguous plain heading.");
+  await expect(selectSyntax("spec.md", "## Other\ntext", { kind: "section", heading: "Missing" })).rejects.toThrow("Markdown section must match one complete heading.");
+});
+
+test("selects a typed function containing JSX without running its source", async () => {
+  const text = 'export const label = (value: string) => <span>{value}</span>;\nthrow new Error("source must never run");';
+  const span = await selectSyntax("label.tsx", text, { kind: "function", name: "label" });
+  expect(text.slice(span.start, span.end)).toBe('label = (value: string) => <span>{value}</span>');
+});
+
+test("parser timeout ends its worker and a later parse remains usable", async () => {
+  using project = new TestProject();
+  const waiting = "waiting-worker.ts";
+  project.write(waiting, "self.onmessage = () => {}; ");
+  const error = await selectSyntax(SOURCE_FILE, SOURCE, { kind: "function", name: "label" }, {
+    timeoutMs: 10, workerUrl: new URL(`file://${project.root}/${waiting}`),
+  }).catch((reason: unknown) => reason);
+  expect(String(error)).toContain("TypeScript evidence parsing timed out.");
+  const span = await selectSyntax(SOURCE_FILE, SOURCE, { kind: "function", name: "label" });
+  expect(SOURCE.slice(span.start, span.end)).toBe(SOURCE);
+});
+
+test("parser startup failure refuses evidence without exposing raw worker details", async () => {
+  const error = await selectSyntax(SOURCE_FILE, SOURCE, { kind: "function", name: "label" }, {
+    workerUrl: new URL("./missing-parser-worker.ts", import.meta.url),
+  }).catch((reason: unknown) => reason);
+  expect(String(error)).toContain("TypeScript evidence could not be parsed.");
+  expect(String(error)).not.toContain("missing-parser-worker");
+});

package/src/judgments/syntax.ts

@@ -0,0 +1,71 @@
+import type { Selector } from "./types";
+import type { ParserReply } from "./syntax_worker";
+
+import { extname } from "node:path";
+
+import { OrlyError } from "../model";
+import { PARSE_TIMEOUT_MS } from "./constants";
+
+type Span = { start: number; end: number };
+type LocatedHeading = { title: string; level: number; start: number };
+const TYPESCRIPT_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx", ".mts", ".cts", ".mjs", ".cjs"]);
+const SECTION_HEADING_ERROR = "Markdown section requires an unambiguous plain heading.";
+type ParserOptions = { timeoutMs?: number; workerUrl?: URL };
+
+export async function selectSyntax(path: string, text: string, selector: Selector, options: ParserOptions = {}): Promise<Span> {
+  if (selector.kind === "file") return { start: 0, end: text.length };
+  if (selector.kind === "section") return selectSection(text, selector.heading);
+  if (!TYPESCRIPT_EXTENSIONS.has(extname(path))) throw new OrlyError("Function and test selectors require TypeScript or JavaScript.");
+  const worker = new Worker(options.workerUrl ?? new URL("./syntax_worker.ts", import.meta.url));
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    return await new Promise<Span>((resolve, reject) => {
+      timer = setTimeout(() => reject(new OrlyError("TypeScript evidence parsing timed out.")), options.timeoutMs ?? PARSE_TIMEOUT_MS);
+      worker.onmessage = (event: MessageEvent<ParserReply>) => {
+        const reply = event.data;
+        if (!reply.ok) reject(new OrlyError(reply.reason));
+        else resolve({ start: reply.start, end: reply.end });
+      };
+      worker.onerror = (event) => { event.preventDefault(); reject(new OrlyError("TypeScript evidence could not be parsed.")); };
+      worker.postMessage({ path, text, selector });
+    });
+  } finally {
+    clearTimeout(timer);
+    worker.terminate();
+  }
+}
+
+function selectSection(text: string, heading: string): Span {
+  const headings = sectionHeadings(text);
+  const indices = headings.flatMap((entry, index) => entry.title === heading ? [index] : []);
+  if (indices.length !== 1) throw new OrlyError("Markdown section must match one complete heading.");
+  const index = indices[0];
+  const selected = index === undefined ? undefined : headings[index];
+  if (!selected || index === undefined) throw new OrlyError("Markdown section is unavailable.");
+  const next = headings.slice(index + 1).find((entry) => entry.level <= selected.level);
+  return { start: selected.start, end: next?.start ?? text.length };
+}
+
+function sectionHeadings(text: string): LocatedHeading[] {
+  const headings: LocatedHeading[] = [];
+  const positions = new Map<string, LocatedHeading>();
+  const seen = new Set<string>();
+  const nonce = Bun.randomUUIDv7();
+  // Mark candidate headings in a parsing copy; Bun decides which belong to real sections.
+  const marked = text.replace(/^( {0,3})(#{1,6})[\t ]+([^\r\n]+?)[\t ]*$/gm,
+    (_line: string, indentation: string, hashes: string, rawTitle: string, offset: number) => {
+      const title = rawTitle.replace(/[\t ]+#+$/, "");
+      const key = `orly-${nonce}-${offset} ${title}`;
+      positions.set(key, { title, level: hashes.length, start: offset });
+      return `${indentation}${hashes} ${key}`;
+    });
+  Bun.markdown.render(marked, { heading: (title, { level }) => {
+    const found = positions.get(title);
+    const identity = `${level}:${found?.title}`;
+    if (!found || found.level !== level || seen.has(identity)) throw new OrlyError(SECTION_HEADING_ERROR);
+    seen.add(identity);
+    headings.push(found);
+    return title;
+  } });
+  return headings;
+}

package/src/judgments/syntax_worker.ts

@@ -0,0 +1,70 @@
+import type { Selector } from "./types";
+
+import { parse, type ParserPlugin } from "@babel/parser";
+import { isArrowFunctionExpression, isCallExpression, isExportDefaultDeclaration, isExportNamedDeclaration, isFunctionDeclaration, isFunctionExpression, isIdentifier, isMemberExpression, isStringLiteral, isVariableDeclarator, traverse, type Node, type TraversalAncestors } from "@babel/types";
+import { extname } from "node:path";
+
+import { SELECTOR_KIND } from "./constants";
+
+export type ParserReply = { ok: true; start: number; end: number } | { ok: false; reason: string };
+type Request = { path: string; text: string; selector: Extract<Selector, { kind: "function" | "test" }> };
+type Span = { start: number; end: number };
+const TEST_FUNCTIONS = new Set(["it", SELECTOR_KIND.test]);
+const SKIPPED_TESTS = new Set(["skip", "todo"]);
+const TYPESCRIPT_JSX_EXTENSION = ".tsx";
+const JSX_EXTENSIONS = new Set([TYPESCRIPT_JSX_EXTENSION, ".jsx"]);
+const TYPESCRIPT_EXTENSIONS = new Set([".ts", TYPESCRIPT_JSX_EXTENSION, ".mts", ".cts"]);
+const SELECTOR_ERROR = "Evidence selector must match one complete, executable unit.";
+const SYNTAX_ERROR = "Selected TypeScript has syntax errors.";
+declare const self: Worker;
+
+self.onmessage = (event: MessageEvent<Request>) => {
+  try { self.postMessage({ ok: true, ...selectProgramSpan(event.data) } satisfies ParserReply); }
+  catch (error) { self.postMessage({ ok: false, reason: error instanceof Error && error.message === SELECTOR_ERROR ? SELECTOR_ERROR : SYNTAX_ERROR } satisfies ParserReply); }
+};
+
+function selectProgramSpan({ path, text, selector }: Request): Span {
+  const extension = extname(path);
+  const plugins: ParserPlugin[] = [];
+  if (TYPESCRIPT_EXTENSIONS.has(extension)) plugins.push("typescript");
+  if (JSX_EXTENSIONS.has(extension)) plugins.push("jsx");
+  const tree = parse(text, { sourceType: "unambiguous", plugins, errorRecovery: true, attachComment: false });
+  const matches: Span[] = [];
+  traverse(tree, (node, ancestors) => {
+    if (ancestors.some(({ node: ancestor }) => isSkipped(ancestor)) || isSkipped(node) || !matchesSelector(node, selector)) return;
+    const selected = declarationSpan(node, ancestors);
+    if (selected.start !== null && selected.start !== undefined && selected.end !== null && selected.end !== undefined) {
+      matches.push({ start: selected.start, end: selected.end });
+    }
+  });
+  const match = matches[0];
+  if (matches.length !== 1 || !match) throw new Error(SELECTOR_ERROR);
+  if (tree.errors?.length) throw new Error(SYNTAX_ERROR);
+  return match;
+}
+
+function declarationSpan(node: Node, ancestors: TraversalAncestors): Node {
+  const parent = ancestors.at(-1)?.node;
+  return parent && (isExportNamedDeclaration(parent) || isExportDefaultDeclaration(parent)) ? parent : node;
+}
+
+function isSkipped(node: Node): boolean {
+  if (!isCallExpression(node) || !isMemberExpression(node.callee)) return false;
+  const { computed, property } = node.callee;
+  const name = computed && isStringLiteral(property) ? property.value : !computed && isIdentifier(property) ? property.name : undefined;
+  return name !== undefined && SKIPPED_TESTS.has(name);
+}
+
+function matchesSelector(node: Node, selector: Request["selector"]): boolean {
+  if (selector.kind === SELECTOR_KIND.function) {
+    if (isFunctionDeclaration(node)) return node.id?.name === selector.name && node.body !== undefined;
+    return isVariableDeclarator(node) && isIdentifier(node.id) && node.id.name === selector.name
+      && (isArrowFunctionExpression(node.init) || isFunctionExpression(node.init));
+  }
+  if (!isCallExpression(node)) return false;
+  const callee = isMemberExpression(node.callee) ? node.callee.object : node.callee;
+  const label = node.arguments[0];
+  const callback = node.arguments[1];
+  return isIdentifier(callee) && TEST_FUNCTIONS.has(callee.name) && isStringLiteral(label)
+    && label.value === selector.name && (isArrowFunctionExpression(callback) || isFunctionExpression(callback));
+}

package/src/judgments/test_support.ts

@@ -0,0 +1,60 @@
+import type { Item, Manifest, Prepared } from "./types";
+import type { Reply } from "./wire";
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { CHOICE, MODEL, REPLY_KEY, ROLE, SELECTOR_KIND } from "./constants";
+import { prepareItem } from "./evidence";
+
+export const SOURCE_FILE = "label.ts";
+export const TEST_FILE = "label.test.ts";
+export const TEST_NAME = "label respects locale";
+export const REQUIREMENT = "label returns the requested locale's year, month and day from Intl.DateTimeFormat, rather than merely a nonempty string.";
+export const SOURCE = `export function label(value: string, locale: string): string {
+  return new Intl.DateTimeFormat(locale, { year: "numeric", month: "short", day: "2-digit" }).format(new Date(value));
+}`;
+export const TEST_SOURCE = `import { test, expect } from "bun:test";
+import { label } from "./label";
+test("${TEST_NAME}", () => {
+  expect(label("2026-10-04T00:00:00Z", "en-GB")).toBeTruthy();
+});`;
+
+export class TestProject {
+  readonly #root = mkdtempSync(join(tmpdir(), "orly-judgments-test-"));
+
+  constructor() {
+    this.write(SOURCE_FILE, SOURCE);
+    this.write(TEST_FILE, TEST_SOURCE);
+  }
+
+  get root(): string { return this.#root; }
+
+  write(path: string, text: string): void { writeFileSync(join(this.#root, path), text); }
+
+  item(): Item {
+    return {
+      id: "locale", question: "verify.assertion", requirement: REQUIREMENT,
+      evidence: [
+        { role: "implementation", path: SOURCE_FILE, selector: { kind: "function", name: "label" } },
+        { role: ROLE.test, path: TEST_FILE, selector: { kind: SELECTOR_KIND.test, name: TEST_NAME } },
+      ],
+    };
+  }
+
+  manifest(): Manifest { return { stage: "verify", items: [this.item()] }; }
+
+  prepare(): Promise<Prepared> { return prepareItem(this.#root, this.item()); }
+
+  [Symbol.dispose](): void { rmSync(this.#root, { recursive: true, force: true }); }
+}
+
+export function choiceReply(choice: string = CHOICE.weak): Reply {
+  const probabilities = Object.fromEntries(Object.values(CHOICE).map((option) => [option, option === choice ? 1 : 0]));
+  return { model: MODEL, answers: { [REPLY_KEY]: { type: "choice", choice, probabilities, confidence: 1 } }, usage: { input_tokens: 100, output_tokens: 20 } };
+}
+
+export function noulReply(probability: number): Reply {
+  return { model: MODEL, answers: { [REPLY_KEY]: { type: "noul", noul: probability } }, usage: { input_tokens: 100, output_tokens: 20 } };
+}