duediligence 0.3.1 → 0.5.0

package/bin/api.mjs

@@ -17,6 +17,9 @@
 //   promptPath(name)                        the absolute path of prompts/<name>.md; anything outside prompts/ refused
 //   renderPrompt(name, values)              prompts/<name>.md with each {{key}} filled from values, as the CLI fills repo/ briefs
 //
+// prompts/sandbox/ holds briefs only a platform renders: hidden tests for a
+// generated project whose cut modules are recording mocks (sandbox-factory).
+//
 // Suite results are what a runner prints (bin/runners/shared.mjs); the
 // rules' inputs and outputs are documented in bin/rules.mjs, the hash's in
 // bin/hash.mjs.

package/bin/ci.mjs

@@ -6,6 +6,8 @@
 //   node bin/ci.mjs --seal <file|https-url> [--repo <checkout>] [--ref <rev>]
 //                   [--name <check name>] [--out <payload.json>] [--record <file>]
 //                   [--no-post] [--exit-zero]
+//   node bin/ci.mjs --platform [<url>] --version <id> --workspace <id> [--base <rev>]
+//                   judged on the hosted platform instead (hosted.mjs)
 //
 // 1. A URL seal is fetched with DUEDILIGENCE_SEAL_AUTH as the
 //    Authorization header, if set, into a temporary file.
@@ -37,6 +39,17 @@ import { fileURLToPath } from "node:url";
 
 const BIN = dirname(fileURLToPath(import.meta.url));
 const argv = process.argv.slice(2);
+
+// --platform: judged on the hosted platform instead of from a seal
+// (hosted.mjs `ciPlatform`). A bare --platform means the default platform.
+if (argv.includes("--platform") || argv.some((arg) => arg.startsWith("--platform="))) {
+  const { run } = await import("./hosted.mjs");
+  const { version } = JSON.parse(readFileSync(join(BIN, "..", "package.json"), "utf8"));
+  const given = argv.map((arg, index) =>
+    arg === "--platform" && (argv[index + 1] === undefined || argv[index + 1].startsWith("--")) ? "--platform=" : arg,
+  );
+  process.exit(await run("ci", given, { client: "duediligence", version }));
+}
 const options = {};
 for (let i = 0; i < argv.length; i += 1) {
   if (!argv[i].startsWith("--")) {

package/bin/cli.mjs

@@ -27,9 +27,10 @@ const COMMANDS = {
   retry: "one retry of a blind run, told its failing scenarios",
   "import-archive": "a calibrated spec from the experiment archive, marked not held out",
   seal: "one spec's suites in one file, for a judge that has no specs/ (CI)",
-  ci: "judge a commit from a seal and report it as a check (GitHub Actions or anywhere)",
+  ci: "judge a commit from a seal and report it as a check; --platform judges it on the hosted platform (OIDC)",
   referee: "a blind referee of rejected runs' failing tests, on any backend (--backend)",
   "red-team-audit": "red-team passes (run --brief red-team) audited: literals, a second suite, a session",
+  push: "a confirmed spec as a bounty in the signed-in platform workspace",
   login: "sign in to the Lunox platform (device code in the browser); --org, --platform, --open",
   whoami: "the account, workspace and token expiry signed in with",
   logout: "delete the saved platform token",

package/bin/hosted.mjs

@@ -0,0 +1,248 @@
+// The commands that hand work to the hosted platform (sandbox-factory)
+// instead of running it here, over platform-client.mjs:
+//
+//   push <slug>       a spec confirmed here becomes a bounty in the signed-in
+//                     workspace: POST /api/v1/orgs/:org/bounties with the
+//                     ticket's title and the confirmed spec.md as its text.
+//                     Recorded in specs/<slug>/platform.json, so pushing
+//                     again names the bounty already made.
+//   ci --platform     a pull request judged by the platform against a
+//                     published version's hidden suite, which never reaches
+//                     the runner: the PR's changed files go to
+//                     POST /api/judge with the run's GitHub OIDC token (no
+//                     secret on either side; the platform checks the token
+//                     names the repository the version was sliced from),
+//                     then GET /api/judge/:id?workspace= until a verdict. The job's own
+//                     pass or fail is the check.
+//
+// Each takes its argv and the outside world as `deps` (account.mjs's, plus
+// `git` and `readFile`), so test/hosted.mjs drives them against a fake
+// platform.
+
+import { execFileSync } from "node:child_process";
+import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { parseArgs } from "node:util";
+
+import { defaultDeps as accountDeps } from "./account.mjs";
+import { confirmation } from "./confirm.mjs";
+import { specDir } from "./lib.mjs";
+import { credentialFor, credentialsPath, platformUrl, PlatformError, request } from "./platform-client.mjs";
+
+/** The platform's limits on one submission (sandbox-factory SUBMISSION_LIMITS). */
+const MAX_FILES = 200;
+const MAX_FILE_BYTES = 512 * 1024;
+const MAX_BYTES = 4_000_000;
+/** How often a verdict is asked for, and for how long. */
+const POLL_MS = 10_000;
+const POLL_LIMIT_MS = 45 * 60_000;
+const TITLE_MAX = 255;
+const DESCRIPTION_MAX = 20_000;
+
+export function defaultDeps() {
+  return {
+    ...accountDeps(),
+    git: (args, cwd) => execFileSync("git", args, { cwd, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }),
+    readFile: (path) => readFileSync(path),
+  };
+}
+
+function options(argv, spec, usage, positionals = false) {
+  try {
+    return parseArgs({ args: argv, options: { ...spec, help: { type: "boolean", short: "h" } }, allowPositionals: positionals });
+  } catch (error) {
+    throw new PlatformError(`${error.message}\n${usage}`);
+  }
+}
+
+/** The first `# ` heading of a markdown text, else its first line. */
+export function titleOf(text) {
+  const heading = /^#\s+(.+)$/m.exec(text)?.[1];
+  const line = heading ?? text.split("\n").find((each) => each.trim() !== "") ?? "";
+  return line.trim().slice(0, TITLE_MAX);
+}
+
+/** `push <slug>`: a locally confirmed spec as a bounty in the signed-in workspace. */
+export async function push(argv, { client, version }, deps) {
+  const usage = `usage: ${client} push <slug> [--platform <url>] [--again]`;
+  const { values, positionals } = options(argv, { platform: { type: "string" }, again: { type: "boolean" } }, usage, true);
+  if (values.help || positionals.length !== 1) {
+    deps.out(
+      `${usage}\n\nPosts specs/<slug>/spec.md, as confirmed (${client} confirm <slug> --by <name>), as a bounty in the workspace\n` +
+        `${client} login signed in to. The bounty's title is ticket.md's (else spec.md's) heading. Pushing again names\n` +
+        `the bounty already made (specs/<slug>/platform.json); --again posts a new one, as after editing the spec.\n`,
+    );
+    return values.help ? 0 : 1;
+  }
+  const [slug] = positionals;
+  const dir = specDir(slug);
+  const confirmed = confirmation(slug);
+  if (!confirmed.ok) throw new PlatformError(`specs/${slug}: ${confirmed.reason}. Confirm it first: ${client} confirm ${slug} --by <name>.`);
+  const base = platformUrl(values.platform, deps.env);
+  const saved = credentialFor(credentialsPath(deps.env), base, client);
+  if (saved === undefined) throw new PlatformError(`Not signed in to ${base}. Run \`${client} login\`.`);
+  const recordPath = join(dir, "platform.json");
+  const recorded = existsSync(recordPath) ? JSON.parse(readFileSync(recordPath, "utf8")) : null;
+  if (recorded !== null && !values.again && recorded.base === base && recorded.organizationId === saved.organizationId) {
+    const stale = recorded.specSha256 !== confirmed.hash;
+    deps.out(
+      `specs/${slug} is bounty ${recorded.bountyId} in @${saved.organizationHandle} (pushed ${recorded.pushedAt.slice(0, 10)}).\n` +
+        (stale ? "spec.md was confirmed again since; --again posts it as a new bounty.\n" : ""),
+    );
+    return 0;
+  }
+  const spec = readFileSync(join(dir, "spec.md"), "utf8");
+  const ticketPath = join(dir, "ticket.md");
+  const title = titleOf(existsSync(ticketPath) ? readFileSync(ticketPath, "utf8") : spec);
+  if (title === "") throw new PlatformError(`specs/${slug}: no title in ticket.md or spec.md.`);
+  if (spec.length > DESCRIPTION_MAX)
+    throw new PlatformError(`specs/${slug}/spec.md is ${spec.length} characters; a bounty takes ${DESCRIPTION_MAX}.`);
+  const made = await request({
+    base,
+    token: saved.token,
+    client,
+    version,
+    fetchImpl: deps.fetchImpl,
+    method: "POST",
+    path: `/api/v1/orgs/${encodeURIComponent(saved.organizationId)}/bounties`,
+    body: { title, description: spec },
+  });
+  const bountyId = made?.bounty?.id;
+  if (typeof bountyId !== "string") throw new PlatformError(`${base} answered without a bounty.`);
+  writeFileSync(
+    recordPath,
+    `${JSON.stringify(
+      {
+        base,
+        organizationId: saved.organizationId,
+        bountyId,
+        specSha256: confirmed.hash,
+        confirmedBy: confirmed.record.by,
+        pushedAt: new Date(deps.now()).toISOString(),
+      },
+      null,
+      2,
+    )}\n`,
+  );
+  deps.out(
+    `Posted specs/${slug} as bounty ${bountyId} in @${saved.organizationHandle} on ${base}.\n` +
+      `Size it there, or with the MCP tool size_bounty; its decisions were confirmed here by ${confirmed.record.by}.\n`,
+  );
+  return 0;
+}
+
+/** A GitHub Actions OIDC token for `audience`; the job needs `permissions: id-token: write`. */
+export async function oidcToken(audience, deps) {
+  const url = deps.env.ACTIONS_ID_TOKEN_REQUEST_URL;
+  const bearer = deps.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN;
+  if (!url || !bearer)
+    throw new PlatformError(
+      "No GitHub OIDC token: run in GitHub Actions with `permissions: id-token: write` (ACTIONS_ID_TOKEN_REQUEST_URL is unset).",
+    );
+  const response = await deps.fetchImpl(`${url}${url.includes("?") ? "&" : "?"}audience=${encodeURIComponent(audience)}`, {
+    headers: { Authorization: `Bearer ${bearer}`, Accept: "application/json" },
+    signal: AbortSignal.timeout(30_000),
+  });
+  const body = await response.json().catch(() => ({}));
+  if (!response.ok || typeof body.value !== "string") throw new PlatformError(`GitHub gave no OIDC token (HTTP ${response.status}).`);
+  return body.value;
+}
+
+/** The PR's added and modified files between `base` and HEAD, whole, as the checkout has them. */
+export function changedFiles(repo, base, deps) {
+  const names = deps
+    .git(["diff", "--name-only", "--diff-filter=AM", "-z", `${base}...HEAD`], repo)
+    .split("\0")
+    .filter((name) => name !== "");
+  if (names.length > MAX_FILES) throw new PlatformError(`The change touches ${names.length} files; the platform judges up to ${MAX_FILES}.`);
+  const files = [];
+  const skipped = [];
+  for (const path of names) {
+    const bytes = deps.readFile(resolve(repo, path));
+    if (bytes.length > MAX_FILE_BYTES || bytes.includes(0)) skipped.push(path);
+    else files.push({ path, text: bytes.toString("utf8") });
+  }
+  const total = files.reduce((sum, file) => sum + Buffer.byteLength(file.text), 0);
+  if (total > MAX_BYTES) throw new PlatformError(`The change is ${total} bytes; the platform judges up to ${MAX_BYTES}.`);
+  return { files, skipped };
+}
+
+const EXIT = { passed: 0, overfit: 2 };
+
+/** `ci --platform`: the change judged on the platform; answers the exit code. */
+export async function ciPlatform(argv, { client, version }, deps) {
+  const usage = `usage: ${client} ci --platform [<url>] --version <sandbox version id> --workspace <workspace id> [--repo <checkout>] [--base <rev>] [--exit-zero]`;
+  const { values } = options(
+    argv,
+    {
+      platform: { type: "string" },
+      version: { type: "string" },
+      workspace: { type: "string" },
+      repo: { type: "string" },
+      base: { type: "string" },
+      "exit-zero": { type: "boolean" },
+    },
+    usage,
+  );
+  if (values.help || !values.version || !values.workspace) {
+    deps.out(
+      `${usage}\n\nJudges this pull request on the platform against a published sandbox version's hidden suite, which never\n` +
+        `reaches the runner. Sends the files the PR added or changed since --base (default origin/$GITHUB_BASE_REF), with\n` +
+        `the run's GitHub OIDC token (the job needs \`permissions: id-token: write\`); the platform accepts it only from the\n` +
+        `repository the version was sliced from. Prints the verdict, counts and failing scenario keys.\n` +
+        `Exit: 0 passed, 2 overfit, 1 otherwise (0 with --exit-zero).\n`,
+    );
+    return values.help ? 0 : 1;
+  }
+  const base = platformUrl(values.platform === "" ? undefined : values.platform, deps.env);
+  const repo = resolve(values.repo ?? ".");
+  const from = values.base ?? (deps.env.GITHUB_BASE_REF ? `origin/${deps.env.GITHUB_BASE_REF}` : "HEAD~1");
+  const { files, skipped } = changedFiles(repo, from, deps);
+  for (const path of skipped) deps.err(`skipped ${path}: binary or over ${MAX_FILE_BYTES / 1024} KiB\n`);
+  if (files.length === 0) {
+    deps.out(`No file changed since ${from}; nothing to judge.\n`);
+    return 0;
+  }
+  const call = async (options) =>
+    request({ base, token: await oidcToken(base, deps), client, version, fetchImpl: deps.fetchImpl, ...options });
+  let { submission } = await call({
+    method: "POST",
+    path: "/api/judge",
+    body: { workspace: values.workspace, versionId: values.version, files },
+  });
+  deps.out(`Submitted ${files.length} file${files.length === 1 ? "" : "s"} as ${submission.id}; waiting for the verdict...\n`);
+  const started = deps.now();
+  while (submission.status === "queued" || submission.status === "running") {
+    if (deps.now() - started > POLL_LIMIT_MS) throw new PlatformError(`No verdict for ${submission.id} after ${POLL_LIMIT_MS / 60_000} minutes.`);
+    await deps.sleep(POLL_MS);
+    ({ submission } = await call({
+      path: `/api/judge/${encodeURIComponent(submission.id)}?workspace=${encodeURIComponent(values.workspace)}`,
+    }));
+  }
+  const verdict = submission.status === "errored" ? "errored" : (submission.verdict ?? submission.status);
+  const counts =
+    submission.public && submission.hidden
+      ? `public ${submission.public.passed}/${submission.public.total}, hidden ${submission.hidden.passed}/${submission.hidden.total}`
+      : "no tests ran";
+  const lines = [
+    `duediligence (platform): ${verdict}, ${counts}`,
+    ...(submission.failedScenarios.length > 0 ? [`failing scenarios: ${submission.failedScenarios.join(", ")}`] : []),
+    ...submission.refused.map((refusal) => `refused ${refusal.path}: ${refusal.reason}`),
+  ];
+  deps.out(`${lines.join("\n")}\n`);
+  if (deps.env.GITHUB_STEP_SUMMARY)
+    appendFileSync(deps.env.GITHUB_STEP_SUMMARY, `### ${lines[0]}\n\n${lines.slice(1).map((line) => `- ${line}`).join("\n")}\n`);
+  if (values["exit-zero"]) return 0;
+  return EXIT[verdict] ?? 1;
+}
+
+/** Runs one, printing a PlatformError as one line on stderr. */
+export async function run(command, argv, meta, deps = defaultDeps()) {
+  try {
+    return await (command === "push" ? push : ciPlatform)(argv, meta, deps);
+  } catch (error) {
+    if (!(error instanceof PlatformError)) throw error;
+    deps.err(`${meta.client} ${command}: ${error.message}\n`);
+    return 1;
+  }
+}

package/bin/push.mjs

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+// `duediligence push <slug>`: see hosted.mjs.
+import { readFileSync } from "node:fs";
+import { run } from "./hosted.mjs";
+
+const { version } = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));
+process.exitCode = await run("push", process.argv.slice(2), { client: "duediligence", version });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "duediligence",
-  "version": "0.3.1",
+  "version": "0.5.0",
   "description": "Due diligence for AI coding agents: independent, hidden test suites written from the spec. The agent works against a public suite; a hidden suite it never sees judges the result.",
   "keywords": [
     "due-diligence",
@@ -58,7 +58,7 @@
     "skills/"
   ],
   "scripts": {
-    "test": "node scripts/skill.mjs check && node test/api.mjs && node test/platform.mjs && node test/smoke.mjs",
+    "test": "node scripts/skill.mjs check && node test/api.mjs && node test/platform.mjs && node test/hosted.mjs && node test/smoke.mjs",
     "version": "node scripts/skill.mjs sync && git add skills .claude-plugin",
     "skill:check": "node scripts/skill.mjs check",
     "skill:sync": "node scripts/skill.mjs sync",

package/prompts/sandbox/hidden-tests.md

@@ -0,0 +1,73 @@
+Write the hidden tests for the bounty: the acceptance tests a developer's
+submission is judged by, which the developer never sees. The project is a
+standalone copy of the code the bounty changes, generated so that it
+installs, builds, runs its walkthrough and passes its public tests. Modules
+it does not copy are cut: each is a typed stub whose runtime values are
+recording mocks. The public tests are what the developer iterates against;
+these tests exist to fail a submission that passes them without doing what
+the spec says.
+
+Write each test under `{{hiddenDir}}/` as a `{{testPattern}}` file, and mark
+it with the outcome it has on the project as it is now. "fail" is a test
+that only a finished bounty passes: it checks the behaviour the bounty asks
+for. "pass" is one that guards behaviour the code already has and a finished
+bounty must keep. At least one must be "fail". A submission is accepted when
+every hidden test passes, so test what the spec says through the code's own
+interface, such that any correct implementation passes and an incomplete one
+does not.
+
+Cover:
+
+- every scenario in the spec, and every scenario kind it uses (happy,
+  boundary, unhappy, and any others it tags);
+- for each public test, at least one case it does not exercise: other
+  values, larger sizes, other orders, other combinations, more items. Name
+  the public test it extends in a comment above it;
+- the case an implementation tuned to the public tests would get wrong. For
+  each public test, ask what shortcut passes the visible cases, and write
+  the case that shortcut fails.
+
+Assert only what the spec states or what follows necessarily from it. Where
+the spec, or a public test, leaves a name, message or structure open, do not
+test it either way.
+
+A "fail" test must fail by its assertions, not by breaking. Hidden tests are
+compiled with the project by `{{build}}`, so they must type-check against the
+code as it is now: a test that imports a function or passes a parameter the
+bounty has yet to add breaks the build, and the whole baseline with it. When
+the bounty adds something new, reach it at run time inside the test, for
+example `const mod: Record<string, unknown> = await import("{{importExample}}")`,
+assert that it exists, and call it through a type you declare in the test. A
+test that throws while its file loads, or imports something that is not
+there, is an error, never the failure it is expected to show. A "pass" is
+read from the runner's report, not only its exit code: at least one test
+must run to completion and none may be skipped, todo or cancelled, and a
+file that ends early or declares no test is an error. Do not name a test
+after a file.
+
+Write them so that a submission can only pass by doing the work:
+
+- Assert observable results first: what the code returns, throws or leaves
+  in the state it exposes. A test that a call fails checks that the error is
+  the one the spec describes, not just that something was thrown. Use
+  `recordedCalls()` only to confirm that a seam was reached, and with what,
+  never as the only evidence that the behaviour is there.
+- Call `resetMocks()` at the start of each test, for example in
+  `beforeEach`, and install the behaviour each test needs inside it, so no
+  test depends on what another left behind.
+- Include at least one case whose inputs differ from the fixtures' default
+  data and the walkthrough's: give the mocked calls other values with
+  `mockImplementation(path, fn)` inside that test, so an implementation that
+  hard-codes the public data fails. Never use the exact fixture values as a
+  test's only inputs.
+- Each hidden test runs twice and must give the same result both times:
+  nothing may depend on timing, randomness, the clock or the order the tests
+  run in.
+
+The toolchain is fixed: {{toolchain}} Import the project's code with
+relative paths from the test's folder; from `{{hiddenDir}}/` a file at
+`src/feature.ts` is `{{importExample}}`. Import
+`{ mockImplementation, recordedCalls, resetMocks }` from `{{mockModule}}` to
+give a mocked call behaviour or to check what the code asked of it. Packages
+compile as `any`. Nothing may reach the network, a database or any outside
+service.

package/skills/duediligence/SKILL.md

@@ -10,7 +10,7 @@ description: >-
   spec, or wants to stop trusting an agent's own green test run.
 license: MIT
 compatibility: >-
-  Node 20.19 or later. Runs the duediligence@0.3.1 CLI from npm, which starts
+  Node 20.19 or later. Runs the duediligence@0.5.0 CLI from npm, which starts
   Claude Code, Codex or Anthropic API sessions and installs its test toolchain
   under ~/.duediligence/ once (network).
 ---
@@ -20,10 +20,10 @@ compatibility: >-
 From one confirmed spec, generate a **public** suite the implementing agent
 works against and a **hidden** suite that judges the result.
 
-`duediligence <command>` below is the CLI, `duediligence@0.3.1`. When
+`duediligence <command>` below is the CLI, `duediligence@0.5.0`. When
 `duediligence --version` prints another version or nothing, ask the person
-before installing it: `npm i -g duediligence@0.3.1`, or put
-`npx duediligence@0.3.1` in place of `duediligence` in each command (in a clone
+before installing it: `npm i -g duediligence@0.5.0`, or put
+`npx duediligence@0.5.0` in place of `duediligence` in each command (in a clone
 of the repository, `node bin/cli.mjs`). Run it from any folder. `specs/` and
 `runs/` below are in the home, `~/.duediligence/` (`DUEDILIGENCE_HOME`);
 `prompts/` and `template*/` are in the CLI's package. `duediligence where`
@@ -86,14 +86,29 @@ _Repository_ below are for that target.
 When the `lunox` MCP tools are available and signed in (the Claude Code plugin
 bundles the server; `/mcp` signs in), they work on the person's workspace on
 the Lunox platform. Reads: `list_workspaces`, `list_bounties`, `get_bounty`,
-`get_proposal`, `get_run`, `get_usage`. Writes, only when the person asks for
+`get_proposal`, `get_run`, `get_usage`, `list_sandboxes`, `get_sandbox`. Writes, only when the person asks for
 that one: `post_bounty`, `import_github_issue`, `import_jira_issue` (a new
-bounty) and `size_bounty` (price it; this starts a paid model run). Decision
+bounty) and `size_bounty` (price it; this starts a paid model run).
+`get_decisions` reads what a priced spec decided where the ticket was silent
+(its `Decided:` items) and whether a person confirmed them; `confirm_link`
+gives the page where the person confirms them. Hand them that link: decision
 confirmations, approving a price and publishing happen in the platform's web
-app: point the person there and never act for them. Hidden tests are not
-prepared or judged through these tools yet; use local mode for that. Never ask
+app, and you never act for them. Once the decisions are confirmed and the
+person asks: `start_sandbox` and `prepare` (build a sandbox version; the
+platform writes hidden tests from the confirmed spec and calibrates them
+against a reference, as `prepare` does locally), then `get_handout` for the
+public files and the hidden suite's commitment hash. You never see the hidden
+tests in hosted mode: they stay on the platform. On a published version,
+`judge` submits an implementation's changed files and `get_verdict` reads the
+verdict (counts and failing scenario keys only); `retry` is a failed one's
+single retry. From a terminal, `duediligence push <slug>` posts a spec
+confirmed here as a bounty, and `duediligence ci --platform --version <id>
+--workspace <id>` (or `action.yml` with `version` and `workspace`, job
+permission `id-token: write`) judges a pull request on the platform. Never ask
 for a token in the chat: the server signs in through `/mcp`, the CLI through
-`duediligence login` in the person's own terminal. Local mode, the steps below,
+`duediligence login` in the person's own terminal. A tool error with code
+`insufficient_scope` means the sign-in predates a scope: ask the person to
+sign in again through `/mcp`. Local mode, the steps below,
 is unchanged and the default, and needs no sign-in.
 
 ## 0. Fit (repository)