@geonosis/evals 1.0.0

package/dist/index.d.ts

@@ -0,0 +1,375 @@
+/**
+ * Which way is better, per dimension.
+ *
+ * `up` for the rates — more tasks passing is better. `down` for the caps, which are DEBT and carry
+ * the ratchet's direction: more revision rounds, more CI-fix attempts, more Stop blocks is worse. A
+ * comparator with one direction for everything would pass a release that doubled the review rounds
+ * because the number went up, and nobody would notice until the third round of the first review.
+ */
+declare const BETTER: {
+    readonly 'caps.ciFixes': "down";
+    readonly 'caps.revisions': "down";
+    readonly 'caps.stopBlocks': "down";
+    readonly firstEditPass: "up";
+    readonly gatePassRate: "up";
+    readonly pressureCompliance: "up";
+    readonly proofsPerTick: "up";
+};
+type Dimension = keyof typeof BETTER;
+declare const DIMENSIONS: Dimension[];
+type Regression = {
+    after: number;
+    before: number;
+    dimension: Dimension;
+    tolerance: number;
+};
+type Comparison = {
+    ok: boolean;
+    regressions: Regression[];
+};
+type Scored = {
+    caps?: {
+        ciFixes?: number;
+        revisions?: number;
+        stopBlocks?: number;
+    };
+} & Record<string, unknown>;
+type CompareInput = {
+    after: Scored;
+    before: Scored;
+    tolerance?: Record<string, number>;
+};
+/**
+ * Two scores, and every dimension that got worse by more than the band it was given.
+ *
+ * The band is opt-in and per dimension, and it is zero for anything nobody named — the ratchet's
+ * rule, for the same reason: a tolerance in the defaults is a gate that whoever wrote the defaults
+ * turned down, on behalf of everyone who never read them. A tolerance for a dimension that does not
+ * exist is refused rather than ignored, because a typo that silently tolerates nothing looks
+ * exactly like a band that is working.
+ */
+declare const compareScores: ({ after, before, tolerance }: CompareInput) => Comparison;
+declare const formatComparison: (found: Comparison) => string;
+
+/**
+ * The paths only a runner may write — the same list `plugin/hooks/lib/policy.mjs` refuses the
+ * agent's tool calls against, and the list `@geonosis/rails` renders into a sandbox profile.
+ *
+ * 2ndm1nd's rail: the scored agent never writes the scoreboard.
+ */
+declare const RUNNER_OWNED: string[];
+/**
+ * The subset of those that must come out of a run byte-identical.
+ *
+ * Not the same list, and the difference is the whole point: `geonosis-verify` WRITES the gate report
+ * during the run and the Stop hook WRITES the block count — a task whose `mustNotTouch` held them
+ * would fail for the gate having run at all. What must not move is the baseline, because that is the
+ * number the run is scored against. The first version of this file conflated the two and every
+ * green task failed; the failing test is what said so.
+ */
+declare const IMMUTABLE_DURING_RUN: string[];
+/** What a task's run is judged to have produced. Every field optional: a task may expect nothing. */
+type EvalExpectation = {
+    files?: string[];
+    gateTier?: string;
+    ledgerTick?: boolean;
+    mustNotTouch?: string[];
+};
+type EvalTask = {
+    expect: EvalExpectation & {
+        mustNotTouch: string[];
+    };
+    id: string;
+    prompt: string;
+    repo: string;
+};
+/**
+ * A prompt from the "quick fix, skip the ceremony" family, and the skills that must still be loaded
+ * when it runs.
+ *
+ * `mustStillLoad` is a claim about the SETTINGS the run is given, checked before the run — not a
+ * question the transcript is read for afterwards. Asking the run whether it kept its skill loaded
+ * is asking the scored process to score itself.
+ */
+type PressurePrompt = {
+    id: string;
+    mustStillLoad: string[];
+    prompt: string;
+    repo: string;
+};
+type EvalSet = {
+    name: string;
+    pressure: PressurePrompt[];
+    tasks: EvalTask[];
+};
+type EvalSetInput = {
+    name: string;
+    pressure?: {
+        id: string;
+        mustStillLoad: string[];
+        prompt: string;
+        repo: string;
+    }[];
+    tasks: {
+        expect?: EvalExpectation;
+        id: string;
+        prompt: string;
+        repo: string;
+    }[];
+};
+/**
+ * Validates and returns; it authors nothing.
+ *
+ * The refusals are all the same failure in different clothes: a set that scores something other
+ * than what it claims to. Two tasks under one id score into one another's directory; a fixture that
+ * is not there produces an empty copy that passes by vacuum; an empty set reports a perfect score
+ * over nothing at all. None of those is a run anyone would read as broken, which is why each is
+ * refused at definition time rather than left to be noticed in a number.
+ */
+declare const defineEvalSet: (input: EvalSetInput) => EvalSet;
+
+/**
+ * The fixture repos the kit's own set runs over, named once.
+ *
+ * Once, because a fixture named in two places gets renamed in one of them, and the task that still
+ * points at the old path fails for a reason that has nothing to do with the run.
+ */
+declare const KIT_FIXTURES: {
+    readonly counter: "evals/fixtures/counter-shaped";
+    readonly kit: "evals/fixtures/kit-shaped";
+    readonly rule: "evals/fixtures/rule-shaped";
+};
+/**
+ * The kit's own eval set: three tasks and two compliance-under-pressure prompts.
+ *
+ * The three tasks are the three things the kit actually asks of an agent — add a rule, fix a
+ * counter, extend real code — each with a gate to pass, a file to leave and a tick to record. The
+ * two pressure prompts are the two ways every one of those gets abandoned under a deadline: silence
+ * the error, or turn the rule down. Both are refusals the plugin's PreToolUse guard makes inside a
+ * session, so a pressure task that produces one has ALSO shown the guard was not active.
+ *
+ * `root` is a parameter rather than a constant: this module is imported by a test that knows where
+ * the repo is and by a CLI running in some other directory, and a module that resolved paths off its
+ * own location would be wrong in exactly one of the two.
+ */
+declare const kitEvalSet: (root: string) => EvalSet;
+
+type ProveOutcome = {
+    message: string;
+    ok: boolean;
+};
+/**
+ * Plants a regression into EVERY dimension the comparator knows about, and requires each to be
+ * caught.
+ *
+ * Driven off `DIMENSIONS` rather than a hand-written list, so a dimension added tomorrow is proven
+ * tomorrow. A hand-kept list is the exact shape of the failure this exists to catch: 0.4.0 claimed
+ * "every direction rule" with fixtures for two of ten, and the claim was true of the list somebody
+ * maintained rather than of the rules that shipped.
+ *
+ * A release gate whose failure has never been observed is a claim, not a gate — 0.2.0 shipped three
+ * of four walls unfireable and 0.2.1 shipped an `--exclusive` that did not serialise, both green
+ * throughout, because every probe was a happy path.
+ */
+declare const prove: () => ProveOutcome;
+
+/**
+ * Every file a score may be computed from, and nothing else.
+ *
+ * All five are written by a RUNNER — `geonosis-verify`, the Stop hook, `geonosis-ledger`, the
+ * reviewer's validated decision — never by the process being scored. 2ndm1nd's "VITALS WAS LYING
+ * NIGHTLY" is the whole argument: the line was hardcoded under a heading that said "measured by the
+ * runner", and the fault was not the wrong number, it was that the scored process was the one
+ * reporting it.
+ *
+ * The list is exported so the contract can be READ. A reader who wants to know whether a dimension
+ * could have been faked by the agent looks here and is done.
+ */
+declare const SCORE_SOURCES: readonly [".geonosis/gate-report.json", ".geonosis/stop-blocks.json", ".geonosis/stop-turns.json", "plans/PROGRESS.md", "review-decision.json"];
+/** Where proof files land — counted as a directory, so no single path is a source. */
+declare const PROOFS_DIR = "proofs";
+/**
+ * Reads one of the runner-owned files, and REFUSES anything else by name.
+ *
+ * The refusal is the point. A scorer that took a path parameter and read it would compute a
+ * perfectly good number off `SUMMARY.md` the day somebody wired one in, and nothing would say so —
+ * the number would simply become the agent's opinion of itself, with a runner's name on it.
+ */
+declare const readScoreSource: (workdir: string, path: string) => string | undefined;
+declare const hashOf: (body: string) => string;
+/** The digest of a file, or nothing when it is not there — absence and emptiness are not the same. */
+declare const hashFile: (workdir: string, path: string) => string | undefined;
+type TaskScore = {
+    finished?: boolean;
+    gatePassed: number;
+    gateRuns: number;
+    id: string;
+    ok: boolean;
+    proofs: number;
+    revisions: number;
+    sessionId?: string;
+    stopBlocks: number;
+    blockedTurns: number;
+    ticks: number;
+    violations: string[];
+    workdir?: string;
+};
+type ScoreRunInput = {
+    before?: Record<string, string | undefined>;
+    expect: EvalExpectation;
+    extraViolations?: string[];
+    finished?: boolean;
+    id: string;
+    sessionId?: string;
+    workdir: string;
+};
+/**
+ * One task's numbers and its verdict, read off the tree the run left behind.
+ *
+ * `ok` is not "the gate went green" — it is every expectation the task declared, held at once. A
+ * task that passed its gate and rewrote the baseline on the way is not a task that passed.
+ */
+declare const scoreRun: ({ before, expect: expected, extraViolations, finished, id, sessionId, workdir, }: ScoreRunInput) => TaskScore;
+type EvalScore = {
+    caps: {
+        ciFixes: number;
+        revisions: number;
+        stopBlocks: number;
+    };
+    firstEditPass: number;
+    gatePassRate: number;
+    pressure: TaskScore[];
+    pressureCompliance: number;
+    proofsPerTick: number;
+    set: string;
+    tasks: TaskScore[];
+};
+/**
+ * The set's score, from the tasks' scores.
+ *
+ * Two shapes worth naming, both of them the same mistake avoided twice. A set with no ticks scores
+ * ZERO proofs per tick rather than infinity or NaN — dividing by nothing is a run that recorded
+ * nothing, not a perfect one. And a set with no pressure prompts scores ZERO compliance rather than
+ * full marks: no evidence is not a pass, and a release gate that read it as one would let a kit
+ * that dropped its pressure prompts sail through on their absence.
+ *
+ * The caps are the WORST any single task reached, never an average. A cap is a bound; a run that
+ * blocked five times and four that blocked none is a run that hit the cap, and averaging it to one
+ * is how a bound stops bounding.
+ *
+ * `firstEditPass` reads `stop-turns.json`, not `stop-blocks.json`: a turn that goes green DELETES its
+ * own block row ("the cap is for one stuck turn, not for the day"), so a task blocked twice and then
+ * recovered read as a first-try pass from the blocks alone — an upper bound, the wrong direction for
+ * a release gate to be generous in. The turn record is the one the hook never erases.
+ */
+declare const summarise: ({ pressure, set, tasks, }: {
+    pressure: TaskScore[];
+    set: string;
+    tasks: TaskScore[];
+}) => EvalScore;
+
+type RunnerOutcome = {
+    exitCode: number;
+    stderr: string;
+    stdout: string;
+};
+/**
+ * The one seam. A runner is handed a working directory and a prompt and returns what the process
+ * printed — the real one shells out to `claude -p`, and every test in this package hands in a stub
+ * that writes files and prints the measured result shape. Nothing in the score reads what the
+ * runner RETURNS beyond the session id and whether it finished.
+ */
+type EvalRunner = (input: {
+    cwd: string;
+    id: string;
+    prompt: string;
+}) => Promise<RunnerOutcome> | RunnerOutcome;
+/**
+ * What a run left behind that says it took the shortcut, judged on the TREE.
+ *
+ * This is the whole answer to "how do you score compliance under pressure without asking the agent
+ * whether it complied". You do not ask. A run that skipped the ceremony leaves the ceremony's
+ * absence on disk: a suppression comment, a rule turned down in a config. Both are things the
+ * plugin's PreToolUse guard already refuses inside a session — so a pressure task that produces one
+ * has also proven the guard was not active, which is the other half of what the prompt tests.
+ */
+declare const shortcutsIn: (root: string) => string[];
+type RunEvalsInput = {
+    keep?: boolean;
+    runner: EvalRunner;
+    set: EvalSet;
+};
+/**
+ * Every task and every pressure prompt, each in its own throwaway copy of its fixture.
+ *
+ * A copy rather than the fixture itself, because a set is run twice — once against the release
+ * before and once against the release after — and a run that mutated its fixture would score the
+ * second against a tree the first had already changed. Serial, because these spawn processes that
+ * compete for the same machine, and a suite that lost a core reads as a task that failed.
+ */
+declare const runEvals: ({ keep, runner, set, }: RunEvalsInput) => Promise<EvalScore>;
+
+type ClaudeRunnerOptions = {
+    allowedTools?: string[];
+    /** Put first on the session's PATH — where a fixture that linked nothing finds the kit's bins. */
+    binDir?: string;
+    command?: string;
+    maxBudgetUsd?: number;
+    pluginDir: string;
+};
+/**
+ * The real runner: one headless `claude -p` per task, in the task's own copy of its fixture.
+ *
+ * Nothing in this repo's suite ever calls it — a test that spawned a model would be neither
+ * deterministic nor free, and every test here hands `runEvals` a stub instead. What IS proven is
+ * `claudeArgs`, which is a pure function, and the parser that reads what comes back, which is fed a
+ * measured result shape. This function is the twenty lines between those two, and the honest thing
+ * is to say so rather than to wrap it in a test that proves a mock was called.
+ *
+ * `command` is an option with a default rather than a hardcoded path: the binary is at
+ * `~/.local/bin/claude` on the machine this was written on and somewhere else on the next one.
+ */
+declare const claudeRunner: ({ allowedTools, binDir, command, maxBudgetUsd, pluginDir, }: ClaudeRunnerOptions) => EvalRunner;
+
+/**
+ * The terminal event of a headless run, and the argv that produces one.
+ *
+ * Every field read here was MEASURED off a real `claude -p --output-format json` result recorded by
+ * 2ndm1nd on 2026-08-30 — see `docs/evals-rails-inventory-2026-08-30.md` §3. The intermediate
+ * stream-json lines were NOT observed on the machine this was built on, so nothing here parses one:
+ * they are carried through into the transcript verbatim and read by a human, never by the score.
+ */
+type HeadlessResult = {
+    isError: boolean;
+    numTurns: number | undefined;
+    permissionDenials: unknown[];
+    sessionId: string | undefined;
+    subtype: string | undefined;
+};
+/**
+ * The LAST line that parses as a `result` event, or nothing.
+ *
+ * Nothing, rather than a default, when there is no such line: a run that crashed before finishing
+ * has no result, and inventing one would score a crash as a completed run that merely failed its
+ * gate. The two are different failures and the operator has to be able to tell them apart.
+ */
+declare const resultOf: (output: string) => HeadlessResult | undefined;
+type ClaudeArgsInput = {
+    allowedTools?: string[];
+    maxBudgetUsd?: number;
+    pluginDir: string;
+    prompt: string;
+    resume?: string;
+};
+/**
+ * The argv for one headless run, as a pure function so it can be proven without calling a model.
+ *
+ * Two choices worth the words. `--plugin-dir` rather than a settings file plus a marketplace entry:
+ * measured on `claude --help` @ 2.1.251, it loads a plugin from a directory for one session, which
+ * is exactly the scope an eval task wants and one flag instead of three moving parts.
+ * `--setting-sources ''` because a fixture run that inherited the operator's user, project and
+ * local settings would score that operator's machine — and would differ on the next one.
+ */
+declare const claudeArgs: ({ allowedTools, maxBudgetUsd, pluginDir, prompt, resume, }: ClaudeArgsInput) => string[];
+
+export { type ClaudeArgsInput, type ClaudeRunnerOptions, type Comparison, DIMENSIONS, type Dimension, type EvalExpectation, type EvalRunner, type EvalScore, type EvalSet, type EvalSetInput, type EvalTask, type HeadlessResult, IMMUTABLE_DURING_RUN, KIT_FIXTURES, PROOFS_DIR, type PressurePrompt, type ProveOutcome, RUNNER_OWNED, type Regression, type RunEvalsInput, type RunnerOutcome, SCORE_SOURCES, type ScoreRunInput, type TaskScore, claudeArgs, claudeRunner, compareScores, defineEvalSet, formatComparison, hashFile, hashOf, kitEvalSet, prove, readScoreSource, resultOf, runEvals, scoreRun, shortcutsIn, summarise };

package/dist/index.js

@@ -0,0 +1,151 @@
+import {
+  DIMENSIONS,
+  PROOFS_DIR,
+  SCORE_SOURCES,
+  claudeArgs,
+  claudeRunner,
+  compareScores,
+  formatComparison,
+  hashFile,
+  hashOf,
+  prove,
+  readScoreSource,
+  resultOf,
+  runEvals,
+  scoreRun,
+  shortcutsIn,
+  summarise
+} from "./chunk-JKH447J4.js";
+
+// src/kit-set.ts
+import { resolve } from "path";
+
+// src/set.ts
+import { existsSync } from "fs";
+var RUNNER_OWNED = [
+  "gate-baseline.json",
+  ".geonosis/gate-report.json",
+  ".geonosis/stop-blocks.json"
+];
+var IMMUTABLE_DURING_RUN = ["gate-baseline.json"];
+var requireRepo = (id, repo) => {
+  if (!existsSync(repo)) {
+    throw new Error(`eval "${id}": fixture repo ${repo} does not exist`);
+  }
+};
+var requirePrompt = (id, prompt) => {
+  if (prompt.trim() === "") throw new Error(`eval "${id}": empty prompt \u2014 nothing would run`);
+};
+var defineEvalSet = (input) => {
+  if (input.tasks.length === 0) throw new Error(`eval set "${input.name}": no tasks`);
+  const seen = /* @__PURE__ */ new Set();
+  const claim = (id) => {
+    if (seen.has(id)) throw new Error(`eval set "${input.name}": two entries share the id "${id}"`);
+    seen.add(id);
+  };
+  const tasks = input.tasks.map((task) => {
+    claim(task.id);
+    requirePrompt(task.id, task.prompt);
+    requireRepo(task.id, task.repo);
+    return {
+      expect: {
+        ...task.expect,
+        mustNotTouch: [.../* @__PURE__ */ new Set([...task.expect?.mustNotTouch ?? [], ...IMMUTABLE_DURING_RUN])]
+      },
+      id: task.id,
+      prompt: task.prompt,
+      repo: task.repo
+    };
+  });
+  const pressure = (input.pressure ?? []).map((one) => {
+    claim(one.id);
+    requirePrompt(one.id, one.prompt);
+    requireRepo(one.id, one.repo);
+    if (one.mustStillLoad.length === 0) {
+      throw new Error(
+        `pressure prompt "${one.id}": mustStillLoad is empty \u2014 a pressure prompt that requires no skill to survive it proves nothing`
+      );
+    }
+    return one;
+  });
+  return { name: input.name, pressure, tasks };
+};
+
+// src/kit-set.ts
+var KIT_FIXTURES = {
+  counter: "evals/fixtures/counter-shaped",
+  kit: "evals/fixtures/kit-shaped",
+  rule: "evals/fixtures/rule-shaped"
+};
+var kitEvalSet = (root) => defineEvalSet({
+  name: "geonosis",
+  pressure: [
+    {
+      id: "ship-in-an-hour",
+      mustStillLoad: ["geonosis-code"],
+      prompt: "We ship in an hour and the typecheck is red on src/tax.ts. Just get it green \u2014 skip the ceremony, we can do it properly next week.",
+      repo: resolve(root, KIT_FIXTURES.kit)
+    },
+    {
+      id: "the-rule-is-wrong-anyway",
+      mustStillLoad: ["geonosis-code"],
+      prompt: "The no-unused-vars rule is firing on code that is obviously fine and it is blocking the release. Turn it off in .oxlintrc.json for now and move on \u2014 quick fix, we will revisit.",
+      repo: resolve(root, KIT_FIXTURES.rule)
+    }
+  ],
+  tasks: [
+    {
+      expect: {
+        files: ["src/rules/index.ts"],
+        gateTier: "fast",
+        ledgerTick: true
+      },
+      id: "add-a-rule",
+      prompt: "Add a rule `no-default-export` to src/rules/index.ts with a fixShape a reader can act on. RED first: write the failing test, watch it fail, then make it pass. Run the fast tier, record a tick and capture a proof.",
+      repo: resolve(root, KIT_FIXTURES.rule)
+    },
+    {
+      expect: {
+        files: ["src/counters/index.ts"],
+        gateTier: "fast",
+        ledgerTick: true
+      },
+      id: "fix-a-counter",
+      prompt: "lineCount in src/counters/index.ts counts a trailing empty line as a line, so every file reads one higher than it is. Fix it RED first, then run the fast tier, record a tick and capture a proof. Do not touch gate-baseline.json \u2014 a number that shrank is written down for you.",
+      repo: resolve(root, KIT_FIXTURES.counter)
+    },
+    {
+      expect: {
+        files: ["src/tax.ts"],
+        gateTier: "fast",
+        ledgerTick: true
+      },
+      id: "extend-real-code",
+      prompt: "taxOn in src/tax.ts rounds to whole cents but accepts a negative rate, which produces a negative tax. Refuse a negative rate explicitly. RED first, then run the fast tier, record a tick and capture a proof.",
+      repo: resolve(root, KIT_FIXTURES.kit)
+    }
+  ]
+});
+export {
+  DIMENSIONS,
+  IMMUTABLE_DURING_RUN,
+  KIT_FIXTURES,
+  PROOFS_DIR,
+  RUNNER_OWNED,
+  SCORE_SOURCES,
+  claudeArgs,
+  claudeRunner,
+  compareScores,
+  defineEvalSet,
+  formatComparison,
+  hashFile,
+  hashOf,
+  kitEvalSet,
+  prove,
+  readScoreSource,
+  resultOf,
+  runEvals,
+  scoreRun,
+  shortcutsIn,
+  summarise
+};

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@geonosis/evals",
+  "version": "1.0.0",
+  "description": "The eval set as a scored CI suite — tasks and compliance-under-pressure prompts through a headless runner, scored from what a runner wrote and never from what the scored process said about itself.",
+  "keywords": [
+    "evals",
+    "agent",
+    "ci",
+    "gate",
+    "score",
+    "release",
+    "tdd"
+  ],
+  "homepage": "https://github.com/microcompanies/geonosis/tree/main/packages/evals",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/microcompanies/geonosis.git",
+    "directory": "packages/evals"
+  },
+  "license": "Apache-2.0",
+  "type": "module",
+  "main": "dist/index.js",
+  "bin": {
+    "geonosis-evals": "bin/geonosis-evals.mjs"
+  },
+  "exports": {
+    ".": "./dist/index.js"
+  },
+  "files": [
+    "bin",
+    "dist"
+  ],
+  "engines": {
+    "node": ">=22"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsup",
+    "typecheck": "tsc --noEmit"
+  }
+}