@moo.md/hum 0.0.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/package.json +2 -2
  2. package/schema/hum.schema.json +2 -2
  3. package/scripts/eval.ts +88 -50
  4. package/src/adapter-api.ts +46 -14
  5. package/src/adapters/admit.ts +2 -2
  6. package/src/adapters/argument.ts +1 -1
  7. package/src/adapters/claim.ts +18 -15
  8. package/src/adapters/claude-bg.ts +72 -32
  9. package/src/adapters/claude-p.ts +26 -25
  10. package/src/adapters/contract.ts +22 -10
  11. package/src/adapters/each-call.ts +9 -0
  12. package/src/adapters/fake.ts +6 -6
  13. package/src/adapters/git.ts +12 -19
  14. package/src/adapters/github-client.ts +102 -0
  15. package/src/adapters/github.ts +50 -136
  16. package/src/adapters/guard.ts +20 -9
  17. package/src/adapters/history.ts +12 -15
  18. package/src/adapters/item-file.ts +1 -1
  19. package/src/adapters/markdown.ts +49 -42
  20. package/src/adapters/marked.ts +15 -13
  21. package/src/adapters/model-call.ts +4 -4
  22. package/src/adapters/once.ts +5 -5
  23. package/src/adapters/owner-commit.ts +13 -11
  24. package/src/adapters/pull-requests.ts +11 -13
  25. package/src/adapters/registry.ts +7 -7
  26. package/src/adapters/roundup.ts +31 -37
  27. package/src/adapters/serial.ts +4 -5
  28. package/src/adapters/transcript.ts +13 -8
  29. package/src/cli/main.ts +1 -0
  30. package/src/cli/ops.ts +32 -14
  31. package/src/cli/service.ts +14 -8
  32. package/src/cli/setup.ts +8 -3
  33. package/src/cli/shared-watch.ts +3 -3
  34. package/src/config/checks.ts +14 -18
  35. package/src/config/machine.ts +5 -7
  36. package/src/config/refusal.ts +7 -0
  37. package/src/config/schema.ts +9 -9
  38. package/src/config/settings.ts +17 -2
  39. package/src/config/state-map.ts +5 -3
  40. package/src/decide/alignment.ts +4 -4
  41. package/src/decide/answer.ts +24 -22
  42. package/src/decide/kind.ts +1 -6
  43. package/src/decide/labels.ts +9 -9
  44. package/src/decide/merge.ts +6 -7
  45. package/src/decide/prose.ts +4 -4
  46. package/src/decide/question.ts +23 -20
  47. package/src/decide/review.ts +4 -2
  48. package/src/decide/risk.ts +25 -19
  49. package/src/decide/taste.ts +3 -3
  50. package/src/exec.ts +10 -4
  51. package/src/index.ts +14 -13
  52. package/src/loop/check-item.ts +3 -3
  53. package/src/loop/ci.ts +1 -1
  54. package/src/loop/github.ts +2 -2
  55. package/src/loop/groups.ts +13 -6
  56. package/src/loop/memory.ts +40 -28
  57. package/src/loop/notes.ts +5 -5
  58. package/src/loop/pair.ts +1 -1
  59. package/src/loop/pass.ts +18 -12
  60. package/src/loop/proof.ts +9 -7
  61. package/src/loop/prove-item.ts +3 -3
  62. package/src/loop/records.ts +12 -13
  63. package/src/loop/signals.ts +6 -6
  64. package/src/measure/inputs.ts +1 -1
  65. package/src/measure/measure.ts +4 -4
  66. package/src/measure/rate.ts +2 -0
  67. package/src/service/ci-tick.ts +2 -2
  68. package/src/service/supervise.ts +3 -2
  69. package/src/service/token.ts +3 -3
  70. package/src/service/wake.ts +2 -2
  71. package/src/service/workflows.ts +2 -2
  72. package/src/setup/propose.ts +4 -4
  73. package/src/setup/prove.ts +20 -14
  74. package/src/setup/scan.ts +31 -24
  75. package/src/setup/shipped.ts +8 -2
  76. package/src/setup/tables.ts +4 -3
  77. package/src/setup/waits.ts +9 -11
  78. package/src/setup/write.ts +17 -10
  79. package/src/slots/pool.ts +32 -24
  80. package/src/state/events.ts +1 -1
  81. package/src/state/faults.ts +13 -5
  82. package/src/state/format.ts +2 -2
  83. package/src/state/items.ts +20 -15
  84. package/src/state/open.ts +2 -2
  85. package/src/state/ops.ts +16 -10
  86. package/src/state/records.ts +5 -5
  87. package/src/state/runs.ts +33 -26
  88. package/src/state/saved.ts +44 -32
  89. package/src/status/file.ts +20 -1
  90. package/src/status/format.ts +47 -57
  91. package/src/status/sections.ts +2 -2
  92. package/src/worker/alignment-report.ts +4 -5
  93. package/src/worker/answers.ts +26 -24
  94. package/src/worker/base-check.ts +4 -5
  95. package/src/worker/checks.ts +25 -11
  96. package/src/worker/kind.ts +23 -19
  97. package/src/worker/land.ts +115 -126
  98. package/src/worker/merged.ts +9 -7
  99. package/src/worker/open.ts +15 -13
  100. package/src/worker/prose-run.ts +3 -3
  101. package/src/worker/questions.ts +15 -12
  102. package/src/worker/scheduler.ts +4 -4
  103. package/src/worker/signature.ts +3 -3
  104. package/src/worker/tally.ts +0 -2
  105. package/src/worker/taste.ts +6 -15
  106. package/src/worker/thin-path.ts +150 -123
  107. package/src/worker/thread.ts +25 -0
  108. package/src/worker/worker.ts +125 -73
  109. package/surface.json +633 -2198
  110. package/src/slots/scan.ts +0 -26
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@moo.md/hum",
3
- "version": "0.0.0",
3
+ "version": "0.2.0",
4
4
  "description": "The hum engine: work items become Claude Code agent runs, the repo's checks decide what they can, and only what nothing settles becomes a question to the owner.",
5
5
  "license": "MIT",
6
6
  "author": {
@@ -32,7 +32,7 @@
32
32
  "surface.json"
33
33
  ],
34
34
  "scripts": {
35
- "typecheck": "tsc --noEmit",
35
+ "typecheck": "test -d ../hum/node_modules || { echo \"hum-engine: the view tests import hum/mod, whose types come from hum's packages; run bun install in hum first\" >&2; exit 1; }; tsc --noEmit",
36
36
  "schema": "bun scripts/write-schema.ts",
37
37
  "eval": "bun scripts/eval.ts",
38
38
  "release:check": "bun tools/release/main.ts check",
@@ -480,7 +480,7 @@
480
480
  "type": "integer",
481
481
  "exclusiveMinimum": 0,
482
482
  "maximum": 9007199254740991,
483
- "description": "How long one git, gh, npm or Claude Code management command may run before it is stopped and its call fails naming it. A run's agent and a repo's checks have their own limits."
483
+ "description": "How long one git, gh, npm or Claude Code management command, one GitHub request or one start check may run before it is stopped and its call fails naming it. A run's agent and a repo's checks have their own limits."
484
484
  },
485
485
  "mergeTries": {
486
486
  "default": 3,
@@ -1251,7 +1251,7 @@
1251
1251
  "type": "integer",
1252
1252
  "exclusiveMinimum": 0,
1253
1253
  "maximum": 9007199254740991,
1254
- "description": "How long running a new check on one note's commit may take before the proof fails."
1254
+ "description": "How long the repo's checks may run, on a run's branch, on the base after a merge or as a new check on one note's commit, before they are stopped and fail."
1255
1255
  },
1256
1256
  "kindFloors": {
1257
1257
  "default": {},
package/scripts/eval.ts CHANGED
@@ -6,10 +6,10 @@ import { AgentDecisionExhaustedError } from "@statelyai/agent";
6
6
  import PQueue from "p-queue";
7
7
  import { z } from "zod";
8
8
  import { type Finished, type Runner, startRunner } from "../src/adapters/claude-p.ts";
9
+ import { gitIn, readGitIdentity } from "../src/adapters/git.ts";
9
10
  import { modelCallOver } from "../src/adapters/model-call.ts";
10
- import { readConfigDir } from "../src/config/load.ts";
11
- import { BOUNDARIES, type Boundary, HumConfig } from "../src/config/schema.ts";
12
- import { alignmentQuestion, raisesLevel } from "../src/decide/alignment.ts";
11
+ import { BOUNDARIES, type Boundary } from "../src/config/schema.ts";
12
+ import { alignmentQuestion, doesRaiseLevel } from "../src/decide/alignment.ts";
13
13
  import { replyQuestion } from "../src/decide/answer.ts";
14
14
  import { kindQuestion } from "../src/decide/kind.ts";
15
15
  import { tasteQuestion } from "../src/decide/taste.ts";
@@ -17,17 +17,21 @@ import { pairQuestion } from "../src/loop/pair.ts";
17
17
  import type { ReviewNote } from "../src/loop/records.ts";
18
18
  import { pairing, prepare } from "../src/loop/signals.ts";
19
19
  import type { JsonSchema, ProsePurpose } from "../src/decide/prose.ts";
20
+ import { SHIPPED_COMMAND_TIMEOUT_MS } from "../src/exec.ts";
21
+ import { shippedRaw, shippedSettingsAs } from "../src/setup/shipped.ts";
20
22
  import { openState } from "../src/state/open.ts";
21
23
  import { decideProse } from "../src/worker/prose-run.ts";
22
24
  import { PROSE_PROMPTS } from "../src/worker/thin-path.ts";
23
25
  import { SECOND_MS } from "../src/time.ts";
26
+ import { rateOf } from "../src/measure/rate.ts";
24
27
 
25
28
  /**
26
29
  * Measures one model-judgment boundary on its labelled cases, through the same prompts and fences the engine runs,
27
30
  * and adds a row to its RESULTS.md. `bun run eval -- --model <id>` measures every boundary on that model id; naming
28
31
  * boundaries measures only those. A case the runner held or limited counts in no rate. `--runs <n>` repeats each
29
32
  * boundary and pools the runs, since one run proves little; `--write <folder>` records each pooled rate under
30
- * `measuredModels` in that folder's `hum.json`, which is how setup measures an owner's own models.
33
+ * `measuredModels` in that folder's `hum.json`, which is how setup measures an owner's own models. A boundary where
34
+ * nothing was measured records no rate, and the script exits non-zero.
31
35
  */
32
36
 
33
37
  const { values, positionals } = parseArgs({ allowPositionals: true, options: { model: { type: "string" }, limit: { type: "string" }, runs: { type: "string" }, write: { type: "string" } } });
@@ -36,16 +40,23 @@ const OWNER_LABELLED: readonly Boundary[] = ["taste"];
36
40
 
37
41
  const boundaries: readonly Boundary[] = positionals.length > 0 ? z.array(z.enum(BOUNDARIES)).parse(positionals) : BOUNDARIES.filter((boundary) => !OWNER_LABELLED.includes(boundary));
38
42
  const root = join(import.meta.dir, "..");
39
- const raw = readConfigDir(join(root, "defaults"));
40
- const config = { hum: HumConfig.parse(raw.hum), roles: raw.roles };
43
+ const settings = shippedSettingsAs(await readGitIdentity(gitIn(process.cwd(), { timeoutMs: SHIPPED_COMMAND_TIMEOUT_MS })));
44
+
45
+ if (!settings.ok) {
46
+ console.error(settings.refusal.message);
47
+ process.exit(1);
48
+ }
49
+
50
+ const config = { hum: settings.value, roles: shippedRaw().roles };
41
51
  const model = values.model ?? config.hum.decisionModel;
42
- const runs = values.runs === undefined ? 1 : Number(values.runs);
43
- const limit = values.limit === undefined ? Number.POSITIVE_INFINITY : Number(values.limit);
52
+ const Count = z.coerce.number().int().positive();
53
+ const runs = values.runs === undefined ? 1 : Count.parse(values.runs);
54
+ const limit = values.limit === undefined ? Number.POSITIVE_INFINITY : Count.parse(values.limit);
44
55
 
45
56
  type Verdict = "right" | "wrong" | "abstained" | "halted";
46
57
 
47
- const state = openState(mkdtempSync(join(tmpdir(), "hum-eval-")));
48
- const started = await startRunner({ state, model });
58
+ const state = openState(mkdtempSync(join(tmpdir(), "hum-eval-")), { now: Date.now });
59
+ const started = await startRunner({ state, model, now: Date.now, env: process.env });
49
60
 
50
61
  if (started.check.kind !== "finished") throw new Error(`the start check ended as ${started.check.kind}`);
51
62
 
@@ -109,7 +120,7 @@ const TasteCase = z.object({ case: z.string(), label: z.enum(["keeps", "breaks"]
109
120
  /** Calls at once: enough to finish a set in minutes, few enough not to trip a rate limit. */
110
121
  const AT_ONCE = 4;
111
122
 
112
- const eachOf = <T>(cases: readonly T[], measure: (found: T) => Promise<Verdict>): Promise<Verdict[]> => new PQueue({ concurrency: AT_ONCE }).addAll(cases.map((found) => () => measure(found)));
123
+ const eachOf = <T, R>(cases: readonly T[], measure: (found: T) => Promise<R>): Promise<R[]> => new PQueue({ concurrency: AT_ONCE }).addAll(cases.map((found) => () => measure(found)));
113
124
 
114
125
  const read = async <T>(boundary: Boundary, schema: z.ZodType<T>, file = "cases.jsonl"): Promise<T[]> =>
115
126
  (await Bun.file(join(root, "evals", boundary, file)).text())
@@ -118,78 +129,98 @@ const read = async <T>(boundary: Boundary, schema: z.ZodType<T>, file = "cases.j
118
129
  .map((line) => schema.parse(JSON.parse(line)));
119
130
 
120
131
  /** The confusion counts of the review-note boundary: its rate is precision, and recall is recorded beside it. */
121
- const pairCounts = { tp: 0, fp: 0, fn: 0, tn: 0, pairedBySignals: 0 };
132
+ type PairCounts = { readonly tp: number; readonly fp: number; readonly fn: number; readonly tn: number; readonly pairedBySignals: number };
133
+
134
+ type Cell = Exclude<keyof PairCounts, "pairedBySignals">;
135
+
136
+ const NO_PAIRS: PairCounts = { tp: 0, fp: 0, fn: 0, tn: 0, pairedBySignals: 0 };
137
+
138
+ const addPairs = (a: PairCounts, b: PairCounts): PairCounts => ({ tp: a.tp + b.tp, fp: a.fp + b.fp, fn: a.fn + b.fn, tn: a.tn + b.tn, pairedBySignals: a.pairedBySignals + b.pairedBySignals });
139
+
140
+ /** One run of a boundary: each case's verdict, and for the review-note boundary its confusion counts. */
141
+ type Measured = { readonly verdicts: readonly Verdict[]; readonly pairs?: PairCounts };
122
142
 
123
143
  const noteOf = (text: string, sourceId: string): ReviewNote => ({ source: "review-comment", sourceId, change: sourceId, hunk: "", text, at: 0 });
124
144
 
125
- const BOUNDARY_RUNS: Record<Boundary, () => Promise<Verdict[]>> = {
145
+ const BOUNDARY_RUNS: Record<Boundary, () => Promise<Measured>> = {
126
146
  kind: async () => {
127
147
  const cases = (await read("kind", KindCase)).slice(0, limit);
128
148
 
129
- return eachOf(cases, async (found) => {
149
+ return {
150
+ verdicts: await eachOf(cases, async (found) => {
130
151
  const judged = await judge(kindQuestion(found, config.hum));
131
152
 
132
153
  return judged.kind === "abstained" ? "abstained" : scored(judged.value.kind, found.label);
133
- });
154
+ }),
155
+ };
134
156
  },
135
157
  alignment: async () => {
136
158
  const sample = (await read("alignment", AlignmentCase, "sample.jsonl")).slice(0, limit);
137
159
 
138
- return eachOf(sample, async (found) => {
160
+ return {
161
+ verdicts: await eachOf(sample, async (found): Promise<Verdict> => {
139
162
  const judged = await judge(alignmentQuestion({ title: found.subject, body: "" }, found.diff, config.hum));
140
163
 
141
164
  if (judged.kind === "abstained") return "abstained";
142
165
 
143
- return raisesLevel(0, config.hum.levels.length, judged.value, config.hum) === (found.label !== "clean") ? "right" : "wrong";
144
- });
166
+ return doesRaiseLevel({ rank: 0, levels: config.hum.levels.length, answer: judged.value, config: config.hum }) === (found.label !== "clean") ? "right" : "wrong";
167
+ }),
168
+ };
145
169
  },
146
170
  "plan-outcome": async () => {
147
171
  const repo = planRepo();
148
172
  const cases = (await read("plan-outcome", PlanCase)).slice(0, limit);
149
173
 
150
- return eachOf(cases, async (found) => scored(await prosePick("plan", PROSE_PROMPTS.plan(config, { id: found.case, title: found.body.split("\n")[0] ?? "", body: found.body }), "", repo), found.label));
174
+ return { verdicts: await eachOf(cases, async (found) => scored(await prosePick("plan", PROSE_PROMPTS.plan(config, { id: found.case, title: found.body.split("\n")[0] ?? "", body: found.body }), "", repo), found.label)) };
151
175
  },
152
176
  "reply-option": async () => {
153
177
  const cases = (await read("reply-option", ReplyCase)).slice(0, limit);
154
178
 
155
- return eachOf(cases, async (found) => {
156
- const judged = await judge(replyQuestion(found.question, found.options, found.reply));
179
+ return {
180
+ verdicts: await eachOf(cases, async (found) => {
181
+ const judged = await judge(replyQuestion({ question: found.question, options: found.options, reply: found.reply }));
157
182
 
158
- return judged.kind === "abstained" ? "abstained" : scored(judged.value.option, found.label);
159
- });
183
+ return judged.kind === "abstained" ? "abstained" : scored(judged.value.option, found.label);
184
+ }),
185
+ };
160
186
  },
161
187
  "repeat-pair": async () => {
162
188
  const cases = (await read("repeat-pair", PairCase)).slice(0, limit);
163
189
  const open = cases.filter((found) => pairing(prepare(noteOf(found.a.text, "a")), prepare(noteOf(found.b.text, "b"))).signal === undefined);
164
190
 
165
- pairCounts.pairedBySignals = cases.length - open.length;
191
+ const judged = await eachOf(open, async (found): Promise<{ verdict: Verdict; cell?: Cell }> => {
192
+ const pair = await judge(pairQuestion(noteOf(found.a.text, "a"), noteOf(found.b.text, "b")));
166
193
 
167
- return eachOf(open, async (found) => {
168
- const judged = await judge(pairQuestion(noteOf(found.a.text, "a"), noteOf(found.b.text, "b")));
194
+ if (pair.kind === "abstained") return { verdict: "abstained" };
169
195
 
170
- if (judged.kind === "abstained") return "abstained";
171
-
172
- const said = judged.value.same;
196
+ const said = pair.value.same;
173
197
  const truth = found.label === "same";
174
198
 
175
- pairCounts[said ? (truth ? "tp" : "fp") : truth ? "fn" : "tn"]++;
176
-
177
- return said === truth ? "right" : "wrong";
199
+ return { verdict: said === truth ? "right" : "wrong", cell: said ? (truth ? "tp" : "fp") : truth ? "fn" : "tn" };
178
200
  });
201
+
202
+ const countOf = (cell: Cell) => judged.filter((found) => found.cell === cell).length;
203
+
204
+ return {
205
+ verdicts: judged.map(({ verdict }) => verdict),
206
+ pairs: { tp: countOf("tp"), fp: countOf("fp"), fn: countOf("fn"), tn: countOf("tn"), pairedBySignals: cases.length - open.length },
207
+ };
179
208
  },
180
209
  taste: async () => {
181
210
  const cases = (await read("taste", TasteCase)).slice(0, limit);
182
211
 
183
- return eachOf(cases, async (found) => {
184
- const judged = await judge(tasteQuestion(found.rule, found.diff, config.hum));
212
+ return {
213
+ verdicts: await eachOf(cases, async (found) => {
214
+ const judged = await judge(tasteQuestion(found.rule, found.diff, config.hum));
185
215
 
186
- return judged.kind === "abstained" ? "abstained" : scored(judged.value.keeps ? "keeps" : "breaks", found.label);
187
- });
216
+ return judged.kind === "abstained" ? "abstained" : scored(judged.value.keeps ? "keeps" : "breaks", found.label);
217
+ }),
218
+ };
188
219
  },
189
220
  "review-verdict": async () => {
190
221
  const cases = (await read("review-verdict", ReviewCase)).slice(0, limit);
191
222
 
192
- return eachOf(cases, async (found) => scored(await prosePick("review", PROSE_PROMPTS.review(config, { id: found.case, title: found.item, body: "" }, found.diff), found.diff, tmpdir()), found.label));
223
+ return { verdicts: await eachOf(cases, async (found) => scored(await prosePick("review", PROSE_PROMPTS.review(config, { id: found.case, title: found.item, body: "" }, found.diff), found.diff, tmpdir()), found.label)) };
193
224
  },
194
225
  };
195
226
 
@@ -197,12 +228,13 @@ const header = "| Date | Model id | n | Right | Wrong | Abstained | Rate | Live
197
228
 
198
229
  const pairHeader = "| Date | Model id | n | TP | FP | FN | TN | Abstained | Paired by signals | Precision | Recall | Live calls |\n|---|---|---|---|---|---|---|---|---|---|---|---|\n";
199
230
 
200
- const ratio = (part: number, whole: number): string => (whole === 0 ? "0.00" : (part / whole).toFixed(2));
231
+ /** A rate as a RESULTS.md cell; a dash where nothing was measured. */
232
+ const rateCell = (rate: number | undefined): string => rate?.toFixed(2) ?? "-";
201
233
 
202
- const pairRow = (day: string, abstained: number, used: number): string =>
203
- `| ${day} | ${model} | ${pairCounts.tp + pairCounts.fp + pairCounts.fn + pairCounts.tn} | ${pairCounts.tp} | ${pairCounts.fp} | ${pairCounts.fn} | ${pairCounts.tn} | ${abstained} | ${pairCounts.pairedBySignals} | ${ratio(pairCounts.tp, pairCounts.tp + pairCounts.fp)} | ${ratio(pairCounts.tp, pairCounts.tp + pairCounts.fn)} | ${used} |`;
234
+ const precisionOf = ({ tp, fp }: PairCounts): number | undefined => rateOf(tp, tp + fp);
204
235
 
205
- const EMPTY_POOL = { right: 0, measured: 0 };
236
+ const pairRow = (day: string, pairs: PairCounts, abstained: number, used: number): string =>
237
+ `| ${day} | ${model} | ${pairs.tp + pairs.fp + pairs.fn + pairs.tn} | ${pairs.tp} | ${pairs.fp} | ${pairs.fn} | ${pairs.tn} | ${abstained} | ${pairs.pairedBySignals} | ${rateCell(precisionOf(pairs))} | ${rateCell(rateOf(pairs.tp, pairs.tp + pairs.fn))} | ${used} |`;
206
238
 
207
239
  /** Sets one pooled rate in the owner's `hum.json`, where trust reads it. */
208
240
  const recordRate = async (folder: string, boundary: Boundary, rate: number): Promise<void> => {
@@ -212,33 +244,39 @@ const recordRate = async (folder: string, boundary: Boundary, rate: number): Pro
212
244
  await Bun.write(file, `${JSON.stringify({ ...written, measuredModels: { ...written.measuredModels, [boundary]: { ...written.measuredModels[boundary], [model]: Number(rate.toFixed(2)) } } }, null, "\t")}\n`);
213
245
  };
214
246
 
215
- for (const boundary of boundaries) {
216
- const pool = { ...EMPTY_POOL };
247
+ const unmeasured: Boundary[] = [];
217
248
 
218
- Object.assign(pairCounts, { tp: 0, fp: 0, fn: 0, tn: 0, pairedBySignals: 0 });
249
+ for (const boundary of boundaries) {
250
+ const pool = { right: 0, measured: 0, pairs: NO_PAIRS };
219
251
 
220
252
  for (let run = 1; run <= runs; run++) {
221
253
  const before = calls;
222
- const verdicts = await BOUNDARY_RUNS[boundary]();
254
+ const { verdicts, pairs = NO_PAIRS } = await BOUNDARY_RUNS[boundary]();
223
255
  const count = (verdict: Verdict) => verdicts.filter((found) => found === verdict).length;
224
256
  const measured = verdicts.length - count("halted");
225
- const rate = measured === 0 ? 0 : count("right") / measured;
226
257
  const day = new Date().toISOString().slice(0, 10);
227
- const row = boundary === "repeat-pair" ? pairRow(day, count("abstained"), calls - before) : `| ${day} | ${model} | ${measured} | ${count("right")} | ${count("wrong")} | ${count("abstained")} | ${rate.toFixed(2)} | ${calls - before} |`;
258
+ const row = boundary === "repeat-pair" ? pairRow(day, pairs, count("abstained"), calls - before) : `| ${day} | ${model} | ${measured} | ${count("right")} | ${count("wrong")} | ${count("abstained")} | ${rateCell(rateOf(count("right"), measured))} | ${calls - before} |`;
228
259
  const results = join(values.write ?? root, "evals", boundary, "RESULTS.md");
229
260
  const earlier = (await Bun.file(results).exists()) ? await Bun.file(results).text() : boundary === "repeat-pair" ? pairHeader : header;
230
261
 
231
262
  pool.right += count("right");
232
263
  pool.measured += measured;
264
+ pool.pairs = addPairs(pool.pairs, pairs);
233
265
  await Bun.write(results, `${earlier}${row}\n`);
234
266
  console.log(`${boundary}: ${row}`);
235
267
  }
236
268
 
237
- const pooled = boundary === "repeat-pair" ? pairCounts.tp / (pairCounts.tp + pairCounts.fp || 1) : pool.right / (pool.measured || 1);
269
+ const pooled = boundary === "repeat-pair" ? precisionOf(pool.pairs) : rateOf(pool.right, pool.measured);
238
270
 
239
- console.log(`${boundary}: pooled over ${runs} run(s): ${pooled.toFixed(2)}`);
271
+ console.log(`${boundary}: pooled over ${runs} run(s): ${rateCell(pooled)}`);
272
+
273
+ if (pooled === undefined) unmeasured.push(boundary);
274
+ else if (values.write !== undefined) await recordRate(values.write, boundary, pooled);
275
+ }
240
276
 
241
- if (values.write !== undefined) await recordRate(values.write, boundary, pooled);
277
+ if (unmeasured.length > 0) {
278
+ console.error(`Nothing was measured on ${unmeasured.join(", ")}, so no rate was recorded for ${unmeasured.length === 1 ? "it" : "them"}.`);
279
+ process.exitCode = 1;
242
280
  }
243
281
 
244
282
  console.log(`${calls} live calls, the start check included. Record each rate under measuredModels in defaults/hum.json; a model decides a boundary alone only at minJudgeRate or above; repeat-pair takes its precision, against repeatPrecision.`);
@@ -1,12 +1,13 @@
1
1
  import { z } from "zod";
2
+ import type { Env } from "./exec.ts";
2
3
  import type { FaultInput } from "./state/faults.ts";
3
4
 
4
5
  export type { FaultInput };
5
6
 
6
7
  /** Bumped on every change to the API an adapter implements; a breaking change bumps the engine's major version. */
7
- export const ADAPTER_API_VERSION = 1;
8
+ export const ADAPTER_API_VERSION = 2;
8
9
 
9
- /** Failure that retrying cannot fix: a reply of the wrong shape, a refused push. The engine surfaces it at once. */
10
+ /** Failure that retrying cannot fix: a reply of the wrong shape, a refused push. The engine surfaces it at once, as it does a `failed: "now"` answer. */
10
11
  export class PermanentError extends Error {
11
12
  override name: string = "PermanentError";
12
13
  }
@@ -21,10 +22,19 @@ export const ItemId = z
21
22
  .string()
22
23
  .max(ITEM_ID_LENGTH, ITEM_ID_MESSAGE)
23
24
  .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/, ITEM_ID_MESSAGE)
24
- .refine((id) => !id.includes("..") && !id.endsWith(".lock"), "an item id has no '..' and does not end in '.lock'");
25
+ .refine((id) => !id.includes("..") && !id.endsWith(".lock"), "an item id has no '..' and does not end in '.lock'")
26
+ .brand<"ItemId">();
25
27
 
26
28
  export type ItemId = z.infer<typeof ItemId>;
27
29
 
30
+ /** Who holds a claim: a plain name, never an item id. */
31
+ export const Holder = z
32
+ .string()
33
+ .regex(/^[A-Za-z0-9][\w.@:-]{0,99}$/, "a holder is a plain name: letters, digits and . _ @ : -")
34
+ .brand<"Holder">();
35
+
36
+ export type Holder = z.infer<typeof Holder>;
37
+
28
38
  const text = z.string().min(1);
29
39
  const time = z.int().nonnegative();
30
40
 
@@ -60,7 +70,7 @@ export const Item = ItemRef.extend({
60
70
 
61
71
  export type Item = z.infer<typeof Item>;
62
72
 
63
- export const Claim = z.strictObject({ item: ItemId, holder: text, expiresAt: time });
73
+ export const Claim = z.strictObject({ item: ItemId, holder: Holder, expiresAt: time });
64
74
 
65
75
  export type Claim = z.infer<typeof Claim>;
66
76
 
@@ -84,39 +94,59 @@ export type FollowUp = { readonly title: string; readonly body: string };
84
94
  /** The work source: the queue, the claims and the item threads. */
85
95
  export interface WorkSource {
86
96
  listReady(): Promise<ItemRef[]>;
87
- read(id: string): Promise<Item>;
97
+ read(id: ItemId): Promise<Item>;
88
98
  /** An atomic take. An unclaimed or expired item goes to the first caller; the same holder renews. */
89
- claim(id: string, holder: string, ttlMs: number): Promise<ClaimResult>;
99
+ claim(id: ItemId, holder: Holder, ttlMs: number): Promise<ClaimResult>;
90
100
  /** Extends the holder's own claim; `won: false` when another worker holds it now. */
91
- renew(id: string, holder: string, ttlMs: number): Promise<ClaimResult>;
92
- comment(id: string, text: string): Promise<void>;
101
+ renew(id: ItemId, holder: Holder, ttlMs: number): Promise<ClaimResult>;
102
+ comment(id: ItemId, text: string): Promise<void>;
93
103
  /** Files a ready item. Filing the same `title` from the same `from` twice returns the first item. */
94
- fileFollowUp(from: string, followUp: FollowUp): Promise<ItemRef>;
104
+ fileFollowUp(from: ItemId, followUp: FollowUp): Promise<ItemRef>;
95
105
  }
96
106
 
97
107
  /** Where a run's questions and the choices made without the owner land. */
98
108
  export interface Questions {
99
- ask(id: string, question: Question): Promise<void>;
109
+ ask(id: ItemId, question: Question): Promise<void>;
100
110
  /** Records a choice made without the owner. */
101
- decide(id: string, text: string): Promise<void>;
111
+ decide(id: ItemId, text: string): Promise<void>;
102
112
  }
103
113
 
104
114
  /** Where the owner's answers arrive. */
105
115
  export interface Answers {
106
- answer(id: string, answer: Answer): Promise<void>;
116
+ answer(id: ItemId, answer: Answer): Promise<void>;
107
117
  /** The one cheap read: has anything changed since `since`. An idle call costs no rate limit. */
108
118
  watch(since?: string): Promise<Watch>;
109
119
  }
110
120
 
111
121
  export interface Release {
112
122
  /** Puts the item's artifact where reviewers see it. */
113
- publish(id: string, ref: string): Promise<void>;
123
+ publish(id: ItemId, ref: string): Promise<void>;
114
124
  /** Ships a published artifact. */
115
- release(id: string, ref: string): Promise<void>;
125
+ release(id: ItemId, ref: string): Promise<void>;
116
126
  }
117
127
 
128
+ /**
129
+ * A call an adapter could not complete, answered in place of its value. `retry`: a later attempt may work, and the
130
+ * engine tries again within the provider timeout. `now`: no attempt will, and the engine raises one card and stops.
131
+ */
132
+ export const Failed = z.strictObject({ failed: z.enum(["retry", "now"]), why: text });
133
+
134
+ export type Failed = z.infer<typeof Failed>;
135
+
136
+ /** An integration as an adapter implements it: every call may answer `Failed` in place of its value. */
137
+ export type Fallible<I> = { readonly [K in keyof I]: I[K] extends (...args: infer A) => Promise<infer R> ? (...args: A) => Promise<R | Failed> : I[K] };
138
+
118
139
  /** An adapter declares the API version it implements and the integrations it provides. */
119
140
  export type Adapter = {
141
+ readonly apiVersion: number;
142
+ readonly workSource?: Fallible<WorkSource>;
143
+ readonly questions?: Fallible<Questions>;
144
+ readonly answers?: Fallible<Answers>;
145
+ readonly release?: Fallible<Release>;
146
+ };
147
+
148
+ /** The integrations with plain replies: the engine calls them so behind the guard, which retries or throws on `Failed`, and an adapter that never answers `Failed` may implement them so. */
149
+ export type Integrations = {
120
150
  readonly apiVersion: number;
121
151
  readonly workSource?: WorkSource;
122
152
  readonly questions?: Questions;
@@ -128,6 +158,8 @@ export type AdapterContext = {
128
158
  /** Raise a fault card, scoped to one item or provider. */
129
159
  readonly report: (fault: FaultInput) => void;
130
160
  readonly now: () => number;
161
+ /** The process environment, for the tokens and addresses an adapter reads. */
162
+ readonly env: Env;
131
163
  };
132
164
 
133
165
  /** The default export of an owner's `adapters/<name>.ts`. */
@@ -1,6 +1,6 @@
1
1
  import { existsSync, readFileSync } from "node:fs";
2
2
  import { extname, relative } from "node:path";
3
- import type { Adapter } from "../adapter-api.ts";
3
+ import type { Adapter, Integrations } from "../adapter-api.ts";
4
4
  import type { Faults } from "../state/faults.ts";
5
5
  import type { Card } from "../status/format.ts";
6
6
  import { type GuardOptions, guarded } from "./guard.ts";
@@ -47,7 +47,7 @@ export const engineImporters = (files: readonly string[]): { file: string; modul
47
47
  export const sourceFiles = (dir: string): string[] =>
48
48
  existsSync(dir) ? [...SOURCE_GLOB.scanSync({ cwd: dir, absolute: true })].filter(isOutsideDependencies).sort() : [];
49
49
 
50
- export type Admission = { readonly ok: true; readonly adapter: Adapter } | { readonly ok: false; readonly cards: Card[] };
50
+ export type Admission = { readonly ok: true; readonly adapter: Integrations } | { readonly ok: false; readonly cards: Card[] };
51
51
 
52
52
  export type AdmitInput = {
53
53
  readonly name: string;
@@ -2,7 +2,7 @@ import { z } from "zod";
2
2
  import { PermanentError } from "../adapter-api.ts";
3
3
 
4
4
  /** A caller's argument, parsed. One that fails is the caller's mistake, which no retry fixes. */
5
- export const parseArgument = <Out, In>(schema: z.ZodType<Out, In>, value: In): Out => {
5
+ export const parseArgument = <S extends z.ZodType>(schema: S, value: z.input<S>): z.output<S> => {
6
6
  const parsed = schema.safeParse(value);
7
7
 
8
8
  if (!parsed.success) throw new PermanentError(z.prettifyError(parsed.error));
@@ -1,23 +1,21 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { z } from "zod";
3
- import { type Claim, type ClaimResult, PermanentError } from "../adapter-api.ts";
3
+ import { type Claim, type ClaimResult, Holder, type ItemId, PermanentError } from "../adapter-api.ts";
4
4
  import { parseArgument } from "./argument.ts";
5
5
 
6
- const Holder = z.string().regex(/^[A-Za-z0-9][\w.@:-]{0,99}$/, "a holder is a plain name: letters, digits and . _ @ : -");
7
-
8
6
  const Ttl = z.int().positive("a claim lasts a positive whole number of milliseconds");
9
7
 
10
8
  const CLAIM_SUBJECT = /^hum-claim (\S+) (\d+)$/;
11
9
 
12
10
  export type ClaimMode = "claim" | "renew";
13
11
 
14
- /** A provider's claims. `S` is the standing claim with whatever the provider needs to write over it. */
15
- export type ClaimStore<S extends { readonly claim: Claim }> = {
16
- readonly standing: (id: string) => Promise<S | undefined>;
12
+ /** A provider's claims. `S` is the standing claim with whatever the provider needs to write over it; `F` is what its write answers when it fails. */
13
+ export type ClaimStore<S extends { readonly claim: Claim }, F = never> = {
14
+ readonly standing: (id: ItemId) => Promise<S | undefined>;
17
15
  /** Asked only before a first claim: a standing one already proves the item was taken while ready. */
18
- readonly isReady: (id: string) => Promise<boolean>;
19
- /** Writes `claim` over `over`; false when another writer moved first. */
20
- readonly write: (claim: Claim, over: S | undefined) => Promise<boolean>;
16
+ readonly isReady: (id: ItemId) => Promise<boolean>;
17
+ /** Writes `claim` over `over`; false when another writer moved first, `F` when the write failed. */
18
+ readonly write: (claim: Claim, over: S | undefined) => Promise<boolean | NoInfer<F>>;
21
19
  };
22
20
 
23
21
  export type ClaimDecision =
@@ -34,7 +32,7 @@ const isHeldByAnother = (standing: Claim, holder: string, now: number): boolean
34
32
  * Who may write a claim, given the one standing. An unclaimed or expired item goes to whoever writes first;
35
33
  * the holder claiming again renews; a renewal by anyone else loses to the standing claim.
36
34
  */
37
- export const decideClaim = (mode: ClaimMode, holder: string, standing: Claim | undefined, now: number): ClaimDecision => {
35
+ const decideClaim = (mode: ClaimMode, holder: string, standing: Claim | undefined, now: number): ClaimDecision => {
38
36
  if (standing && isHeldByAnother(standing, holder, now)) return { kind: "held", claim: standing };
39
37
 
40
38
  if (mode === "renew" && standing?.holder !== holder) return standing ? { kind: "held", claim: standing } : { kind: "unheld" };
@@ -44,8 +42,8 @@ export const decideClaim = (mode: ClaimMode, holder: string, standing: Claim | u
44
42
 
45
43
  /** `claim` and `renew` over one provider's store: an atomic take whose race the provider's write settles. */
46
44
  export const claimer =
47
- <S extends { readonly claim: Claim }>(store: ClaimStore<S>, now: () => number) =>
48
- async (mode: ClaimMode, item: string, holder: string, ttlMs: number): Promise<ClaimResult> => {
45
+ <S extends { readonly claim: Claim }, F = never>(store: ClaimStore<S, F>, now: () => number) =>
46
+ async (mode: ClaimMode, item: ItemId, holder: Holder, ttlMs: number): Promise<ClaimResult | F> => {
49
47
  const who = parseArgument(Holder, holder);
50
48
  const ttl = parseArgument(Ttl, ttlMs);
51
49
  const standing = await store.standing(item);
@@ -61,7 +59,11 @@ export const claimer =
61
59
 
62
60
  const claim: Claim = { item, holder: who, expiresAt: now() + ttl };
63
61
 
64
- if (await store.write(claim, standing)) return { won: true, claim };
62
+ const written = await store.write(claim, standing);
63
+
64
+ if (written === true) return { won: true, claim };
65
+
66
+ if (written !== false) return written;
65
67
 
66
68
  const winner = await store.standing(item);
67
69
 
@@ -85,9 +87,10 @@ export const claimCommitMessage = (claim: Claim): string => {
85
87
  };
86
88
 
87
89
  /** The claim a commit message holds, or undefined for any other commit. */
88
- export const claimOfCommit = (item: string, message: string): Claim | undefined => {
90
+ export const claimOfCommit = (item: ItemId, message: string): Claim | undefined => {
89
91
  const [subject = ""] = message.split("\n");
90
92
  const match = CLAIM_SUBJECT.exec(subject);
93
+ const holder = Holder.safeParse(match?.[1]);
91
94
 
92
- return match?.[1] && match[2] ? { item, holder: match[1], expiresAt: Number(match[2]) } : undefined;
95
+ return holder.success && match?.[2] ? { item, holder: holder.data, expiresAt: Number(match[2]) } : undefined;
93
96
  };