@moo.md/hum 0.4.1 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/defaults/hum.json +39 -12
  2. package/npm-shrinkwrap.json +2 -2
  3. package/package.json +2 -1
  4. package/release.json +1 -1
  5. package/schema/hum.schema.json +63 -17
  6. package/scripts/eval.ts +58 -27
  7. package/src/adapters/claude-p.ts +7 -0
  8. package/src/adapters/model-call.ts +8 -4
  9. package/src/adapters/pull-requests.ts +11 -7
  10. package/src/cli/setup.ts +13 -18
  11. package/src/config/json.ts +3 -0
  12. package/src/config/migrate.ts +11 -0
  13. package/src/config/schema.ts +21 -6
  14. package/src/decide/alignment.ts +1 -1
  15. package/src/decide/merge.ts +21 -1
  16. package/src/decide/prose.ts +12 -2
  17. package/src/decide/risk.ts +1 -0
  18. package/src/loop/breaks.ts +17 -0
  19. package/src/loop/notes.ts +2 -2
  20. package/src/loop/pair.ts +2 -2
  21. package/src/loop/pass.ts +6 -3
  22. package/src/loop/records.ts +24 -1
  23. package/src/loop/signals.ts +2 -2
  24. package/src/measure/cases.ts +102 -0
  25. package/src/measure/inputs.ts +15 -1
  26. package/src/measure/judges.ts +52 -0
  27. package/src/measure/measure.ts +22 -3
  28. package/src/measure/rate.ts +9 -6
  29. package/src/measure/remeasure.ts +31 -0
  30. package/src/publish/handoff.ts +1 -1
  31. package/src/service/workflows.ts +4 -2
  32. package/src/setup/write.ts +2 -4
  33. package/src/state/events.ts +201 -19
  34. package/src/state/format.ts +9 -2
  35. package/src/state/item-log.ts +133 -0
  36. package/src/state/items.ts +6 -5
  37. package/src/state/open.ts +7 -2
  38. package/src/state/runs.ts +5 -0
  39. package/src/state/saved.ts +7 -2
  40. package/src/status/format.ts +12 -3
  41. package/src/worker/land.ts +36 -15
  42. package/src/worker/merged.ts +18 -8
  43. package/src/worker/open.ts +5 -1
  44. package/src/worker/prose-run.ts +6 -2
  45. package/src/worker/signature.ts +11 -5
  46. package/src/worker/thin-path.ts +69 -53
  47. package/src/worker/worker.ts +82 -12
  48. package/surface.json +969 -151
package/defaults/hum.json CHANGED
@@ -1,5 +1,5 @@
1
1
  {
2
- "schemaVersion": 2,
2
+ "schemaVersion": 3,
3
3
  "levels": [
4
4
  {
5
5
  "name": "low",
@@ -212,28 +212,55 @@
212
212
  "decisionModel": "claude-haiku-5-5",
213
213
  "modelCallTimeoutSeconds": 120,
214
214
  "minJudgeRate": 0.8,
215
+ "evalRuns": 3,
215
216
  "measuredModels": {
216
217
  "kind": {
217
- "claude-haiku-5-5": 0.58,
218
- "claude-sonnet-5-5": 0.63
218
+ "claude-haiku-5-5": {
219
+ "rate": 0.58,
220
+ "judgeHash": "38d9c55d5a9ecca44ee320fd408cc34140fe32219a9d199f6d7114fced6800ef"
221
+ },
222
+ "claude-sonnet-5-5": {
223
+ "rate": 0.63,
224
+ "judgeHash": "38d9c55d5a9ecca44ee320fd408cc34140fe32219a9d199f6d7114fced6800ef"
225
+ }
219
226
  },
220
227
  "alignment": {
221
- "claude-haiku-5-5": 0.68,
222
- "claude-sonnet-5-5": 0.68
228
+ "claude-haiku-5-5": {
229
+ "rate": 0.68,
230
+ "judgeHash": "dc59bc1941fa786d738dd1d8810a4d0644b8e672dd4239bea340102d2960fbbd"
231
+ },
232
+ "claude-sonnet-5-5": {
233
+ "rate": 0.68,
234
+ "judgeHash": "dc59bc1941fa786d738dd1d8810a4d0644b8e672dd4239bea340102d2960fbbd"
235
+ }
223
236
  },
224
237
  "plan-outcome": {
225
- "claude-haiku-5-5": 1,
226
- "claude-sonnet-5-5": 1
238
+ "claude-haiku-5-5": {
239
+ "rate": 1
240
+ },
241
+ "claude-sonnet-5-5": {
242
+ "rate": 1
243
+ }
227
244
  },
228
245
  "review-verdict": {
229
- "claude-haiku-5-5": 0.92,
230
- "claude-sonnet-5-5": 0.83
246
+ "claude-haiku-5-5": {
247
+ "rate": 0.92
248
+ },
249
+ "claude-sonnet-5-5": {
250
+ "rate": 0.83
251
+ }
231
252
  },
232
253
  "reply-option": {
233
- "claude-haiku-5-5": 0.87
254
+ "claude-haiku-5-5": {
255
+ "rate": 0.87,
256
+ "judgeHash": "75d15bd34fb5907209ae421ff529574447cda9eff461173f2d8ad6771e2ddd47"
257
+ }
234
258
  },
235
259
  "repeat-pair": {
236
- "claude-haiku-5-5": 1
260
+ "claude-haiku-5-5": {
261
+ "rate": 1,
262
+ "judgeHash": "fceeafd3501e0597d2c460ce2ff88021a1746eee91fdc18adc56a334b711d331"
263
+ }
237
264
  },
238
265
  "taste": {}
239
266
  },
@@ -330,4 +357,4 @@
330
357
  "mac",
331
358
  "ci"
332
359
  ]
333
- }
360
+ }
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "@moo.md/hum",
3
- "version": "0.4.1",
3
+ "version": "0.6.1",
4
4
  "lockfileVersion": 3,
5
5
  "requires": true,
6
6
  "packages": {
7
7
  "": {
8
8
  "name": "@moo.md/hum",
9
- "version": "0.4.1",
9
+ "version": "0.6.1",
10
10
  "license": "MIT",
11
11
  "dependencies": {
12
12
  "@actions/core": "3.0.1",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@moo.md/hum",
3
- "version": "0.4.1",
3
+ "version": "0.6.1",
4
4
  "description": "The hum engine: work items become Claude Code agent runs, the repo's checks decide what they can, and only what nothing settles becomes a question to the owner.",
5
5
  "license": "MIT",
6
6
  "author": {
@@ -87,6 +87,7 @@
87
87
  "@types/proper-lockfile": "4.1.4",
88
88
  "@types/semver": "7.8.0",
89
89
  "@types/write-file-atomic": "4.0.3",
90
+ "graphql": "17.1.0",
90
91
  "microdiff": "1.6.0",
91
92
  "oxlint": "1.89.0",
92
93
  "typescript": "5.9.2"
package/release.json CHANGED
@@ -1,3 +1,3 @@
1
1
  {
2
- "commit": "37df02ce6f86e819eb09c929597eabe8e04ee65e"
2
+ "commit": "e5c5a183dfaa8fb645a5e9092a8fc5f5ca6bb188"
3
3
  }
@@ -4,7 +4,7 @@
4
4
  "properties": {
5
5
  "schemaVersion": {
6
6
  "type": "number",
7
- "const": 2
7
+ "const": 3
8
8
  },
9
9
  "levels": {
10
10
  "minItems": 1,
@@ -375,7 +375,7 @@
375
375
  "type": "integer",
376
376
  "minimum": 0,
377
377
  "maximum": 9007199254740991,
378
- "description": "Fix runs an item gets after failed checks before it ends stuck."
378
+ "description": "Fix runs an item gets after failed checks, counted since the owner last answered, before it ends stuck."
379
379
  },
380
380
  "serviceLabel": {
381
381
  "default": "md.moo.hum",
@@ -1084,26 +1084,52 @@
1084
1084
  "measuredModels": {
1085
1085
  "default": {
1086
1086
  "kind": {
1087
- "claude-haiku-5-5": 0.58,
1088
- "claude-sonnet-5-5": 0.63
1087
+ "claude-haiku-5-5": {
1088
+ "rate": 0.58,
1089
+ "judgeHash": "38d9c55d5a9ecca44ee320fd408cc34140fe32219a9d199f6d7114fced6800ef"
1090
+ },
1091
+ "claude-sonnet-5-5": {
1092
+ "rate": 0.63,
1093
+ "judgeHash": "38d9c55d5a9ecca44ee320fd408cc34140fe32219a9d199f6d7114fced6800ef"
1094
+ }
1089
1095
  },
1090
1096
  "alignment": {
1091
- "claude-haiku-5-5": 0.68,
1092
- "claude-sonnet-5-5": 0.68
1097
+ "claude-haiku-5-5": {
1098
+ "rate": 0.68,
1099
+ "judgeHash": "dc59bc1941fa786d738dd1d8810a4d0644b8e672dd4239bea340102d2960fbbd"
1100
+ },
1101
+ "claude-sonnet-5-5": {
1102
+ "rate": 0.68,
1103
+ "judgeHash": "dc59bc1941fa786d738dd1d8810a4d0644b8e672dd4239bea340102d2960fbbd"
1104
+ }
1093
1105
  },
1094
1106
  "plan-outcome": {
1095
- "claude-haiku-5-5": 1,
1096
- "claude-sonnet-5-5": 1
1107
+ "claude-haiku-5-5": {
1108
+ "rate": 1
1109
+ },
1110
+ "claude-sonnet-5-5": {
1111
+ "rate": 1
1112
+ }
1097
1113
  },
1098
1114
  "review-verdict": {
1099
- "claude-haiku-5-5": 0.92,
1100
- "claude-sonnet-5-5": 0.83
1115
+ "claude-haiku-5-5": {
1116
+ "rate": 0.92
1117
+ },
1118
+ "claude-sonnet-5-5": {
1119
+ "rate": 0.83
1120
+ }
1101
1121
  },
1102
1122
  "reply-option": {
1103
- "claude-haiku-5-5": 0.87
1123
+ "claude-haiku-5-5": {
1124
+ "rate": 0.87,
1125
+ "judgeHash": "75d15bd34fb5907209ae421ff529574447cda9eff461173f2d8ad6771e2ddd47"
1126
+ }
1104
1127
  },
1105
1128
  "repeat-pair": {
1106
- "claude-haiku-5-5": 1
1129
+ "claude-haiku-5-5": {
1130
+ "rate": 1,
1131
+ "judgeHash": "fceeafd3501e0597d2c460ce2ff88021a1746eee91fdc18adc56a334b711d331"
1132
+ }
1107
1133
  },
1108
1134
  "taste": {}
1109
1135
  },
@@ -1127,9 +1153,22 @@
1127
1153
  "minLength": 1
1128
1154
  },
1129
1155
  "additionalProperties": {
1130
- "type": "number",
1131
- "minimum": 0,
1132
- "maximum": 1
1156
+ "type": "object",
1157
+ "properties": {
1158
+ "rate": {
1159
+ "type": "number",
1160
+ "minimum": 0,
1161
+ "maximum": 1
1162
+ },
1163
+ "judgeHash": {
1164
+ "type": "string",
1165
+ "pattern": "^[0-9a-f]{64}$"
1166
+ }
1167
+ },
1168
+ "required": [
1169
+ "rate"
1170
+ ],
1171
+ "additionalProperties": false
1133
1172
  }
1134
1173
  },
1135
1174
  "required": [
@@ -1141,7 +1180,7 @@
1141
1180
  "repeat-pair",
1142
1181
  "taste"
1143
1182
  ],
1144
- "description": "Per model-judgment boundary, the rate each model id scored on its labelled set; a pick from an id below 'minJudgeRate', or not measured, counts as an abstention."
1183
+ "description": "Per model-judgment boundary, each model id's rate on its labelled set, and the judgeHash of the prompt, role text and answer form its judge read then; a pick from an id below 'minJudgeRate', or not measured, counts as an abstention. A rate whose judge has changed since keeps gating until a new measurement replaces it: the Mac worker measures it again."
1145
1184
  },
1146
1185
  "minJudgeRate": {
1147
1186
  "default": 0.8,
@@ -1150,6 +1189,13 @@
1150
1189
  "maximum": 1,
1151
1190
  "description": "The lowest measured rate at which hum trusts a model's pick at a boundary; below it the call abstains."
1152
1191
  },
1192
+ "evalRuns": {
1193
+ "default": 3,
1194
+ "type": "integer",
1195
+ "exclusiveMinimum": 0,
1196
+ "maximum": 9007199254740991,
1197
+ "description": "Runs pooled per boundary each time hum measures a model: in `hum setup measure`, and when a judge's prompt or role text changed since its rate was measured."
1198
+ },
1153
1199
  "questionLimits": {
1154
1200
  "default": {
1155
1201
  "cells": 80,
@@ -1485,7 +1531,7 @@
1485
1531
  "type": "integer",
1486
1532
  "exclusiveMinimum": 0,
1487
1533
  "maximum": 9007199254740991,
1488
- "description": "The time between passes that collect review notes, file check items and confirm CI ran their checks."
1534
+ "description": "The time between passes that collect review notes, file check items and confirm CI ran their checks, and between tries to measure again a judge whose prompt or role text changed."
1489
1535
  },
1490
1536
  "proofTimeoutSeconds": {
1491
1537
  "default": 900,
package/scripts/eval.ts CHANGED
@@ -8,6 +8,9 @@ import { z } from "zod";
8
8
  import { type Finished, type Runner, startRunner } from "../src/adapters/claude-p.ts";
9
9
  import { gitIn, readGitIdentity } from "../src/adapters/git.ts";
10
10
  import { modelCallOver } from "../src/adapters/model-call.ts";
11
+ import { loadConfig, readConfigDir } from "../src/config/load.ts";
12
+ import { migrateConfig } from "../src/config/migrate.ts";
13
+ import type { Result } from "../src/config/refusal.ts";
11
14
  import { BOUNDARIES, type Boundary } from "../src/config/schema.ts";
12
15
  import { alignmentQuestion, doesRaiseLevel } from "../src/decide/alignment.ts";
13
16
  import { replyQuestion } from "../src/decide/answer.ts";
@@ -19,35 +22,52 @@ import { pairing, prepare } from "../src/loop/signals.ts";
19
22
  import type { JsonSchema, ProsePurpose } from "../src/decide/prose.ts";
20
23
  import { SHIPPED_COMMAND_TIMEOUT_MS } from "../src/exec.ts";
21
24
  import { shippedRaw, shippedSettingsAs } from "../src/setup/shipped.ts";
22
- import { openState } from "../src/state/open.ts";
25
+ import { defaultStateDir, openState } from "../src/state/open.ts";
23
26
  import { decideProse } from "../src/worker/prose-run.ts";
24
- import { PROSE_PROMPTS } from "../src/worker/thin-path.ts";
27
+ import { PROSE_PROMPTS, type RunConfig } from "../src/worker/thin-path.ts";
25
28
  import { SECOND_MS } from "../src/time.ts";
26
29
  import { rateOf, withRate } from "../src/measure/rate.ts";
30
+ import { judgeHash, SHIPPED_SETS } from "../src/measure/judges.ts";
31
+ import { EVALS_DIR } from "../src/measure/remeasure.ts";
32
+ import { AlignmentCase, MERGED_CASES, ReviewCase } from "../src/measure/cases.ts";
27
33
 
28
34
  /**
29
35
  * Measures one model-judgment boundary on its labelled cases, through the same prompts and fences the engine runs,
30
36
  * and adds a row to its RESULTS.md. `bun run eval -- --model <id>` measures every boundary on that model id; naming
31
37
  * boundaries measures only those. A case the runner held or limited counts in no rate. `--runs <n>` repeats each
32
- * boundary and pools the runs, since one run proves little; `--write <folder>` records each pooled rate under
33
- * `measuredModels` in that folder's `hum.json`, which is how setup measures an owner's own models. A boundary where
34
- * nothing was measured records no rate, and the script exits non-zero.
38
+ * boundary and pools the runs, since one run proves little. `--write <folder>` measures the judges as the config in
39
+ * that `.hum/`-shaped folder has them, its role files and kind meanings with them, and records each pooled rate with
40
+ * its judge's hash under `measuredModels` in the folder's `hum.json`, in one write once every boundary is done: that
41
+ * is how setup, and the Mac worker when a judge changed, measure an owner's own models. The alignment and review sets
42
+ * then take the cases the repo's merged items gave them too. Without it the judges and sets are the shipped ones. A
43
+ * boundary where nothing was measured records no rate, and the script exits non-zero.
35
44
  */
36
45
 
37
46
  const { values, positionals } = parseArgs({ allowPositionals: true, options: { model: { type: "string" }, limit: { type: "string" }, runs: { type: "string" }, write: { type: "string" } } });
38
- /** Boundaries whose labelled cases only the owner can give; they are measured when named, from the cases the owner puts in their evals folder. */
39
- const OWNER_LABELLED: readonly Boundary[] = ["taste"];
40
47
 
41
- const boundaries: readonly Boundary[] = positionals.length > 0 ? z.array(z.enum(BOUNDARIES)).parse(positionals) : BOUNDARIES.filter((boundary) => !OWNER_LABELLED.includes(boundary));
48
+ const boundaries: readonly Boundary[] = positionals.length > 0 ? z.array(z.enum(BOUNDARIES)).parse(positionals) : SHIPPED_SETS;
42
49
  const root = join(import.meta.dir, "..");
43
- const settings = shippedSettingsAs(await readGitIdentity(gitIn(process.cwd(), { timeoutMs: SHIPPED_COMMAND_TIMEOUT_MS })));
44
50
 
45
- if (!settings.ok) {
46
- console.error(settings.refusal.message);
51
+ /** The judges as an owner's `.hum/` folder has them, or as hum ships them, to the owner's git identity. */
52
+ const judgesConfig = async (folder: string | undefined): Promise<Result<RunConfig>> => {
53
+ if (folder !== undefined) return loadConfig(readConfigDir(folder), { env: {} });
54
+
55
+ const settings = shippedSettingsAs(await readGitIdentity(gitIn(process.cwd(), { timeoutMs: SHIPPED_COMMAND_TIMEOUT_MS })));
56
+
57
+ return settings.ok ? { ok: true, value: { hum: settings.value, roles: shippedRaw().roles } } : settings;
58
+ };
59
+
60
+ const loaded = await judgesConfig(values.write);
61
+
62
+ if (!loaded.ok) {
63
+ console.error(loaded.refusal.message);
47
64
  process.exit(1);
48
65
  }
49
66
 
50
- const config = { hum: settings.value, roles: shippedRaw().roles };
67
+ const config = loaded.value;
68
+
69
+ /** The cases merged items gave the judges, kept in the state of the repo whose `.hum/` is measured; the shipped judges take none. */
70
+ const merged = values.write === undefined ? [] : openState(await defaultStateDir(dirname(values.write)), { now: Date.now }).cases.list();
51
71
  const model = values.model ?? config.hum.decisionModel;
52
72
  const Count = z.coerce.number().int().positive();
53
73
  const runs = values.runs === undefined ? 1 : Count.parse(values.runs);
@@ -124,10 +144,8 @@ const planRepo = (): string => {
124
144
  };
125
145
 
126
146
  const KindCase = z.object({ case: z.string(), label: z.string(), title: z.string(), body: z.string() });
127
- const AlignmentCase = z.object({ case: z.string(), label: z.enum(["reverted", "fixed", "clean"]), subject: z.string(), diff: z.string() });
128
147
  const PlanCase = z.object({ case: z.string(), label: z.string(), body: z.string() });
129
148
  const ReplyCase = z.object({ case: z.string(), label: z.string(), question: z.string(), options: z.array(z.string()), reply: z.string() });
130
- const ReviewCase = z.object({ case: z.string(), label: z.string(), item: z.string(), diff: z.string() });
131
149
  const PairCase = z.object({ case: z.string(), label: z.enum(["same", "different"]), a: z.object({ text: z.string() }), b: z.object({ text: z.string() }) });
132
150
  const TasteCase = z.object({ case: z.string(), label: z.enum(["keeps", "breaks"]), rule: z.string(), diff: z.string() });
133
151
 
@@ -137,7 +155,7 @@ const AT_ONCE = 4;
137
155
  const eachOf = <T, R>(cases: readonly T[], measure: (found: T) => Promise<R>): Promise<R[]> => new PQueue({ concurrency: AT_ONCE }).addAll(cases.map((found) => () => measure(found)));
138
156
 
139
157
  const read = async <T>(boundary: Boundary, schema: z.ZodType<T>, file = "cases.jsonl"): Promise<T[]> =>
140
- (await Bun.file(join(root, "evals", boundary, file)).text())
158
+ (await Bun.file(join(root, EVALS_DIR, boundary, file)).text())
141
159
  .split("\n")
142
160
  .filter(Boolean)
143
161
  .map((line) => schema.parse(JSON.parse(line)));
@@ -169,11 +187,11 @@ const BOUNDARY_RUNS: Record<Boundary, () => Promise<Measured>> = {
169
187
  };
170
188
  },
171
189
  alignment: async () => {
172
- const sample = (await read("alignment", AlignmentCase, "sample.jsonl")).slice(0, limit);
190
+ const sample = [...(await read("alignment", AlignmentCase, "sample.jsonl")), ...MERGED_CASES.alignment(merged).map((found) => AlignmentCase.parse(found))].slice(0, limit);
173
191
 
174
192
  return {
175
193
  verdicts: await eachOf(sample, async (found): Promise<Verdict> => {
176
- const judged = await judge(alignmentQuestion({ title: found.subject, body: "" }, found.diff, config.hum));
194
+ const judged = await judge(alignmentQuestion({ title: found.subject, body: found.body }, found.diff, config.hum));
177
195
 
178
196
  if (judged.kind === "abstained") return "abstained";
179
197
 
@@ -232,9 +250,9 @@ const BOUNDARY_RUNS: Record<Boundary, () => Promise<Measured>> = {
232
250
  };
233
251
  },
234
252
  "review-verdict": async () => {
235
- const cases = (await read("review-verdict", ReviewCase)).slice(0, limit);
253
+ const cases = [...(await read("review-verdict", ReviewCase)), ...MERGED_CASES["review-verdict"](merged).map((found) => ReviewCase.parse(found))].slice(0, limit);
236
254
 
237
- return { verdicts: await eachOf(cases, async (found) => scored(await prosePick("review", PROSE_PROMPTS.review(config, { id: found.case, title: found.item, body: "" }, found.diff), found.diff, tmpdir()), found.label)) };
255
+ return { verdicts: await eachOf(cases, async (found) => scored(await prosePick("review", PROSE_PROMPTS.review(config, { id: found.case, title: found.item, body: found.body }, found.diff), found.diff, tmpdir()), found.label)) };
238
256
  },
239
257
  };
240
258
 
@@ -250,15 +268,26 @@ const precisionOf = ({ tp, fp }: PairCounts): number | undefined => rateOf(tp, t
250
268
  const pairRow = (day: string, pairs: PairCounts, abstained: number, used: number): string =>
251
269
  `| ${day} | ${model} | ${pairs.tp + pairs.fp + pairs.fn + pairs.tn} | ${pairs.tp} | ${pairs.fp} | ${pairs.fn} | ${pairs.tn} | ${abstained} | ${pairs.pairedBySignals} | ${rateCell(precisionOf(pairs))} | ${rateCell(rateOf(pairs.tp, pairs.tp + pairs.fn))} | ${used} |`;
252
270
 
253
- /** Sets one pooled rate in the owner's `hum.json`, where trust reads it, under the model as named and each id that answered for it. */
254
- const recordRate = async (folder: string, boundary: Boundary, rate: number): Promise<void> => {
271
+ /** One boundary's pooled rate, and the model ids that answered while it was measured. */
272
+ type Pooled = { readonly boundary: Boundary; readonly rate: number; readonly answered: readonly string[] };
273
+
274
+ /**
275
+ * Sets each pooled rate, beside its judge's hash, in the owner's `hum.json` at the current schema, where trust reads
276
+ * it, under the model as named and each id that answered for it; every other rate stays as the config had it.
277
+ */
278
+ const recordRates = async (folder: string, pooled: readonly Pooled[]): Promise<void> => {
255
279
  const file = join(folder, "hum.json");
256
- const written = z.looseObject({ measuredModels: z.partialRecord(z.enum(BOUNDARIES), z.record(z.string(), z.number())).default({}) }).parse(JSON.parse(await Bun.file(file).text()));
280
+ const migrated = migrateConfig(JSON.parse(await Bun.file(file).text()));
257
281
 
258
- await Bun.write(file, `${JSON.stringify({ ...written, measuredModels: withRate(written.measuredModels, { boundary, model, answered: [...answered], rate }) }, null, "\t")}\n`);
282
+ if (!migrated.ok) throw new Error(migrated.refusal.message);
283
+
284
+ const measuredModels = pooled.reduce((measured, { boundary, rate, answered }) => withRate(measured, { boundary, model, answered, rate, judgeHash: judgeHash(boundary, config) }), config.hum.measuredModels);
285
+
286
+ await Bun.write(file, `${JSON.stringify({ ...migrated.value, measuredModels }, null, "\t")}\n`);
259
287
  };
260
288
 
261
289
  const unmeasured: Boundary[] = [];
290
+ const recorded: Pooled[] = [];
262
291
 
263
292
  for (const boundary of boundaries) {
264
293
  const pool = { right: 0, measured: 0, pairs: NO_PAIRS };
@@ -272,7 +301,7 @@ for (const boundary of boundaries) {
272
301
  const measured = verdicts.length - count("halted");
273
302
  const day = new Date().toISOString().slice(0, 10);
274
303
  const row = boundary === "repeat-pair" ? pairRow(day, pairs, count("abstained"), calls - before) : `| ${day} | ${model} | ${measured} | ${count("right")} | ${count("wrong")} | ${count("abstained")} | ${rateCell(rateOf(count("right"), measured))} | ${calls - before} |`;
275
- const results = join(values.write ?? root, "evals", boundary, "RESULTS.md");
304
+ const results = join(values.write ?? root, EVALS_DIR, boundary, "RESULTS.md");
276
305
  const earlier = (await Bun.file(results).exists()) ? await Bun.file(results).text() : boundary === "repeat-pair" ? pairHeader : header;
277
306
 
278
307
  pool.right += count("right");
@@ -284,15 +313,17 @@ for (const boundary of boundaries) {
284
313
 
285
314
  const pooled = boundary === "repeat-pair" ? precisionOf(pool.pairs) : rateOf(pool.right, pool.measured);
286
315
 
287
- console.log(`${boundary}: pooled over ${runs} run(s): ${rateCell(pooled)}`);
316
+ console.log(`${boundary}: pooled over ${runs} run(s): ${rateCell(pooled)}, judge ${judgeHash(boundary, config)}`);
288
317
 
289
318
  if (pooled === undefined) unmeasured.push(boundary);
290
- else if (values.write !== undefined) await recordRate(values.write, boundary, pooled);
319
+ else recorded.push({ boundary, rate: pooled, answered: [...answered] });
291
320
  }
292
321
 
322
+ if (values.write !== undefined && recorded.length > 0) await recordRates(values.write, recorded);
323
+
293
324
  if (unmeasured.length > 0) {
294
325
  console.error(`Nothing was measured on ${unmeasured.join(", ")}, so no rate was recorded for ${unmeasured.length === 1 ? "it" : "them"}.`);
295
326
  process.exitCode = 1;
296
327
  }
297
328
 
298
- console.log(`${calls} live calls, the start check included. Record each rate under measuredModels in defaults/hum.json; a model decides a boundary alone only at minJudgeRate or above; repeat-pair takes its precision, against repeatPrecision.`);
329
+ console.log(`${calls} live calls, the start check included. Record each rate as { rate, judgeHash } under measuredModels in defaults/hum.json; a model decides a boundary alone only at minJudgeRate or above; repeat-pair takes its precision, against repeatPrecision.`);
@@ -228,6 +228,13 @@ type Ended = RunOutcome | LimitEnded;
228
228
 
229
229
  export const hasSucceeded = (outcome: Finished): boolean => outcome.subtype === "success" && !outcome.isError;
230
230
 
231
+ /** A run that ended with no answer and no token to count: a usage limit, a hold, or a stop. */
232
+ export type Paused = Extract<RunOutcome, { kind: "limited" | "held" | "stopped" }>;
233
+
234
+ const PAUSED_KINDS: readonly string[] = ["limited", "held", "stopped"] satisfies readonly Paused["kind"][];
235
+
236
+ export const isPaused = (outcome: RunOutcome): outcome is Paused => PAUSED_KINDS.includes(outcome.kind);
237
+
231
238
  const LOCK_TEXT = /refresh[^\n]*lock|lock[^\n]*refresh/i;
232
239
 
233
240
  const isLimitRefusal = (t: Transcript, result: Result): boolean => result.is_error && (t.rejectedUntil !== undefined || LIMIT_TEXT.test(result.result));
@@ -1,7 +1,7 @@
1
1
  import { tmpdir } from "node:os";
2
2
  import { z } from "zod";
3
3
  import type { Boundary, HumConfig } from "../config/schema.ts";
4
- import { isTrusted, jsonSchemaOf } from "../decide/prose.ts";
4
+ import { isTrusted, jsonSchemaOf, type ModelAsk } from "../decide/prose.ts";
5
5
  import { SECOND_MS } from "../time.ts";
6
6
  import { type RunOutcome, type Runner, hasSucceeded } from "./claude-p.ts";
7
7
 
@@ -38,16 +38,20 @@ export type ModelCallOptions = {
38
38
  readonly onSpent?: (item: string, tokens: number) => void;
39
39
  };
40
40
 
41
+ /** What a model is sent for a question: its prompt, and the schema its answer must match. */
42
+ export const askOf = ({ prompt, answer }: Pick<Question<unknown>, "prompt" | "answer">): ModelAsk => ({ prompt, schema: jsonSchemaOf(answer) });
43
+
41
44
  /** The cheap model call through `claude -p`, as hope's judge calls it, with a typed answer by JSON schema. */
42
45
  export const modelCallOver =
43
46
  (runner: Runner, { model, timeoutMs, trusts, onSpent }: ModelCallOptions): ModelCall =>
44
- async ({ boundary, prompt, answer, item, reportOnly }) => {
47
+ async (question) => {
48
+ const { boundary, answer, item, reportOnly } = question;
49
+
45
50
  const outcome = await runner.run({
46
- prompt,
51
+ ...askOf(question),
47
52
  cwd: tmpdir(),
48
53
  model,
49
54
  effort: "low",
50
- schema: jsonSchemaOf(answer),
51
55
  call: "judgment",
52
56
  signal: AbortSignal.timeout(timeoutMs),
53
57
  });
@@ -18,11 +18,10 @@ export type Pull = {
18
18
  readonly id: string;
19
19
  readonly number: number;
20
20
  readonly url: string;
21
- readonly state: "OPEN" | "MERGED" | "CLOSED";
22
21
  readonly autoMerge: boolean;
23
22
  /** Its branch is behind the base, which a strict required check makes wait. */
24
23
  readonly behind: boolean;
25
- };
24
+ } & ({ readonly state: "OPEN" | "CLOSED" } | { readonly state: "MERGED"; /** The commit its merge made on the base. */ readonly mergeCommit: string });
26
25
 
27
26
  /** Pull requests on the repo a run's changes land in. */
28
27
  export type PullRequests = {
@@ -62,17 +61,22 @@ const PULL_REQUEST_RULE = "pull_request";
62
61
 
63
62
  const SIGNATURES_RULE = "required_signatures";
64
63
 
65
- const PullNode = z.object({
64
+ const pullFields = {
66
65
  id: z.string(),
67
66
  number: z.int(),
68
67
  url: z.string(),
69
- state: z.enum(["OPEN", "MERGED", "CLOSED"]),
70
68
  mergeStateStatus: z.string(),
71
69
  autoMergeRequest: z.object({ enabledAt: z.string().nullable() }).nullable(),
72
- });
70
+ };
71
+
72
+ /** GitHub names the commit a merge made once the pull request merged, and none before. */
73
+ const PullNode = z.discriminatedUnion("state", [
74
+ z.object({ ...pullFields, state: z.enum(["OPEN", "CLOSED"]) }),
75
+ z.object({ ...pullFields, state: z.literal("MERGED"), mergeCommit: z.object({ oid: z.string().min(1) }) }),
76
+ ]);
73
77
 
74
78
  const PULL_FIELDS =
75
- "id number url state mergeStateStatus autoMergeRequest { enabledAt }";
79
+ "id number url state mergeStateStatus autoMergeRequest { enabledAt } mergeCommit { oid }";
76
80
 
77
81
  const BEHIND = "BEHIND";
78
82
 
@@ -125,9 +129,9 @@ const pullOf = (node: z.infer<typeof PullNode>): Pull => ({
125
129
  id: node.id,
126
130
  number: node.number,
127
131
  url: node.url,
128
- state: node.state,
129
132
  autoMerge: node.autoMergeRequest !== null,
130
133
  behind: node.mergeStateStatus === BEHIND,
134
+ ...(node.state === "MERGED" ? { state: node.state, mergeCommit: node.mergeCommit.oid } : { state: node.state }),
131
135
  });
132
136
 
133
137
  /** The method the owner set when the branch's rules allow it, else the first they allow. */
package/src/cli/setup.ts CHANGED
@@ -1,11 +1,11 @@
1
1
  import { randomUUID } from "node:crypto";
2
- import { join } from "node:path";
3
2
  import type { Command } from "commander";
4
- import { x } from "tinyexec";
5
3
  import { z } from "zod";
6
4
  import type { Item } from "../adapter-api.ts";
7
5
  import { gitIn, readGitIdentity } from "../adapters/git.ts";
8
- import { HumConfig } from "../config/schema.ts";
6
+ import { HumConfig, SCHEMA_VERSION } from "../config/schema.ts";
7
+ import { runModels } from "../measure/judges.ts";
8
+ import { measureOn } from "../measure/remeasure.ts";
9
9
  import { CommandTimedOut, exec, SHIPPED_COMMAND_TIMEOUT_MS } from "../exec.ts";
10
10
  import { readSettings } from "../config/settings.ts";
11
11
  import { kindCoverage, proposeSetup } from "../setup/propose.ts";
@@ -28,8 +28,6 @@ const GhIssues = z.array(z.object({ body: z.string().default(""), labels: z.arra
28
28
 
29
29
  const GhLabels = z.array(z.object({ name: z.string() }));
30
30
 
31
- const EVAL_SCRIPT = join(import.meta.dir, "..", "..", "scripts", "eval.ts");
32
-
33
31
  /** What the scan reads from GitHub. Each fact is read alone, and one whose read fails is unknown while the rest stand. */
34
32
  type GithubFacts = {
35
33
  readonly items: readonly Pick<Item, "body" | "labels">[] | undefined;
@@ -80,14 +78,9 @@ const githubFacts = async (repo: string): Promise<GithubFacts> => ({
80
78
 
81
79
  const printStep = ({ name, status, detail, ms }: Step): string => `${status.padEnd(7)} ${name} (${ms} ms): ${detail}`;
82
80
 
83
- const ModelSettings = z.looseObject({ models: HumConfig.shape.models, decisionModel: HumConfig.shape.decisionModel });
84
-
85
- /** The distinct model ids the owner's tiers and cheap call run on. */
86
- const modelsOf = (repo: string): string[] => {
87
- const { models, decisionModel } = readSettings(repo, ModelSettings);
81
+ const MeasureSettings = z.looseObject({ models: HumConfig.shape.models, decisionModel: HumConfig.shape.decisionModel, evalRuns: HumConfig.shape.evalRuns });
88
82
 
89
- return [...new Set([...Object.values(models), decisionModel])];
90
- };
83
+ const Runs = z.coerce.number().int().positive();
91
84
 
92
85
  /** `hum setup scan | write | prove | measure`: the facts and writes behind the /hum skill. */
93
86
  export const addSetupCommands = (root: Command, io: Io, done: (code: number) => void): void => {
@@ -113,7 +106,7 @@ export const addSetupCommands = (root: Command, io: Io, done: (code: number) =>
113
106
  setup
114
107
  .command("write")
115
108
  .description("Write .hum/ and the service files from the answers file, or write nothing and say why.")
116
- .requiredOption("--answers <file>", 'JSON: { "hum": {settings}, "waitHours": { "kind/state": hours }, "schemaVersion": 2 }')
109
+ .requiredOption("--answers <file>", `JSON: { "hum": {settings}, "waitHours": { "kind/state": hours }, "schemaVersion": ${SCHEMA_VERSION} }`)
117
110
  .option("--home <folder>", "where the service unit goes; default the user's home")
118
111
  .action(async ({ answers, home }: { answers: string; home?: string }) => {
119
112
  const result = writeSetup(io.cwd, readAnswers(answers), { installedSkills: await installedSkills(), home, gitIdentity: await readGitIdentity(setupGit(io.cwd)), isPublicRepo: await isPublicRepo(io.cwd) });
@@ -137,13 +130,15 @@ export const addSetupCommands = (root: Command, io: Io, done: (code: number) =>
137
130
  setup
138
131
  .command("measure")
139
132
  .description("Measure the owner's models on every labelled set and record the pooled rates in .hum/hum.json.")
140
- .option("--runs <n>", "runs pooled per boundary", "3")
133
+ .option("--runs <n>", "runs pooled per boundary; default evalRuns in hum.json")
141
134
  .option("--model <id...>", "model ids to measure; default every model the tiers and the cheap call use")
142
- .action(async ({ runs, model }: { runs: string; model?: string[] }) => {
143
- for (const id of model ?? modelsOf(io.cwd)) {
144
- const ran = await x(process.execPath, [EVAL_SCRIPT, "--model", id, "--runs", runs, "--write", join(io.cwd, ".hum")], { throwOnError: false, nodeOptions: { cwd: io.cwd, stdio: "inherit" } });
135
+ .action(async ({ runs, model }: { runs?: string; model?: string[] }) => {
136
+ const settings = readSettings(io.cwd, MeasureSettings);
137
+
138
+ for (const id of model ?? runModels(settings)) {
139
+ const measured = await measureOn(io.cwd, { model: id, runs: runs === undefined ? settings.evalRuns : Runs.parse(runs) }, { inherit: true });
145
140
 
146
- if (ran.exitCode !== 0) return done(EXIT_FAILED);
141
+ if (!measured.ok) return done(EXIT_FAILED);
147
142
  }
148
143
 
149
144
  done(EXIT_OK);
@@ -4,3 +4,6 @@ import { z } from "zod";
4
4
  export const Json = z.json();
5
5
 
6
6
  export type Json = z.infer<typeof Json>;
7
+
8
+ /** A value as the JSON it saves: what `JSON.stringify` writes, checked as Json on the way back. Undefined, which saves nothing, does not type-check. */
9
+ export const asJson = (value: NonNullable<unknown>): Json => Json.parse(JSON.parse(JSON.stringify(value)));
@@ -21,10 +21,21 @@ export const climb = <T extends object>(value: T, from: number, to: number, step
21
21
  /** Per earlier schema version: rewrites a config at that version into the next version's shape. */
22
22
  export type Migrations = Steps<ConfigJson>;
23
23
 
24
+ const RatesAt2 = z.record(z.string(), z.record(z.string(), z.number()));
25
+
26
+ /** A version-2 `measuredModels`, each rate as a score with no judge hash; anything else is left for the schema to refuse. */
27
+ const scoresOf = (measured: Json): Json => {
28
+ const rates = RatesAt2.safeParse(measured);
29
+
30
+ return rates.success ? Object.fromEntries(Object.entries(rates.data).map(([boundary, byModel]) => [boundary, Object.fromEntries(Object.entries(byModel).map(([model, rate]) => [model, { rate }]))])) : measured;
31
+ };
32
+
24
33
  /** One entry per earlier schema version. Setup climbs the owner's answers, a part of a config, through the same steps, so a step rewrites only the keys it finds. */
25
34
  export const MIGRATIONS = {
26
35
  /** 2 holds the pull request token as one of its sources: the secret 1 named becomes `{ from: "secret", secret }`. */
27
36
  1: ({ pullRequestTokenSecret, ...config }) => (pullRequestTokenSecret === undefined ? config : { ...config, pullRequestToken: { from: "secret", secret: pullRequestTokenSecret } }),
37
+ /** 3 keeps each measured rate as `{ rate, judgeHash }`; a rate 2 kept has no hash, so its judge counts as changed. */
38
+ 2: ({ measuredModels, ...config }) => (measuredModels === undefined ? config : { ...config, measuredModels: scoresOf(measuredModels) }),
28
39
  } satisfies Migrations;
29
40
 
30
41
  const Raw = z.record(z.string(), Json);