@moo.md/hum 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/defaults/hum.json +9 -2
  2. package/package.json +2 -2
  3. package/schema/hum.schema.json +208 -7
  4. package/scripts/eval.ts +88 -50
  5. package/src/adapter-api.ts +46 -14
  6. package/src/adapters/admit.ts +2 -2
  7. package/src/adapters/argument.ts +1 -1
  8. package/src/adapters/claim.ts +18 -15
  9. package/src/adapters/claude-bg.ts +50 -27
  10. package/src/adapters/claude-p.ts +18 -23
  11. package/src/adapters/contract.ts +22 -10
  12. package/src/adapters/each-call.ts +9 -0
  13. package/src/adapters/fake.ts +6 -6
  14. package/src/adapters/git.ts +10 -18
  15. package/src/adapters/github-client.ts +102 -0
  16. package/src/adapters/github.ts +61 -146
  17. package/src/adapters/guard.ts +20 -9
  18. package/src/adapters/history.ts +12 -15
  19. package/src/adapters/item-file.ts +1 -1
  20. package/src/adapters/markdown.ts +49 -42
  21. package/src/adapters/marked.ts +6 -12
  22. package/src/adapters/model-call.ts +4 -4
  23. package/src/adapters/once.ts +5 -5
  24. package/src/adapters/owner-commit.ts +13 -11
  25. package/src/adapters/pull-requests.ts +11 -13
  26. package/src/adapters/registry.ts +9 -8
  27. package/src/adapters/roundup.ts +27 -35
  28. package/src/adapters/serial.ts +4 -5
  29. package/src/adapters/transcript.ts +13 -8
  30. package/src/cli/main.ts +1 -0
  31. package/src/cli/ops.ts +32 -14
  32. package/src/cli/service.ts +14 -8
  33. package/src/cli/setup.ts +22 -7
  34. package/src/cli/shared-watch.ts +3 -3
  35. package/src/config/checks.ts +27 -18
  36. package/src/config/machine.ts +5 -7
  37. package/src/config/migrate.ts +5 -2
  38. package/src/config/refusal.ts +15 -2
  39. package/src/config/schema.ts +74 -12
  40. package/src/config/settings.ts +17 -2
  41. package/src/config/state-map.ts +5 -3
  42. package/src/config/wake-job.ts +74 -0
  43. package/src/decide/alignment.ts +4 -4
  44. package/src/decide/answer.ts +24 -22
  45. package/src/decide/kind.ts +1 -6
  46. package/src/decide/labels.ts +9 -9
  47. package/src/decide/merge.ts +6 -7
  48. package/src/decide/prose.ts +4 -4
  49. package/src/decide/question.ts +23 -20
  50. package/src/decide/review.ts +4 -2
  51. package/src/decide/risk.ts +25 -19
  52. package/src/decide/taste.ts +3 -3
  53. package/src/exec.ts +4 -3
  54. package/src/index.ts +14 -13
  55. package/src/loop/check-item.ts +3 -3
  56. package/src/loop/ci.ts +1 -1
  57. package/src/loop/github.ts +2 -2
  58. package/src/loop/groups.ts +13 -6
  59. package/src/loop/memory.ts +40 -28
  60. package/src/loop/notes.ts +5 -5
  61. package/src/loop/pair.ts +1 -1
  62. package/src/loop/pass.ts +18 -12
  63. package/src/loop/proof.ts +9 -7
  64. package/src/loop/prove-item.ts +1 -1
  65. package/src/loop/records.ts +12 -13
  66. package/src/loop/signals.ts +6 -6
  67. package/src/measure/inputs.ts +1 -1
  68. package/src/measure/measure.ts +4 -4
  69. package/src/measure/rate.ts +2 -0
  70. package/src/service/ci-tick.ts +2 -2
  71. package/src/service/supervise.ts +3 -2
  72. package/src/service/token.ts +3 -3
  73. package/src/service/wake.ts +2 -2
  74. package/src/service/workflows.ts +165 -48
  75. package/src/setup/propose.ts +78 -7
  76. package/src/setup/prove.ts +18 -12
  77. package/src/setup/scan.ts +91 -28
  78. package/src/setup/shipped.ts +8 -2
  79. package/src/setup/tables.ts +22 -3
  80. package/src/setup/waits.ts +9 -11
  81. package/src/setup/write.ts +67 -27
  82. package/src/slots/pool.ts +32 -24
  83. package/src/state/events.ts +1 -1
  84. package/src/state/faults.ts +13 -5
  85. package/src/state/format.ts +2 -2
  86. package/src/state/items.ts +9 -10
  87. package/src/state/open.ts +2 -2
  88. package/src/state/ops.ts +16 -10
  89. package/src/state/records.ts +5 -5
  90. package/src/state/runs.ts +33 -26
  91. package/src/state/saved.ts +44 -32
  92. package/src/status/file.ts +20 -1
  93. package/src/status/format.ts +47 -57
  94. package/src/status/sections.ts +2 -2
  95. package/src/worker/alignment-report.ts +4 -5
  96. package/src/worker/answers.ts +26 -24
  97. package/src/worker/base-check.ts +2 -2
  98. package/src/worker/kind.ts +23 -19
  99. package/src/worker/land.ts +114 -126
  100. package/src/worker/merged.ts +9 -7
  101. package/src/worker/open.ts +14 -12
  102. package/src/worker/prose-run.ts +3 -3
  103. package/src/worker/questions.ts +15 -12
  104. package/src/worker/scheduler.ts +4 -4
  105. package/src/worker/signature.ts +3 -3
  106. package/src/worker/tally.ts +0 -2
  107. package/src/worker/taste.ts +6 -15
  108. package/src/worker/thin-path.ts +148 -121
  109. package/src/worker/thread.ts +25 -0
  110. package/src/worker/worker.ts +77 -47
  111. package/surface.json +1091 -2251
  112. package/src/slots/scan.ts +0 -26
package/defaults/hum.json CHANGED
@@ -1,5 +1,5 @@
1
1
  {
2
- "schemaVersion": 1,
2
+ "schemaVersion": 2,
3
3
  "levels": [
4
4
  {
5
5
  "name": "low",
@@ -139,7 +139,14 @@
139
139
  },
140
140
  "restartDelaySeconds": 10,
141
141
  "ciWakeTimeoutMinutes": 60,
142
- "pullRequestTokenSecret": "HUM_GITHUB_TOKEN",
142
+ "pullRequestToken": {
143
+ "from": "secret",
144
+ "secret": "HUM_GITHUB_TOKEN"
145
+ },
146
+ "ciRunner": "ubuntu-latest",
147
+ "ciContainer": null,
148
+ "ciEnv": {},
149
+ "ciClaudeAuth": "oauth-token",
143
150
  "strictPaths": [],
144
151
  "launches": [],
145
152
  "sizeRange": {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@moo.md/hum",
3
- "version": "0.1.0",
3
+ "version": "0.3.0",
4
4
  "description": "The hum engine: work items become Claude Code agent runs, the repo's checks decide what they can, and only what nothing settles becomes a question to the owner.",
5
5
  "license": "MIT",
6
6
  "author": {
@@ -32,7 +32,7 @@
32
32
  "surface.json"
33
33
  ],
34
34
  "scripts": {
35
- "typecheck": "tsc --noEmit",
35
+ "typecheck": "test -d ../hum/node_modules || { echo \"hum-engine: the view tests import hum/mod, whose types come from hum's packages; run bun install in hum first\" >&2; exit 1; }; tsc --noEmit",
36
36
  "schema": "bun scripts/write-schema.ts",
37
37
  "eval": "bun scripts/eval.ts",
38
38
  "release:check": "bun tools/release/main.ts check",
@@ -4,7 +4,7 @@
4
4
  "properties": {
5
5
  "schemaVersion": {
6
6
  "type": "number",
7
- "const": 1
7
+ "const": 2
8
8
  },
9
9
  "levels": {
10
10
  "minItems": 1,
@@ -533,11 +533,55 @@
533
533
  "maximum": 9007199254740991,
534
534
  "description": "Seconds the service manager waits before it starts a failed start script again."
535
535
  },
536
- "pullRequestTokenSecret": {
537
- "default": "HUM_GITHUB_TOKEN",
538
- "type": "string",
539
- "minLength": 1,
540
- "description": "The repository secret holding a GitHub App or fine-grained token for CI runs. A pull request opened with the workflow token starts no workflows, so its required checks never run; one opened with this token does. A CI run falls back to the workflow token while the secret is unset."
536
+ "pullRequestToken": {
537
+ "default": {
538
+ "from": "secret",
539
+ "secret": "HUM_GITHUB_TOKEN"
540
+ },
541
+ "oneOf": [
542
+ {
543
+ "type": "object",
544
+ "properties": {
545
+ "from": {
546
+ "type": "string",
547
+ "const": "secret"
548
+ },
549
+ "secret": {
550
+ "type": "string",
551
+ "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
552
+ }
553
+ },
554
+ "required": [
555
+ "from",
556
+ "secret"
557
+ ],
558
+ "additionalProperties": false
559
+ },
560
+ {
561
+ "type": "object",
562
+ "properties": {
563
+ "from": {
564
+ "type": "string",
565
+ "const": "app"
566
+ },
567
+ "clientIdSecret": {
568
+ "type": "string",
569
+ "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
570
+ },
571
+ "privateKeySecret": {
572
+ "type": "string",
573
+ "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
574
+ }
575
+ },
576
+ "required": [
577
+ "from",
578
+ "clientIdSecret",
579
+ "privateKeySecret"
580
+ ],
581
+ "additionalProperties": false
582
+ }
583
+ ],
584
+ "description": "Where a CI run's GitHub token comes from: { from: 'secret', secret } names a repository secret that holds a fine-grained token; { from: 'app', clientIdSecret, privateKeySecret } names the two secrets a GitHub App mints a token from each run, with the workflow's own permissions. clientIdSecret holds the App's client ID or its older App ID: the action that mints the token reads either as the same input. A pull request opened with the workflow token starts no workflows, so its required checks never run; one opened with this token does. A CI run falls back to the workflow token while the secret, or the client ID secret, is unset."
541
585
  },
542
586
  "ciWakeTimeoutMinutes": {
543
587
  "default": 60,
@@ -546,6 +590,163 @@
546
590
  "maximum": 9007199254740991,
547
591
  "description": "Minutes a CI wake may run before GitHub stops it."
548
592
  },
593
+ "ciRunner": {
594
+ "default": "ubuntu-latest",
595
+ "anyOf": [
596
+ {
597
+ "anyOf": [
598
+ {
599
+ "type": "string",
600
+ "minLength": 1
601
+ },
602
+ {
603
+ "minItems": 1,
604
+ "type": "array",
605
+ "items": {
606
+ "type": "string",
607
+ "minLength": 1
608
+ }
609
+ }
610
+ ]
611
+ },
612
+ {
613
+ "type": "object",
614
+ "properties": {
615
+ "group": {
616
+ "type": "string",
617
+ "minLength": 1
618
+ },
619
+ "labels": {
620
+ "anyOf": [
621
+ {
622
+ "type": "string",
623
+ "minLength": 1
624
+ },
625
+ {
626
+ "minItems": 1,
627
+ "type": "array",
628
+ "items": {
629
+ "type": "string",
630
+ "minLength": 1
631
+ }
632
+ }
633
+ ]
634
+ }
635
+ },
636
+ "required": [
637
+ "group"
638
+ ],
639
+ "additionalProperties": false
640
+ }
641
+ ],
642
+ "description": "What the CI wake job runs on, as a workflow's runs-on holds it: one runner label, a list of labels the runner must carry all of, or { group, labels } for a runner group. On a public repo, a runner GitHub does not host keeps what a wake leaves on it, and setup asks before it writes one."
643
+ },
644
+ "ciContainer": {
645
+ "default": null,
646
+ "anyOf": [
647
+ {
648
+ "anyOf": [
649
+ {
650
+ "type": "string",
651
+ "minLength": 1
652
+ },
653
+ {
654
+ "type": "object",
655
+ "properties": {
656
+ "image": {
657
+ "type": "string",
658
+ "minLength": 1
659
+ },
660
+ "credentials": {
661
+ "type": "object",
662
+ "properties": {
663
+ "username": {
664
+ "type": "string",
665
+ "minLength": 1
666
+ },
667
+ "password": {
668
+ "type": "string",
669
+ "minLength": 1
670
+ }
671
+ },
672
+ "required": [
673
+ "username",
674
+ "password"
675
+ ],
676
+ "additionalProperties": false
677
+ },
678
+ "env": {
679
+ "type": "object",
680
+ "propertyNames": {
681
+ "type": "string",
682
+ "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
683
+ },
684
+ "additionalProperties": {
685
+ "type": "string"
686
+ }
687
+ },
688
+ "ports": {
689
+ "type": "array",
690
+ "items": {
691
+ "anyOf": [
692
+ {
693
+ "type": "string",
694
+ "minLength": 1
695
+ },
696
+ {
697
+ "type": "integer",
698
+ "exclusiveMinimum": 0,
699
+ "maximum": 9007199254740991
700
+ }
701
+ ]
702
+ }
703
+ },
704
+ "volumes": {
705
+ "type": "array",
706
+ "items": {
707
+ "type": "string",
708
+ "minLength": 1
709
+ }
710
+ },
711
+ "options": {
712
+ "type": "string",
713
+ "minLength": 1
714
+ }
715
+ },
716
+ "required": [
717
+ "image"
718
+ ],
719
+ "additionalProperties": false
720
+ }
721
+ ]
722
+ },
723
+ {
724
+ "type": "null"
725
+ }
726
+ ],
727
+ "description": "The container the CI wake job runs in: its image, or { image, credentials: { username, password }, env, ports, volumes, options } as a workflow's container holds them, or null to run on the runner itself. The image needs git, node and npm; a container that runs as root needs IS_SANDBOX=1 in ciEnv, or Claude Code refuses to skip its permission prompts."
728
+ },
729
+ "ciEnv": {
730
+ "default": {},
731
+ "type": "object",
732
+ "propertyNames": {
733
+ "type": "string",
734
+ "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
735
+ },
736
+ "additionalProperties": {
737
+ "type": "string"
738
+ },
739
+ "description": "Env the CI wake job sets, such as a gateway's settings; a value may be a workflow expression. A value that reads a secret reaches only the step that runs hum; every other value reaches every step, the installs too. A name the job sets itself, such as GH_TOKEN, is refused."
740
+ },
741
+ "ciClaudeAuth": {
742
+ "default": "oauth-token",
743
+ "type": "string",
744
+ "enum": [
745
+ "oauth-token",
746
+ "env"
747
+ ],
748
+ "description": "How a CI run's Claude Code signs in: 'oauth-token' requires the CLAUDE_CODE_OAUTH_TOKEN secret, from `claude setup-token`; 'env' asks for no secret, and ciEnv carries the credentials, such as a gateway's settings."
749
+ },
549
750
  "strictPaths": {
550
751
  "default": [],
551
752
  "type": "array",
@@ -1193,7 +1394,7 @@
1193
1394
  "type": "string",
1194
1395
  "minLength": 1
1195
1396
  },
1196
- "description": "GitHub logins whose pull request reviews and comments are the owner's review notes."
1397
+ "description": "GitHub logins whose pull request reviews and comments are the owner's review notes, and whose issue comments hum reads as replies. hum also reads the issue comments of the repo's owner, the owning organisation's members and invited collaborators, and reads no one else's."
1197
1398
  },
1198
1399
  "repeatAt": {
1199
1400
  "default": 2,
package/scripts/eval.ts CHANGED
@@ -6,10 +6,10 @@ import { AgentDecisionExhaustedError } from "@statelyai/agent";
6
6
  import PQueue from "p-queue";
7
7
  import { z } from "zod";
8
8
  import { type Finished, type Runner, startRunner } from "../src/adapters/claude-p.ts";
9
+ import { gitIn, readGitIdentity } from "../src/adapters/git.ts";
9
10
  import { modelCallOver } from "../src/adapters/model-call.ts";
10
- import { readConfigDir } from "../src/config/load.ts";
11
- import { BOUNDARIES, type Boundary, HumConfig } from "../src/config/schema.ts";
12
- import { alignmentQuestion, raisesLevel } from "../src/decide/alignment.ts";
11
+ import { BOUNDARIES, type Boundary } from "../src/config/schema.ts";
12
+ import { alignmentQuestion, doesRaiseLevel } from "../src/decide/alignment.ts";
13
13
  import { replyQuestion } from "../src/decide/answer.ts";
14
14
  import { kindQuestion } from "../src/decide/kind.ts";
15
15
  import { tasteQuestion } from "../src/decide/taste.ts";
@@ -17,17 +17,21 @@ import { pairQuestion } from "../src/loop/pair.ts";
17
17
  import type { ReviewNote } from "../src/loop/records.ts";
18
18
  import { pairing, prepare } from "../src/loop/signals.ts";
19
19
  import type { JsonSchema, ProsePurpose } from "../src/decide/prose.ts";
20
+ import { SHIPPED_COMMAND_TIMEOUT_MS } from "../src/exec.ts";
21
+ import { shippedRaw, shippedSettingsAs } from "../src/setup/shipped.ts";
20
22
  import { openState } from "../src/state/open.ts";
21
23
  import { decideProse } from "../src/worker/prose-run.ts";
22
24
  import { PROSE_PROMPTS } from "../src/worker/thin-path.ts";
23
25
  import { SECOND_MS } from "../src/time.ts";
26
+ import { rateOf } from "../src/measure/rate.ts";
24
27
 
25
28
  /**
26
29
  * Measures one model-judgment boundary on its labelled cases, through the same prompts and fences the engine runs,
27
30
  * and adds a row to its RESULTS.md. `bun run eval -- --model <id>` measures every boundary on that model id; naming
28
31
  * boundaries measures only those. A case the runner held or limited counts in no rate. `--runs <n>` repeats each
29
32
  * boundary and pools the runs, since one run proves little; `--write <folder>` records each pooled rate under
30
- * `measuredModels` in that folder's `hum.json`, which is how setup measures an owner's own models.
33
+ * `measuredModels` in that folder's `hum.json`, which is how setup measures an owner's own models. A boundary where
34
+ * nothing was measured records no rate, and the script exits non-zero.
31
35
  */
32
36
 
33
37
  const { values, positionals } = parseArgs({ allowPositionals: true, options: { model: { type: "string" }, limit: { type: "string" }, runs: { type: "string" }, write: { type: "string" } } });
@@ -36,16 +40,23 @@ const OWNER_LABELLED: readonly Boundary[] = ["taste"];
36
40
 
37
41
  const boundaries: readonly Boundary[] = positionals.length > 0 ? z.array(z.enum(BOUNDARIES)).parse(positionals) : BOUNDARIES.filter((boundary) => !OWNER_LABELLED.includes(boundary));
38
42
  const root = join(import.meta.dir, "..");
39
- const raw = readConfigDir(join(root, "defaults"));
40
- const config = { hum: HumConfig.parse(raw.hum), roles: raw.roles };
43
+ const settings = shippedSettingsAs(await readGitIdentity(gitIn(process.cwd(), { timeoutMs: SHIPPED_COMMAND_TIMEOUT_MS })));
44
+
45
+ if (!settings.ok) {
46
+ console.error(settings.refusal.message);
47
+ process.exit(1);
48
+ }
49
+
50
+ const config = { hum: settings.value, roles: shippedRaw().roles };
41
51
  const model = values.model ?? config.hum.decisionModel;
42
- const runs = values.runs === undefined ? 1 : Number(values.runs);
43
- const limit = values.limit === undefined ? Number.POSITIVE_INFINITY : Number(values.limit);
52
+ const Count = z.coerce.number().int().positive();
53
+ const runs = values.runs === undefined ? 1 : Count.parse(values.runs);
54
+ const limit = values.limit === undefined ? Number.POSITIVE_INFINITY : Count.parse(values.limit);
44
55
 
45
56
  type Verdict = "right" | "wrong" | "abstained" | "halted";
46
57
 
47
- const state = openState(mkdtempSync(join(tmpdir(), "hum-eval-")));
48
- const started = await startRunner({ state, model });
58
+ const state = openState(mkdtempSync(join(tmpdir(), "hum-eval-")), { now: Date.now });
59
+ const started = await startRunner({ state, model, now: Date.now, env: process.env });
49
60
 
50
61
  if (started.check.kind !== "finished") throw new Error(`the start check ended as ${started.check.kind}`);
51
62
 
@@ -109,7 +120,7 @@ const TasteCase = z.object({ case: z.string(), label: z.enum(["keeps", "breaks"]
109
120
  /** Calls at once: enough to finish a set in minutes, few enough not to trip a rate limit. */
110
121
  const AT_ONCE = 4;
111
122
 
112
- const eachOf = <T>(cases: readonly T[], measure: (found: T) => Promise<Verdict>): Promise<Verdict[]> => new PQueue({ concurrency: AT_ONCE }).addAll(cases.map((found) => () => measure(found)));
123
+ const eachOf = <T, R>(cases: readonly T[], measure: (found: T) => Promise<R>): Promise<R[]> => new PQueue({ concurrency: AT_ONCE }).addAll(cases.map((found) => () => measure(found)));
113
124
 
114
125
  const read = async <T>(boundary: Boundary, schema: z.ZodType<T>, file = "cases.jsonl"): Promise<T[]> =>
115
126
  (await Bun.file(join(root, "evals", boundary, file)).text())
@@ -118,78 +129,98 @@ const read = async <T>(boundary: Boundary, schema: z.ZodType<T>, file = "cases.j
118
129
  .map((line) => schema.parse(JSON.parse(line)));
119
130
 
120
131
  /** The confusion counts of the review-note boundary: its rate is precision, and recall is recorded beside it. */
121
- const pairCounts = { tp: 0, fp: 0, fn: 0, tn: 0, pairedBySignals: 0 };
132
+ type PairCounts = { readonly tp: number; readonly fp: number; readonly fn: number; readonly tn: number; readonly pairedBySignals: number };
133
+
134
+ type Cell = Exclude<keyof PairCounts, "pairedBySignals">;
135
+
136
+ const NO_PAIRS: PairCounts = { tp: 0, fp: 0, fn: 0, tn: 0, pairedBySignals: 0 };
137
+
138
+ const addPairs = (a: PairCounts, b: PairCounts): PairCounts => ({ tp: a.tp + b.tp, fp: a.fp + b.fp, fn: a.fn + b.fn, tn: a.tn + b.tn, pairedBySignals: a.pairedBySignals + b.pairedBySignals });
139
+
140
+ /** One run of a boundary: each case's verdict, and for the review-note boundary its confusion counts. */
141
+ type Measured = { readonly verdicts: readonly Verdict[]; readonly pairs?: PairCounts };
122
142
 
123
143
  const noteOf = (text: string, sourceId: string): ReviewNote => ({ source: "review-comment", sourceId, change: sourceId, hunk: "", text, at: 0 });
124
144
 
125
- const BOUNDARY_RUNS: Record<Boundary, () => Promise<Verdict[]>> = {
145
+ const BOUNDARY_RUNS: Record<Boundary, () => Promise<Measured>> = {
126
146
  kind: async () => {
127
147
  const cases = (await read("kind", KindCase)).slice(0, limit);
128
148
 
129
- return eachOf(cases, async (found) => {
149
+ return {
150
+ verdicts: await eachOf(cases, async (found) => {
130
151
  const judged = await judge(kindQuestion(found, config.hum));
131
152
 
132
153
  return judged.kind === "abstained" ? "abstained" : scored(judged.value.kind, found.label);
133
- });
154
+ }),
155
+ };
134
156
  },
135
157
  alignment: async () => {
136
158
  const sample = (await read("alignment", AlignmentCase, "sample.jsonl")).slice(0, limit);
137
159
 
138
- return eachOf(sample, async (found) => {
160
+ return {
161
+ verdicts: await eachOf(sample, async (found): Promise<Verdict> => {
139
162
  const judged = await judge(alignmentQuestion({ title: found.subject, body: "" }, found.diff, config.hum));
140
163
 
141
164
  if (judged.kind === "abstained") return "abstained";
142
165
 
143
- return raisesLevel(0, config.hum.levels.length, judged.value, config.hum) === (found.label !== "clean") ? "right" : "wrong";
144
- });
166
+ return doesRaiseLevel({ rank: 0, levels: config.hum.levels.length, answer: judged.value, config: config.hum }) === (found.label !== "clean") ? "right" : "wrong";
167
+ }),
168
+ };
145
169
  },
146
170
  "plan-outcome": async () => {
147
171
  const repo = planRepo();
148
172
  const cases = (await read("plan-outcome", PlanCase)).slice(0, limit);
149
173
 
150
- return eachOf(cases, async (found) => scored(await prosePick("plan", PROSE_PROMPTS.plan(config, { id: found.case, title: found.body.split("\n")[0] ?? "", body: found.body }), "", repo), found.label));
174
+ return { verdicts: await eachOf(cases, async (found) => scored(await prosePick("plan", PROSE_PROMPTS.plan(config, { id: found.case, title: found.body.split("\n")[0] ?? "", body: found.body }), "", repo), found.label)) };
151
175
  },
152
176
  "reply-option": async () => {
153
177
  const cases = (await read("reply-option", ReplyCase)).slice(0, limit);
154
178
 
155
- return eachOf(cases, async (found) => {
156
- const judged = await judge(replyQuestion(found.question, found.options, found.reply));
179
+ return {
180
+ verdicts: await eachOf(cases, async (found) => {
181
+ const judged = await judge(replyQuestion({ question: found.question, options: found.options, reply: found.reply }));
157
182
 
158
- return judged.kind === "abstained" ? "abstained" : scored(judged.value.option, found.label);
159
- });
183
+ return judged.kind === "abstained" ? "abstained" : scored(judged.value.option, found.label);
184
+ }),
185
+ };
160
186
  },
161
187
  "repeat-pair": async () => {
162
188
  const cases = (await read("repeat-pair", PairCase)).slice(0, limit);
163
189
  const open = cases.filter((found) => pairing(prepare(noteOf(found.a.text, "a")), prepare(noteOf(found.b.text, "b"))).signal === undefined);
164
190
 
165
- pairCounts.pairedBySignals = cases.length - open.length;
191
+ const judged = await eachOf(open, async (found): Promise<{ verdict: Verdict; cell?: Cell }> => {
192
+ const pair = await judge(pairQuestion(noteOf(found.a.text, "a"), noteOf(found.b.text, "b")));
166
193
 
167
- return eachOf(open, async (found) => {
168
- const judged = await judge(pairQuestion(noteOf(found.a.text, "a"), noteOf(found.b.text, "b")));
194
+ if (pair.kind === "abstained") return { verdict: "abstained" };
169
195
 
170
- if (judged.kind === "abstained") return "abstained";
171
-
172
- const said = judged.value.same;
196
+ const said = pair.value.same;
173
197
  const truth = found.label === "same";
174
198
 
175
- pairCounts[said ? (truth ? "tp" : "fp") : truth ? "fn" : "tn"]++;
176
-
177
- return said === truth ? "right" : "wrong";
199
+ return { verdict: said === truth ? "right" : "wrong", cell: said ? (truth ? "tp" : "fp") : truth ? "fn" : "tn" };
178
200
  });
201
+
202
+ const countOf = (cell: Cell) => judged.filter((found) => found.cell === cell).length;
203
+
204
+ return {
205
+ verdicts: judged.map(({ verdict }) => verdict),
206
+ pairs: { tp: countOf("tp"), fp: countOf("fp"), fn: countOf("fn"), tn: countOf("tn"), pairedBySignals: cases.length - open.length },
207
+ };
179
208
  },
180
209
  taste: async () => {
181
210
  const cases = (await read("taste", TasteCase)).slice(0, limit);
182
211
 
183
- return eachOf(cases, async (found) => {
184
- const judged = await judge(tasteQuestion(found.rule, found.diff, config.hum));
212
+ return {
213
+ verdicts: await eachOf(cases, async (found) => {
214
+ const judged = await judge(tasteQuestion(found.rule, found.diff, config.hum));
185
215
 
186
- return judged.kind === "abstained" ? "abstained" : scored(judged.value.keeps ? "keeps" : "breaks", found.label);
187
- });
216
+ return judged.kind === "abstained" ? "abstained" : scored(judged.value.keeps ? "keeps" : "breaks", found.label);
217
+ }),
218
+ };
188
219
  },
189
220
  "review-verdict": async () => {
190
221
  const cases = (await read("review-verdict", ReviewCase)).slice(0, limit);
191
222
 
192
- return eachOf(cases, async (found) => scored(await prosePick("review", PROSE_PROMPTS.review(config, { id: found.case, title: found.item, body: "" }, found.diff), found.diff, tmpdir()), found.label));
223
+ return { verdicts: await eachOf(cases, async (found) => scored(await prosePick("review", PROSE_PROMPTS.review(config, { id: found.case, title: found.item, body: "" }, found.diff), found.diff, tmpdir()), found.label)) };
193
224
  },
194
225
  };
195
226
 
@@ -197,12 +228,13 @@ const header = "| Date | Model id | n | Right | Wrong | Abstained | Rate | Live
197
228
 
198
229
  const pairHeader = "| Date | Model id | n | TP | FP | FN | TN | Abstained | Paired by signals | Precision | Recall | Live calls |\n|---|---|---|---|---|---|---|---|---|---|---|---|\n";
199
230
 
200
- const ratio = (part: number, whole: number): string => (whole === 0 ? "0.00" : (part / whole).toFixed(2));
231
+ /** A rate as a RESULTS.md cell; a dash where nothing was measured. */
232
+ const rateCell = (rate: number | undefined): string => rate?.toFixed(2) ?? "-";
201
233
 
202
- const pairRow = (day: string, abstained: number, used: number): string =>
203
- `| ${day} | ${model} | ${pairCounts.tp + pairCounts.fp + pairCounts.fn + pairCounts.tn} | ${pairCounts.tp} | ${pairCounts.fp} | ${pairCounts.fn} | ${pairCounts.tn} | ${abstained} | ${pairCounts.pairedBySignals} | ${ratio(pairCounts.tp, pairCounts.tp + pairCounts.fp)} | ${ratio(pairCounts.tp, pairCounts.tp + pairCounts.fn)} | ${used} |`;
234
+ const precisionOf = ({ tp, fp }: PairCounts): number | undefined => rateOf(tp, tp + fp);
204
235
 
205
- const EMPTY_POOL = { right: 0, measured: 0 };
236
+ const pairRow = (day: string, pairs: PairCounts, abstained: number, used: number): string =>
237
+ `| ${day} | ${model} | ${pairs.tp + pairs.fp + pairs.fn + pairs.tn} | ${pairs.tp} | ${pairs.fp} | ${pairs.fn} | ${pairs.tn} | ${abstained} | ${pairs.pairedBySignals} | ${rateCell(precisionOf(pairs))} | ${rateCell(rateOf(pairs.tp, pairs.tp + pairs.fn))} | ${used} |`;
206
238
 
207
239
  /** Sets one pooled rate in the owner's `hum.json`, where trust reads it. */
208
240
  const recordRate = async (folder: string, boundary: Boundary, rate: number): Promise<void> => {
@@ -212,33 +244,39 @@ const recordRate = async (folder: string, boundary: Boundary, rate: number): Pro
212
244
  await Bun.write(file, `${JSON.stringify({ ...written, measuredModels: { ...written.measuredModels, [boundary]: { ...written.measuredModels[boundary], [model]: Number(rate.toFixed(2)) } } }, null, "\t")}\n`);
213
245
  };
214
246
 
215
- for (const boundary of boundaries) {
216
- const pool = { ...EMPTY_POOL };
247
+ const unmeasured: Boundary[] = [];
217
248
 
218
- Object.assign(pairCounts, { tp: 0, fp: 0, fn: 0, tn: 0, pairedBySignals: 0 });
249
+ for (const boundary of boundaries) {
250
+ const pool = { right: 0, measured: 0, pairs: NO_PAIRS };
219
251
 
220
252
  for (let run = 1; run <= runs; run++) {
221
253
  const before = calls;
222
- const verdicts = await BOUNDARY_RUNS[boundary]();
254
+ const { verdicts, pairs = NO_PAIRS } = await BOUNDARY_RUNS[boundary]();
223
255
  const count = (verdict: Verdict) => verdicts.filter((found) => found === verdict).length;
224
256
  const measured = verdicts.length - count("halted");
225
- const rate = measured === 0 ? 0 : count("right") / measured;
226
257
  const day = new Date().toISOString().slice(0, 10);
227
- const row = boundary === "repeat-pair" ? pairRow(day, count("abstained"), calls - before) : `| ${day} | ${model} | ${measured} | ${count("right")} | ${count("wrong")} | ${count("abstained")} | ${rate.toFixed(2)} | ${calls - before} |`;
258
+ const row = boundary === "repeat-pair" ? pairRow(day, pairs, count("abstained"), calls - before) : `| ${day} | ${model} | ${measured} | ${count("right")} | ${count("wrong")} | ${count("abstained")} | ${rateCell(rateOf(count("right"), measured))} | ${calls - before} |`;
228
259
  const results = join(values.write ?? root, "evals", boundary, "RESULTS.md");
229
260
  const earlier = (await Bun.file(results).exists()) ? await Bun.file(results).text() : boundary === "repeat-pair" ? pairHeader : header;
230
261
 
231
262
  pool.right += count("right");
232
263
  pool.measured += measured;
264
+ pool.pairs = addPairs(pool.pairs, pairs);
233
265
  await Bun.write(results, `${earlier}${row}\n`);
234
266
  console.log(`${boundary}: ${row}`);
235
267
  }
236
268
 
237
- const pooled = boundary === "repeat-pair" ? pairCounts.tp / (pairCounts.tp + pairCounts.fp || 1) : pool.right / (pool.measured || 1);
269
+ const pooled = boundary === "repeat-pair" ? precisionOf(pool.pairs) : rateOf(pool.right, pool.measured);
238
270
 
239
- console.log(`${boundary}: pooled over ${runs} run(s): ${pooled.toFixed(2)}`);
271
+ console.log(`${boundary}: pooled over ${runs} run(s): ${rateCell(pooled)}`);
272
+
273
+ if (pooled === undefined) unmeasured.push(boundary);
274
+ else if (values.write !== undefined) await recordRate(values.write, boundary, pooled);
275
+ }
240
276
 
241
- if (values.write !== undefined) await recordRate(values.write, boundary, pooled);
277
+ if (unmeasured.length > 0) {
278
+ console.error(`Nothing was measured on ${unmeasured.join(", ")}, so no rate was recorded for ${unmeasured.length === 1 ? "it" : "them"}.`);
279
+ process.exitCode = 1;
242
280
  }
243
281
 
244
282
  console.log(`${calls} live calls, the start check included. Record each rate under measuredModels in defaults/hum.json; a model decides a boundary alone only at minJudgeRate or above; repeat-pair takes its precision, against repeatPrecision.`);