@moo.md/hum 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/defaults/hum.json +9 -2
- package/package.json +2 -2
- package/schema/hum.schema.json +208 -7
- package/scripts/eval.ts +88 -50
- package/src/adapter-api.ts +46 -14
- package/src/adapters/admit.ts +2 -2
- package/src/adapters/argument.ts +1 -1
- package/src/adapters/claim.ts +18 -15
- package/src/adapters/claude-bg.ts +50 -27
- package/src/adapters/claude-p.ts +18 -23
- package/src/adapters/contract.ts +22 -10
- package/src/adapters/each-call.ts +9 -0
- package/src/adapters/fake.ts +6 -6
- package/src/adapters/git.ts +10 -18
- package/src/adapters/github-client.ts +102 -0
- package/src/adapters/github.ts +61 -146
- package/src/adapters/guard.ts +20 -9
- package/src/adapters/history.ts +12 -15
- package/src/adapters/item-file.ts +1 -1
- package/src/adapters/markdown.ts +49 -42
- package/src/adapters/marked.ts +6 -12
- package/src/adapters/model-call.ts +4 -4
- package/src/adapters/once.ts +5 -5
- package/src/adapters/owner-commit.ts +13 -11
- package/src/adapters/pull-requests.ts +11 -13
- package/src/adapters/registry.ts +9 -8
- package/src/adapters/roundup.ts +27 -35
- package/src/adapters/serial.ts +4 -5
- package/src/adapters/transcript.ts +13 -8
- package/src/cli/main.ts +1 -0
- package/src/cli/ops.ts +32 -14
- package/src/cli/service.ts +14 -8
- package/src/cli/setup.ts +22 -7
- package/src/cli/shared-watch.ts +3 -3
- package/src/config/checks.ts +27 -18
- package/src/config/machine.ts +5 -7
- package/src/config/migrate.ts +5 -2
- package/src/config/refusal.ts +15 -2
- package/src/config/schema.ts +74 -12
- package/src/config/settings.ts +17 -2
- package/src/config/state-map.ts +5 -3
- package/src/config/wake-job.ts +74 -0
- package/src/decide/alignment.ts +4 -4
- package/src/decide/answer.ts +24 -22
- package/src/decide/kind.ts +1 -6
- package/src/decide/labels.ts +9 -9
- package/src/decide/merge.ts +6 -7
- package/src/decide/prose.ts +4 -4
- package/src/decide/question.ts +23 -20
- package/src/decide/review.ts +4 -2
- package/src/decide/risk.ts +25 -19
- package/src/decide/taste.ts +3 -3
- package/src/exec.ts +4 -3
- package/src/index.ts +14 -13
- package/src/loop/check-item.ts +3 -3
- package/src/loop/ci.ts +1 -1
- package/src/loop/github.ts +2 -2
- package/src/loop/groups.ts +13 -6
- package/src/loop/memory.ts +40 -28
- package/src/loop/notes.ts +5 -5
- package/src/loop/pair.ts +1 -1
- package/src/loop/pass.ts +18 -12
- package/src/loop/proof.ts +9 -7
- package/src/loop/prove-item.ts +1 -1
- package/src/loop/records.ts +12 -13
- package/src/loop/signals.ts +6 -6
- package/src/measure/inputs.ts +1 -1
- package/src/measure/measure.ts +4 -4
- package/src/measure/rate.ts +2 -0
- package/src/service/ci-tick.ts +2 -2
- package/src/service/supervise.ts +3 -2
- package/src/service/token.ts +3 -3
- package/src/service/wake.ts +2 -2
- package/src/service/workflows.ts +165 -48
- package/src/setup/propose.ts +78 -7
- package/src/setup/prove.ts +18 -12
- package/src/setup/scan.ts +91 -28
- package/src/setup/shipped.ts +8 -2
- package/src/setup/tables.ts +22 -3
- package/src/setup/waits.ts +9 -11
- package/src/setup/write.ts +67 -27
- package/src/slots/pool.ts +32 -24
- package/src/state/events.ts +1 -1
- package/src/state/faults.ts +13 -5
- package/src/state/format.ts +2 -2
- package/src/state/items.ts +9 -10
- package/src/state/open.ts +2 -2
- package/src/state/ops.ts +16 -10
- package/src/state/records.ts +5 -5
- package/src/state/runs.ts +33 -26
- package/src/state/saved.ts +44 -32
- package/src/status/file.ts +20 -1
- package/src/status/format.ts +47 -57
- package/src/status/sections.ts +2 -2
- package/src/worker/alignment-report.ts +4 -5
- package/src/worker/answers.ts +26 -24
- package/src/worker/base-check.ts +2 -2
- package/src/worker/kind.ts +23 -19
- package/src/worker/land.ts +114 -126
- package/src/worker/merged.ts +9 -7
- package/src/worker/open.ts +14 -12
- package/src/worker/prose-run.ts +3 -3
- package/src/worker/questions.ts +15 -12
- package/src/worker/scheduler.ts +4 -4
- package/src/worker/signature.ts +3 -3
- package/src/worker/tally.ts +0 -2
- package/src/worker/taste.ts +6 -15
- package/src/worker/thin-path.ts +148 -121
- package/src/worker/thread.ts +25 -0
- package/src/worker/worker.ts +77 -47
- package/surface.json +1091 -2251
- package/src/slots/scan.ts +0 -26
package/defaults/hum.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"schemaVersion":
|
|
2
|
+
"schemaVersion": 2,
|
|
3
3
|
"levels": [
|
|
4
4
|
{
|
|
5
5
|
"name": "low",
|
|
@@ -139,7 +139,14 @@
|
|
|
139
139
|
},
|
|
140
140
|
"restartDelaySeconds": 10,
|
|
141
141
|
"ciWakeTimeoutMinutes": 60,
|
|
142
|
-
"
|
|
142
|
+
"pullRequestToken": {
|
|
143
|
+
"from": "secret",
|
|
144
|
+
"secret": "HUM_GITHUB_TOKEN"
|
|
145
|
+
},
|
|
146
|
+
"ciRunner": "ubuntu-latest",
|
|
147
|
+
"ciContainer": null,
|
|
148
|
+
"ciEnv": {},
|
|
149
|
+
"ciClaudeAuth": "oauth-token",
|
|
143
150
|
"strictPaths": [],
|
|
144
151
|
"launches": [],
|
|
145
152
|
"sizeRange": {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@moo.md/hum",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "The hum engine: work items become Claude Code agent runs, the repo's checks decide what they can, and only what nothing settles becomes a question to the owner.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": {
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"surface.json"
|
|
33
33
|
],
|
|
34
34
|
"scripts": {
|
|
35
|
-
"typecheck": "tsc --noEmit",
|
|
35
|
+
"typecheck": "test -d ../hum/node_modules || { echo \"hum-engine: the view tests import hum/mod, whose types come from hum's packages; run bun install in hum first\" >&2; exit 1; }; tsc --noEmit",
|
|
36
36
|
"schema": "bun scripts/write-schema.ts",
|
|
37
37
|
"eval": "bun scripts/eval.ts",
|
|
38
38
|
"release:check": "bun tools/release/main.ts check",
|
package/schema/hum.schema.json
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
"properties": {
|
|
5
5
|
"schemaVersion": {
|
|
6
6
|
"type": "number",
|
|
7
|
-
"const":
|
|
7
|
+
"const": 2
|
|
8
8
|
},
|
|
9
9
|
"levels": {
|
|
10
10
|
"minItems": 1,
|
|
@@ -533,11 +533,55 @@
|
|
|
533
533
|
"maximum": 9007199254740991,
|
|
534
534
|
"description": "Seconds the service manager waits before it starts a failed start script again."
|
|
535
535
|
},
|
|
536
|
-
"
|
|
537
|
-
"default":
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
536
|
+
"pullRequestToken": {
|
|
537
|
+
"default": {
|
|
538
|
+
"from": "secret",
|
|
539
|
+
"secret": "HUM_GITHUB_TOKEN"
|
|
540
|
+
},
|
|
541
|
+
"oneOf": [
|
|
542
|
+
{
|
|
543
|
+
"type": "object",
|
|
544
|
+
"properties": {
|
|
545
|
+
"from": {
|
|
546
|
+
"type": "string",
|
|
547
|
+
"const": "secret"
|
|
548
|
+
},
|
|
549
|
+
"secret": {
|
|
550
|
+
"type": "string",
|
|
551
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
|
|
552
|
+
}
|
|
553
|
+
},
|
|
554
|
+
"required": [
|
|
555
|
+
"from",
|
|
556
|
+
"secret"
|
|
557
|
+
],
|
|
558
|
+
"additionalProperties": false
|
|
559
|
+
},
|
|
560
|
+
{
|
|
561
|
+
"type": "object",
|
|
562
|
+
"properties": {
|
|
563
|
+
"from": {
|
|
564
|
+
"type": "string",
|
|
565
|
+
"const": "app"
|
|
566
|
+
},
|
|
567
|
+
"clientIdSecret": {
|
|
568
|
+
"type": "string",
|
|
569
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
|
|
570
|
+
},
|
|
571
|
+
"privateKeySecret": {
|
|
572
|
+
"type": "string",
|
|
573
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
|
|
574
|
+
}
|
|
575
|
+
},
|
|
576
|
+
"required": [
|
|
577
|
+
"from",
|
|
578
|
+
"clientIdSecret",
|
|
579
|
+
"privateKeySecret"
|
|
580
|
+
],
|
|
581
|
+
"additionalProperties": false
|
|
582
|
+
}
|
|
583
|
+
],
|
|
584
|
+
"description": "Where a CI run's GitHub token comes from: { from: 'secret', secret } names a repository secret that holds a fine-grained token; { from: 'app', clientIdSecret, privateKeySecret } names the two secrets a GitHub App mints a token from each run, with the workflow's own permissions. clientIdSecret holds the App's client ID or its older App ID: the action that mints the token reads either as the same input. A pull request opened with the workflow token starts no workflows, so its required checks never run; one opened with this token does. A CI run falls back to the workflow token while the secret, or the client ID secret, is unset."
|
|
541
585
|
},
|
|
542
586
|
"ciWakeTimeoutMinutes": {
|
|
543
587
|
"default": 60,
|
|
@@ -546,6 +590,163 @@
|
|
|
546
590
|
"maximum": 9007199254740991,
|
|
547
591
|
"description": "Minutes a CI wake may run before GitHub stops it."
|
|
548
592
|
},
|
|
593
|
+
"ciRunner": {
|
|
594
|
+
"default": "ubuntu-latest",
|
|
595
|
+
"anyOf": [
|
|
596
|
+
{
|
|
597
|
+
"anyOf": [
|
|
598
|
+
{
|
|
599
|
+
"type": "string",
|
|
600
|
+
"minLength": 1
|
|
601
|
+
},
|
|
602
|
+
{
|
|
603
|
+
"minItems": 1,
|
|
604
|
+
"type": "array",
|
|
605
|
+
"items": {
|
|
606
|
+
"type": "string",
|
|
607
|
+
"minLength": 1
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
]
|
|
611
|
+
},
|
|
612
|
+
{
|
|
613
|
+
"type": "object",
|
|
614
|
+
"properties": {
|
|
615
|
+
"group": {
|
|
616
|
+
"type": "string",
|
|
617
|
+
"minLength": 1
|
|
618
|
+
},
|
|
619
|
+
"labels": {
|
|
620
|
+
"anyOf": [
|
|
621
|
+
{
|
|
622
|
+
"type": "string",
|
|
623
|
+
"minLength": 1
|
|
624
|
+
},
|
|
625
|
+
{
|
|
626
|
+
"minItems": 1,
|
|
627
|
+
"type": "array",
|
|
628
|
+
"items": {
|
|
629
|
+
"type": "string",
|
|
630
|
+
"minLength": 1
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
]
|
|
634
|
+
}
|
|
635
|
+
},
|
|
636
|
+
"required": [
|
|
637
|
+
"group"
|
|
638
|
+
],
|
|
639
|
+
"additionalProperties": false
|
|
640
|
+
}
|
|
641
|
+
],
|
|
642
|
+
"description": "What the CI wake job runs on, as a workflow's runs-on holds it: one runner label, a list of labels the runner must carry all of, or { group, labels } for a runner group. On a public repo, a runner GitHub does not host keeps what a wake leaves on it, and setup asks before it writes one."
|
|
643
|
+
},
|
|
644
|
+
"ciContainer": {
|
|
645
|
+
"default": null,
|
|
646
|
+
"anyOf": [
|
|
647
|
+
{
|
|
648
|
+
"anyOf": [
|
|
649
|
+
{
|
|
650
|
+
"type": "string",
|
|
651
|
+
"minLength": 1
|
|
652
|
+
},
|
|
653
|
+
{
|
|
654
|
+
"type": "object",
|
|
655
|
+
"properties": {
|
|
656
|
+
"image": {
|
|
657
|
+
"type": "string",
|
|
658
|
+
"minLength": 1
|
|
659
|
+
},
|
|
660
|
+
"credentials": {
|
|
661
|
+
"type": "object",
|
|
662
|
+
"properties": {
|
|
663
|
+
"username": {
|
|
664
|
+
"type": "string",
|
|
665
|
+
"minLength": 1
|
|
666
|
+
},
|
|
667
|
+
"password": {
|
|
668
|
+
"type": "string",
|
|
669
|
+
"minLength": 1
|
|
670
|
+
}
|
|
671
|
+
},
|
|
672
|
+
"required": [
|
|
673
|
+
"username",
|
|
674
|
+
"password"
|
|
675
|
+
],
|
|
676
|
+
"additionalProperties": false
|
|
677
|
+
},
|
|
678
|
+
"env": {
|
|
679
|
+
"type": "object",
|
|
680
|
+
"propertyNames": {
|
|
681
|
+
"type": "string",
|
|
682
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
|
|
683
|
+
},
|
|
684
|
+
"additionalProperties": {
|
|
685
|
+
"type": "string"
|
|
686
|
+
}
|
|
687
|
+
},
|
|
688
|
+
"ports": {
|
|
689
|
+
"type": "array",
|
|
690
|
+
"items": {
|
|
691
|
+
"anyOf": [
|
|
692
|
+
{
|
|
693
|
+
"type": "string",
|
|
694
|
+
"minLength": 1
|
|
695
|
+
},
|
|
696
|
+
{
|
|
697
|
+
"type": "integer",
|
|
698
|
+
"exclusiveMinimum": 0,
|
|
699
|
+
"maximum": 9007199254740991
|
|
700
|
+
}
|
|
701
|
+
]
|
|
702
|
+
}
|
|
703
|
+
},
|
|
704
|
+
"volumes": {
|
|
705
|
+
"type": "array",
|
|
706
|
+
"items": {
|
|
707
|
+
"type": "string",
|
|
708
|
+
"minLength": 1
|
|
709
|
+
}
|
|
710
|
+
},
|
|
711
|
+
"options": {
|
|
712
|
+
"type": "string",
|
|
713
|
+
"minLength": 1
|
|
714
|
+
}
|
|
715
|
+
},
|
|
716
|
+
"required": [
|
|
717
|
+
"image"
|
|
718
|
+
],
|
|
719
|
+
"additionalProperties": false
|
|
720
|
+
}
|
|
721
|
+
]
|
|
722
|
+
},
|
|
723
|
+
{
|
|
724
|
+
"type": "null"
|
|
725
|
+
}
|
|
726
|
+
],
|
|
727
|
+
"description": "The container the CI wake job runs in: its image, or { image, credentials: { username, password }, env, ports, volumes, options } as a workflow's container holds them, or null to run on the runner itself. The image needs git, node and npm; a container that runs as root needs IS_SANDBOX=1 in ciEnv, or Claude Code refuses to skip its permission prompts."
|
|
728
|
+
},
|
|
729
|
+
"ciEnv": {
|
|
730
|
+
"default": {},
|
|
731
|
+
"type": "object",
|
|
732
|
+
"propertyNames": {
|
|
733
|
+
"type": "string",
|
|
734
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
|
|
735
|
+
},
|
|
736
|
+
"additionalProperties": {
|
|
737
|
+
"type": "string"
|
|
738
|
+
},
|
|
739
|
+
"description": "Env the CI wake job sets, such as a gateway's settings; a value may be a workflow expression. A value that reads a secret reaches only the step that runs hum; every other value reaches every step, the installs too. A name the job sets itself, such as GH_TOKEN, is refused."
|
|
740
|
+
},
|
|
741
|
+
"ciClaudeAuth": {
|
|
742
|
+
"default": "oauth-token",
|
|
743
|
+
"type": "string",
|
|
744
|
+
"enum": [
|
|
745
|
+
"oauth-token",
|
|
746
|
+
"env"
|
|
747
|
+
],
|
|
748
|
+
"description": "How a CI run's Claude Code signs in: 'oauth-token' requires the CLAUDE_CODE_OAUTH_TOKEN secret, from `claude setup-token`; 'env' asks for no secret, and ciEnv carries the credentials, such as a gateway's settings."
|
|
749
|
+
},
|
|
549
750
|
"strictPaths": {
|
|
550
751
|
"default": [],
|
|
551
752
|
"type": "array",
|
|
@@ -1193,7 +1394,7 @@
|
|
|
1193
1394
|
"type": "string",
|
|
1194
1395
|
"minLength": 1
|
|
1195
1396
|
},
|
|
1196
|
-
"description": "GitHub logins whose pull request reviews and comments are the owner's review notes."
|
|
1397
|
+
"description": "GitHub logins whose pull request reviews and comments are the owner's review notes, and whose issue comments hum reads as replies. hum also reads the issue comments of the repo's owner, the owning organisation's members and invited collaborators, and reads no one else's."
|
|
1197
1398
|
},
|
|
1198
1399
|
"repeatAt": {
|
|
1199
1400
|
"default": 2,
|
package/scripts/eval.ts
CHANGED
|
@@ -6,10 +6,10 @@ import { AgentDecisionExhaustedError } from "@statelyai/agent";
|
|
|
6
6
|
import PQueue from "p-queue";
|
|
7
7
|
import { z } from "zod";
|
|
8
8
|
import { type Finished, type Runner, startRunner } from "../src/adapters/claude-p.ts";
|
|
9
|
+
import { gitIn, readGitIdentity } from "../src/adapters/git.ts";
|
|
9
10
|
import { modelCallOver } from "../src/adapters/model-call.ts";
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import { alignmentQuestion, raisesLevel } from "../src/decide/alignment.ts";
|
|
11
|
+
import { BOUNDARIES, type Boundary } from "../src/config/schema.ts";
|
|
12
|
+
import { alignmentQuestion, doesRaiseLevel } from "../src/decide/alignment.ts";
|
|
13
13
|
import { replyQuestion } from "../src/decide/answer.ts";
|
|
14
14
|
import { kindQuestion } from "../src/decide/kind.ts";
|
|
15
15
|
import { tasteQuestion } from "../src/decide/taste.ts";
|
|
@@ -17,17 +17,21 @@ import { pairQuestion } from "../src/loop/pair.ts";
|
|
|
17
17
|
import type { ReviewNote } from "../src/loop/records.ts";
|
|
18
18
|
import { pairing, prepare } from "../src/loop/signals.ts";
|
|
19
19
|
import type { JsonSchema, ProsePurpose } from "../src/decide/prose.ts";
|
|
20
|
+
import { SHIPPED_COMMAND_TIMEOUT_MS } from "../src/exec.ts";
|
|
21
|
+
import { shippedRaw, shippedSettingsAs } from "../src/setup/shipped.ts";
|
|
20
22
|
import { openState } from "../src/state/open.ts";
|
|
21
23
|
import { decideProse } from "../src/worker/prose-run.ts";
|
|
22
24
|
import { PROSE_PROMPTS } from "../src/worker/thin-path.ts";
|
|
23
25
|
import { SECOND_MS } from "../src/time.ts";
|
|
26
|
+
import { rateOf } from "../src/measure/rate.ts";
|
|
24
27
|
|
|
25
28
|
/**
|
|
26
29
|
* Measures one model-judgment boundary on its labelled cases, through the same prompts and fences the engine runs,
|
|
27
30
|
* and adds a row to its RESULTS.md. `bun run eval -- --model <id>` measures every boundary on that model id; naming
|
|
28
31
|
* boundaries measures only those. A case the runner held or limited counts in no rate. `--runs <n>` repeats each
|
|
29
32
|
* boundary and pools the runs, since one run proves little; `--write <folder>` records each pooled rate under
|
|
30
|
-
* `measuredModels` in that folder's `hum.json`, which is how setup measures an owner's own models.
|
|
33
|
+
* `measuredModels` in that folder's `hum.json`, which is how setup measures an owner's own models. A boundary where
|
|
34
|
+
* nothing was measured records no rate, and the script exits non-zero.
|
|
31
35
|
*/
|
|
32
36
|
|
|
33
37
|
const { values, positionals } = parseArgs({ allowPositionals: true, options: { model: { type: "string" }, limit: { type: "string" }, runs: { type: "string" }, write: { type: "string" } } });
|
|
@@ -36,16 +40,23 @@ const OWNER_LABELLED: readonly Boundary[] = ["taste"];
|
|
|
36
40
|
|
|
37
41
|
const boundaries: readonly Boundary[] = positionals.length > 0 ? z.array(z.enum(BOUNDARIES)).parse(positionals) : BOUNDARIES.filter((boundary) => !OWNER_LABELLED.includes(boundary));
|
|
38
42
|
const root = join(import.meta.dir, "..");
|
|
39
|
-
const
|
|
40
|
-
|
|
43
|
+
const settings = shippedSettingsAs(await readGitIdentity(gitIn(process.cwd(), { timeoutMs: SHIPPED_COMMAND_TIMEOUT_MS })));
|
|
44
|
+
|
|
45
|
+
if (!settings.ok) {
|
|
46
|
+
console.error(settings.refusal.message);
|
|
47
|
+
process.exit(1);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const config = { hum: settings.value, roles: shippedRaw().roles };
|
|
41
51
|
const model = values.model ?? config.hum.decisionModel;
|
|
42
|
-
const
|
|
43
|
-
const
|
|
52
|
+
const Count = z.coerce.number().int().positive();
|
|
53
|
+
const runs = values.runs === undefined ? 1 : Count.parse(values.runs);
|
|
54
|
+
const limit = values.limit === undefined ? Number.POSITIVE_INFINITY : Count.parse(values.limit);
|
|
44
55
|
|
|
45
56
|
type Verdict = "right" | "wrong" | "abstained" | "halted";
|
|
46
57
|
|
|
47
|
-
const state = openState(mkdtempSync(join(tmpdir(), "hum-eval-")));
|
|
48
|
-
const started = await startRunner({ state, model });
|
|
58
|
+
const state = openState(mkdtempSync(join(tmpdir(), "hum-eval-")), { now: Date.now });
|
|
59
|
+
const started = await startRunner({ state, model, now: Date.now, env: process.env });
|
|
49
60
|
|
|
50
61
|
if (started.check.kind !== "finished") throw new Error(`the start check ended as ${started.check.kind}`);
|
|
51
62
|
|
|
@@ -109,7 +120,7 @@ const TasteCase = z.object({ case: z.string(), label: z.enum(["keeps", "breaks"]
|
|
|
109
120
|
/** Calls at once: enough to finish a set in minutes, few enough not to trip a rate limit. */
|
|
110
121
|
const AT_ONCE = 4;
|
|
111
122
|
|
|
112
|
-
const eachOf = <T>(cases: readonly T[], measure: (found: T) => Promise<
|
|
123
|
+
const eachOf = <T, R>(cases: readonly T[], measure: (found: T) => Promise<R>): Promise<R[]> => new PQueue({ concurrency: AT_ONCE }).addAll(cases.map((found) => () => measure(found)));
|
|
113
124
|
|
|
114
125
|
const read = async <T>(boundary: Boundary, schema: z.ZodType<T>, file = "cases.jsonl"): Promise<T[]> =>
|
|
115
126
|
(await Bun.file(join(root, "evals", boundary, file)).text())
|
|
@@ -118,78 +129,98 @@ const read = async <T>(boundary: Boundary, schema: z.ZodType<T>, file = "cases.j
|
|
|
118
129
|
.map((line) => schema.parse(JSON.parse(line)));
|
|
119
130
|
|
|
120
131
|
/** The confusion counts of the review-note boundary: its rate is precision, and recall is recorded beside it. */
|
|
121
|
-
|
|
132
|
+
type PairCounts = { readonly tp: number; readonly fp: number; readonly fn: number; readonly tn: number; readonly pairedBySignals: number };
|
|
133
|
+
|
|
134
|
+
type Cell = Exclude<keyof PairCounts, "pairedBySignals">;
|
|
135
|
+
|
|
136
|
+
const NO_PAIRS: PairCounts = { tp: 0, fp: 0, fn: 0, tn: 0, pairedBySignals: 0 };
|
|
137
|
+
|
|
138
|
+
const addPairs = (a: PairCounts, b: PairCounts): PairCounts => ({ tp: a.tp + b.tp, fp: a.fp + b.fp, fn: a.fn + b.fn, tn: a.tn + b.tn, pairedBySignals: a.pairedBySignals + b.pairedBySignals });
|
|
139
|
+
|
|
140
|
+
/** One run of a boundary: each case's verdict, and for the review-note boundary its confusion counts. */
|
|
141
|
+
type Measured = { readonly verdicts: readonly Verdict[]; readonly pairs?: PairCounts };
|
|
122
142
|
|
|
123
143
|
const noteOf = (text: string, sourceId: string): ReviewNote => ({ source: "review-comment", sourceId, change: sourceId, hunk: "", text, at: 0 });
|
|
124
144
|
|
|
125
|
-
const BOUNDARY_RUNS: Record<Boundary, () => Promise<
|
|
145
|
+
const BOUNDARY_RUNS: Record<Boundary, () => Promise<Measured>> = {
|
|
126
146
|
kind: async () => {
|
|
127
147
|
const cases = (await read("kind", KindCase)).slice(0, limit);
|
|
128
148
|
|
|
129
|
-
return
|
|
149
|
+
return {
|
|
150
|
+
verdicts: await eachOf(cases, async (found) => {
|
|
130
151
|
const judged = await judge(kindQuestion(found, config.hum));
|
|
131
152
|
|
|
132
153
|
return judged.kind === "abstained" ? "abstained" : scored(judged.value.kind, found.label);
|
|
133
|
-
|
|
154
|
+
}),
|
|
155
|
+
};
|
|
134
156
|
},
|
|
135
157
|
alignment: async () => {
|
|
136
158
|
const sample = (await read("alignment", AlignmentCase, "sample.jsonl")).slice(0, limit);
|
|
137
159
|
|
|
138
|
-
return
|
|
160
|
+
return {
|
|
161
|
+
verdicts: await eachOf(sample, async (found): Promise<Verdict> => {
|
|
139
162
|
const judged = await judge(alignmentQuestion({ title: found.subject, body: "" }, found.diff, config.hum));
|
|
140
163
|
|
|
141
164
|
if (judged.kind === "abstained") return "abstained";
|
|
142
165
|
|
|
143
|
-
return
|
|
144
|
-
|
|
166
|
+
return doesRaiseLevel({ rank: 0, levels: config.hum.levels.length, answer: judged.value, config: config.hum }) === (found.label !== "clean") ? "right" : "wrong";
|
|
167
|
+
}),
|
|
168
|
+
};
|
|
145
169
|
},
|
|
146
170
|
"plan-outcome": async () => {
|
|
147
171
|
const repo = planRepo();
|
|
148
172
|
const cases = (await read("plan-outcome", PlanCase)).slice(0, limit);
|
|
149
173
|
|
|
150
|
-
return eachOf(cases, async (found) => scored(await prosePick("plan", PROSE_PROMPTS.plan(config, { id: found.case, title: found.body.split("\n")[0] ?? "", body: found.body }), "", repo), found.label));
|
|
174
|
+
return { verdicts: await eachOf(cases, async (found) => scored(await prosePick("plan", PROSE_PROMPTS.plan(config, { id: found.case, title: found.body.split("\n")[0] ?? "", body: found.body }), "", repo), found.label)) };
|
|
151
175
|
},
|
|
152
176
|
"reply-option": async () => {
|
|
153
177
|
const cases = (await read("reply-option", ReplyCase)).slice(0, limit);
|
|
154
178
|
|
|
155
|
-
return
|
|
156
|
-
|
|
179
|
+
return {
|
|
180
|
+
verdicts: await eachOf(cases, async (found) => {
|
|
181
|
+
const judged = await judge(replyQuestion({ question: found.question, options: found.options, reply: found.reply }));
|
|
157
182
|
|
|
158
|
-
|
|
159
|
-
|
|
183
|
+
return judged.kind === "abstained" ? "abstained" : scored(judged.value.option, found.label);
|
|
184
|
+
}),
|
|
185
|
+
};
|
|
160
186
|
},
|
|
161
187
|
"repeat-pair": async () => {
|
|
162
188
|
const cases = (await read("repeat-pair", PairCase)).slice(0, limit);
|
|
163
189
|
const open = cases.filter((found) => pairing(prepare(noteOf(found.a.text, "a")), prepare(noteOf(found.b.text, "b"))).signal === undefined);
|
|
164
190
|
|
|
165
|
-
|
|
191
|
+
const judged = await eachOf(open, async (found): Promise<{ verdict: Verdict; cell?: Cell }> => {
|
|
192
|
+
const pair = await judge(pairQuestion(noteOf(found.a.text, "a"), noteOf(found.b.text, "b")));
|
|
166
193
|
|
|
167
|
-
|
|
168
|
-
const judged = await judge(pairQuestion(noteOf(found.a.text, "a"), noteOf(found.b.text, "b")));
|
|
194
|
+
if (pair.kind === "abstained") return { verdict: "abstained" };
|
|
169
195
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
const said = judged.value.same;
|
|
196
|
+
const said = pair.value.same;
|
|
173
197
|
const truth = found.label === "same";
|
|
174
198
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
return said === truth ? "right" : "wrong";
|
|
199
|
+
return { verdict: said === truth ? "right" : "wrong", cell: said ? (truth ? "tp" : "fp") : truth ? "fn" : "tn" };
|
|
178
200
|
});
|
|
201
|
+
|
|
202
|
+
const countOf = (cell: Cell) => judged.filter((found) => found.cell === cell).length;
|
|
203
|
+
|
|
204
|
+
return {
|
|
205
|
+
verdicts: judged.map(({ verdict }) => verdict),
|
|
206
|
+
pairs: { tp: countOf("tp"), fp: countOf("fp"), fn: countOf("fn"), tn: countOf("tn"), pairedBySignals: cases.length - open.length },
|
|
207
|
+
};
|
|
179
208
|
},
|
|
180
209
|
taste: async () => {
|
|
181
210
|
const cases = (await read("taste", TasteCase)).slice(0, limit);
|
|
182
211
|
|
|
183
|
-
return
|
|
184
|
-
|
|
212
|
+
return {
|
|
213
|
+
verdicts: await eachOf(cases, async (found) => {
|
|
214
|
+
const judged = await judge(tasteQuestion(found.rule, found.diff, config.hum));
|
|
185
215
|
|
|
186
|
-
|
|
187
|
-
|
|
216
|
+
return judged.kind === "abstained" ? "abstained" : scored(judged.value.keeps ? "keeps" : "breaks", found.label);
|
|
217
|
+
}),
|
|
218
|
+
};
|
|
188
219
|
},
|
|
189
220
|
"review-verdict": async () => {
|
|
190
221
|
const cases = (await read("review-verdict", ReviewCase)).slice(0, limit);
|
|
191
222
|
|
|
192
|
-
return eachOf(cases, async (found) => scored(await prosePick("review", PROSE_PROMPTS.review(config, { id: found.case, title: found.item, body: "" }, found.diff), found.diff, tmpdir()), found.label));
|
|
223
|
+
return { verdicts: await eachOf(cases, async (found) => scored(await prosePick("review", PROSE_PROMPTS.review(config, { id: found.case, title: found.item, body: "" }, found.diff), found.diff, tmpdir()), found.label)) };
|
|
193
224
|
},
|
|
194
225
|
};
|
|
195
226
|
|
|
@@ -197,12 +228,13 @@ const header = "| Date | Model id | n | Right | Wrong | Abstained | Rate | Live
|
|
|
197
228
|
|
|
198
229
|
const pairHeader = "| Date | Model id | n | TP | FP | FN | TN | Abstained | Paired by signals | Precision | Recall | Live calls |\n|---|---|---|---|---|---|---|---|---|---|---|---|\n";
|
|
199
230
|
|
|
200
|
-
|
|
231
|
+
/** A rate as a RESULTS.md cell; a dash where nothing was measured. */
|
|
232
|
+
const rateCell = (rate: number | undefined): string => rate?.toFixed(2) ?? "-";
|
|
201
233
|
|
|
202
|
-
const
|
|
203
|
-
`| ${day} | ${model} | ${pairCounts.tp + pairCounts.fp + pairCounts.fn + pairCounts.tn} | ${pairCounts.tp} | ${pairCounts.fp} | ${pairCounts.fn} | ${pairCounts.tn} | ${abstained} | ${pairCounts.pairedBySignals} | ${ratio(pairCounts.tp, pairCounts.tp + pairCounts.fp)} | ${ratio(pairCounts.tp, pairCounts.tp + pairCounts.fn)} | ${used} |`;
|
|
234
|
+
const precisionOf = ({ tp, fp }: PairCounts): number | undefined => rateOf(tp, tp + fp);
|
|
204
235
|
|
|
205
|
-
const
|
|
236
|
+
const pairRow = (day: string, pairs: PairCounts, abstained: number, used: number): string =>
|
|
237
|
+
`| ${day} | ${model} | ${pairs.tp + pairs.fp + pairs.fn + pairs.tn} | ${pairs.tp} | ${pairs.fp} | ${pairs.fn} | ${pairs.tn} | ${abstained} | ${pairs.pairedBySignals} | ${rateCell(precisionOf(pairs))} | ${rateCell(rateOf(pairs.tp, pairs.tp + pairs.fn))} | ${used} |`;
|
|
206
238
|
|
|
207
239
|
/** Sets one pooled rate in the owner's `hum.json`, where trust reads it. */
|
|
208
240
|
const recordRate = async (folder: string, boundary: Boundary, rate: number): Promise<void> => {
|
|
@@ -212,33 +244,39 @@ const recordRate = async (folder: string, boundary: Boundary, rate: number): Pro
|
|
|
212
244
|
await Bun.write(file, `${JSON.stringify({ ...written, measuredModels: { ...written.measuredModels, [boundary]: { ...written.measuredModels[boundary], [model]: Number(rate.toFixed(2)) } } }, null, "\t")}\n`);
|
|
213
245
|
};
|
|
214
246
|
|
|
215
|
-
|
|
216
|
-
const pool = { ...EMPTY_POOL };
|
|
247
|
+
const unmeasured: Boundary[] = [];
|
|
217
248
|
|
|
218
|
-
|
|
249
|
+
for (const boundary of boundaries) {
|
|
250
|
+
const pool = { right: 0, measured: 0, pairs: NO_PAIRS };
|
|
219
251
|
|
|
220
252
|
for (let run = 1; run <= runs; run++) {
|
|
221
253
|
const before = calls;
|
|
222
|
-
const verdicts = await BOUNDARY_RUNS[boundary]();
|
|
254
|
+
const { verdicts, pairs = NO_PAIRS } = await BOUNDARY_RUNS[boundary]();
|
|
223
255
|
const count = (verdict: Verdict) => verdicts.filter((found) => found === verdict).length;
|
|
224
256
|
const measured = verdicts.length - count("halted");
|
|
225
|
-
const rate = measured === 0 ? 0 : count("right") / measured;
|
|
226
257
|
const day = new Date().toISOString().slice(0, 10);
|
|
227
|
-
const row = boundary === "repeat-pair" ? pairRow(day, count("abstained"), calls - before) : `| ${day} | ${model} | ${measured} | ${count("right")} | ${count("wrong")} | ${count("abstained")} | ${
|
|
258
|
+
const row = boundary === "repeat-pair" ? pairRow(day, pairs, count("abstained"), calls - before) : `| ${day} | ${model} | ${measured} | ${count("right")} | ${count("wrong")} | ${count("abstained")} | ${rateCell(rateOf(count("right"), measured))} | ${calls - before} |`;
|
|
228
259
|
const results = join(values.write ?? root, "evals", boundary, "RESULTS.md");
|
|
229
260
|
const earlier = (await Bun.file(results).exists()) ? await Bun.file(results).text() : boundary === "repeat-pair" ? pairHeader : header;
|
|
230
261
|
|
|
231
262
|
pool.right += count("right");
|
|
232
263
|
pool.measured += measured;
|
|
264
|
+
pool.pairs = addPairs(pool.pairs, pairs);
|
|
233
265
|
await Bun.write(results, `${earlier}${row}\n`);
|
|
234
266
|
console.log(`${boundary}: ${row}`);
|
|
235
267
|
}
|
|
236
268
|
|
|
237
|
-
const pooled = boundary === "repeat-pair" ?
|
|
269
|
+
const pooled = boundary === "repeat-pair" ? precisionOf(pool.pairs) : rateOf(pool.right, pool.measured);
|
|
238
270
|
|
|
239
|
-
console.log(`${boundary}: pooled over ${runs} run(s): ${pooled
|
|
271
|
+
console.log(`${boundary}: pooled over ${runs} run(s): ${rateCell(pooled)}`);
|
|
272
|
+
|
|
273
|
+
if (pooled === undefined) unmeasured.push(boundary);
|
|
274
|
+
else if (values.write !== undefined) await recordRate(values.write, boundary, pooled);
|
|
275
|
+
}
|
|
240
276
|
|
|
241
|
-
|
|
277
|
+
if (unmeasured.length > 0) {
|
|
278
|
+
console.error(`Nothing was measured on ${unmeasured.join(", ")}, so no rate was recorded for ${unmeasured.length === 1 ? "it" : "them"}.`);
|
|
279
|
+
process.exitCode = 1;
|
|
242
280
|
}
|
|
243
281
|
|
|
244
282
|
console.log(`${calls} live calls, the start check included. Record each rate under measuredModels in defaults/hum.json; a model decides a boundary alone only at minJudgeRate or above; repeat-pair takes its precision, against repeatPrecision.`);
|