@moo.md/hum 0.4.1 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/defaults/hum.json +39 -12
- package/npm-shrinkwrap.json +2 -2
- package/package.json +2 -1
- package/release.json +1 -1
- package/schema/hum.schema.json +63 -17
- package/scripts/eval.ts +58 -27
- package/src/adapters/claude-p.ts +7 -0
- package/src/adapters/model-call.ts +8 -4
- package/src/adapters/pull-requests.ts +11 -7
- package/src/cli/setup.ts +13 -18
- package/src/config/json.ts +3 -0
- package/src/config/migrate.ts +11 -0
- package/src/config/schema.ts +21 -6
- package/src/decide/alignment.ts +1 -1
- package/src/decide/merge.ts +21 -1
- package/src/decide/prose.ts +12 -2
- package/src/decide/risk.ts +1 -0
- package/src/loop/breaks.ts +17 -0
- package/src/loop/notes.ts +2 -2
- package/src/loop/pair.ts +2 -2
- package/src/loop/pass.ts +6 -3
- package/src/loop/records.ts +24 -1
- package/src/loop/signals.ts +2 -2
- package/src/measure/cases.ts +102 -0
- package/src/measure/inputs.ts +15 -1
- package/src/measure/judges.ts +52 -0
- package/src/measure/measure.ts +22 -3
- package/src/measure/rate.ts +9 -6
- package/src/measure/remeasure.ts +31 -0
- package/src/publish/handoff.ts +1 -1
- package/src/service/workflows.ts +4 -2
- package/src/setup/write.ts +2 -4
- package/src/state/events.ts +201 -19
- package/src/state/format.ts +9 -2
- package/src/state/item-log.ts +133 -0
- package/src/state/items.ts +6 -5
- package/src/state/open.ts +7 -2
- package/src/state/runs.ts +5 -0
- package/src/state/saved.ts +7 -2
- package/src/status/format.ts +12 -3
- package/src/worker/land.ts +36 -15
- package/src/worker/merged.ts +18 -8
- package/src/worker/open.ts +5 -1
- package/src/worker/prose-run.ts +6 -2
- package/src/worker/signature.ts +11 -5
- package/src/worker/thin-path.ts +69 -53
- package/src/worker/worker.ts +82 -12
- package/surface.json +969 -151
package/defaults/hum.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"schemaVersion":
|
|
2
|
+
"schemaVersion": 3,
|
|
3
3
|
"levels": [
|
|
4
4
|
{
|
|
5
5
|
"name": "low",
|
|
@@ -212,28 +212,55 @@
|
|
|
212
212
|
"decisionModel": "claude-haiku-5-5",
|
|
213
213
|
"modelCallTimeoutSeconds": 120,
|
|
214
214
|
"minJudgeRate": 0.8,
|
|
215
|
+
"evalRuns": 3,
|
|
215
216
|
"measuredModels": {
|
|
216
217
|
"kind": {
|
|
217
|
-
"claude-haiku-5-5":
|
|
218
|
-
|
|
218
|
+
"claude-haiku-5-5": {
|
|
219
|
+
"rate": 0.58,
|
|
220
|
+
"judgeHash": "38d9c55d5a9ecca44ee320fd408cc34140fe32219a9d199f6d7114fced6800ef"
|
|
221
|
+
},
|
|
222
|
+
"claude-sonnet-5-5": {
|
|
223
|
+
"rate": 0.63,
|
|
224
|
+
"judgeHash": "38d9c55d5a9ecca44ee320fd408cc34140fe32219a9d199f6d7114fced6800ef"
|
|
225
|
+
}
|
|
219
226
|
},
|
|
220
227
|
"alignment": {
|
|
221
|
-
"claude-haiku-5-5":
|
|
222
|
-
|
|
228
|
+
"claude-haiku-5-5": {
|
|
229
|
+
"rate": 0.68,
|
|
230
|
+
"judgeHash": "dc59bc1941fa786d738dd1d8810a4d0644b8e672dd4239bea340102d2960fbbd"
|
|
231
|
+
},
|
|
232
|
+
"claude-sonnet-5-5": {
|
|
233
|
+
"rate": 0.68,
|
|
234
|
+
"judgeHash": "dc59bc1941fa786d738dd1d8810a4d0644b8e672dd4239bea340102d2960fbbd"
|
|
235
|
+
}
|
|
223
236
|
},
|
|
224
237
|
"plan-outcome": {
|
|
225
|
-
"claude-haiku-5-5":
|
|
226
|
-
|
|
238
|
+
"claude-haiku-5-5": {
|
|
239
|
+
"rate": 1
|
|
240
|
+
},
|
|
241
|
+
"claude-sonnet-5-5": {
|
|
242
|
+
"rate": 1
|
|
243
|
+
}
|
|
227
244
|
},
|
|
228
245
|
"review-verdict": {
|
|
229
|
-
"claude-haiku-5-5":
|
|
230
|
-
|
|
246
|
+
"claude-haiku-5-5": {
|
|
247
|
+
"rate": 0.92
|
|
248
|
+
},
|
|
249
|
+
"claude-sonnet-5-5": {
|
|
250
|
+
"rate": 0.83
|
|
251
|
+
}
|
|
231
252
|
},
|
|
232
253
|
"reply-option": {
|
|
233
|
-
"claude-haiku-5-5":
|
|
254
|
+
"claude-haiku-5-5": {
|
|
255
|
+
"rate": 0.87,
|
|
256
|
+
"judgeHash": "75d15bd34fb5907209ae421ff529574447cda9eff461173f2d8ad6771e2ddd47"
|
|
257
|
+
}
|
|
234
258
|
},
|
|
235
259
|
"repeat-pair": {
|
|
236
|
-
"claude-haiku-5-5":
|
|
260
|
+
"claude-haiku-5-5": {
|
|
261
|
+
"rate": 1,
|
|
262
|
+
"judgeHash": "fceeafd3501e0597d2c460ce2ff88021a1746eee91fdc18adc56a334b711d331"
|
|
263
|
+
}
|
|
237
264
|
},
|
|
238
265
|
"taste": {}
|
|
239
266
|
},
|
|
@@ -330,4 +357,4 @@
|
|
|
330
357
|
"mac",
|
|
331
358
|
"ci"
|
|
332
359
|
]
|
|
333
|
-
}
|
|
360
|
+
}
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@moo.md/hum",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@moo.md/hum",
|
|
9
|
-
"version": "0.
|
|
9
|
+
"version": "0.6.1",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@actions/core": "3.0.1",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@moo.md/hum",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"description": "The hum engine: work items become Claude Code agent runs, the repo's checks decide what they can, and only what nothing settles becomes a question to the owner.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": {
|
|
@@ -87,6 +87,7 @@
|
|
|
87
87
|
"@types/proper-lockfile": "4.1.4",
|
|
88
88
|
"@types/semver": "7.8.0",
|
|
89
89
|
"@types/write-file-atomic": "4.0.3",
|
|
90
|
+
"graphql": "17.1.0",
|
|
90
91
|
"microdiff": "1.6.0",
|
|
91
92
|
"oxlint": "1.89.0",
|
|
92
93
|
"typescript": "5.9.2"
|
package/release.json
CHANGED
package/schema/hum.schema.json
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
"properties": {
|
|
5
5
|
"schemaVersion": {
|
|
6
6
|
"type": "number",
|
|
7
|
-
"const":
|
|
7
|
+
"const": 3
|
|
8
8
|
},
|
|
9
9
|
"levels": {
|
|
10
10
|
"minItems": 1,
|
|
@@ -375,7 +375,7 @@
|
|
|
375
375
|
"type": "integer",
|
|
376
376
|
"minimum": 0,
|
|
377
377
|
"maximum": 9007199254740991,
|
|
378
|
-
"description": "Fix runs an item gets after failed checks before it ends stuck."
|
|
378
|
+
"description": "Fix runs an item gets after failed checks, counted since the owner last answered, before it ends stuck."
|
|
379
379
|
},
|
|
380
380
|
"serviceLabel": {
|
|
381
381
|
"default": "md.moo.hum",
|
|
@@ -1084,26 +1084,52 @@
|
|
|
1084
1084
|
"measuredModels": {
|
|
1085
1085
|
"default": {
|
|
1086
1086
|
"kind": {
|
|
1087
|
-
"claude-haiku-5-5":
|
|
1088
|
-
|
|
1087
|
+
"claude-haiku-5-5": {
|
|
1088
|
+
"rate": 0.58,
|
|
1089
|
+
"judgeHash": "38d9c55d5a9ecca44ee320fd408cc34140fe32219a9d199f6d7114fced6800ef"
|
|
1090
|
+
},
|
|
1091
|
+
"claude-sonnet-5-5": {
|
|
1092
|
+
"rate": 0.63,
|
|
1093
|
+
"judgeHash": "38d9c55d5a9ecca44ee320fd408cc34140fe32219a9d199f6d7114fced6800ef"
|
|
1094
|
+
}
|
|
1089
1095
|
},
|
|
1090
1096
|
"alignment": {
|
|
1091
|
-
"claude-haiku-5-5":
|
|
1092
|
-
|
|
1097
|
+
"claude-haiku-5-5": {
|
|
1098
|
+
"rate": 0.68,
|
|
1099
|
+
"judgeHash": "dc59bc1941fa786d738dd1d8810a4d0644b8e672dd4239bea340102d2960fbbd"
|
|
1100
|
+
},
|
|
1101
|
+
"claude-sonnet-5-5": {
|
|
1102
|
+
"rate": 0.68,
|
|
1103
|
+
"judgeHash": "dc59bc1941fa786d738dd1d8810a4d0644b8e672dd4239bea340102d2960fbbd"
|
|
1104
|
+
}
|
|
1093
1105
|
},
|
|
1094
1106
|
"plan-outcome": {
|
|
1095
|
-
"claude-haiku-5-5":
|
|
1096
|
-
|
|
1107
|
+
"claude-haiku-5-5": {
|
|
1108
|
+
"rate": 1
|
|
1109
|
+
},
|
|
1110
|
+
"claude-sonnet-5-5": {
|
|
1111
|
+
"rate": 1
|
|
1112
|
+
}
|
|
1097
1113
|
},
|
|
1098
1114
|
"review-verdict": {
|
|
1099
|
-
"claude-haiku-5-5":
|
|
1100
|
-
|
|
1115
|
+
"claude-haiku-5-5": {
|
|
1116
|
+
"rate": 0.92
|
|
1117
|
+
},
|
|
1118
|
+
"claude-sonnet-5-5": {
|
|
1119
|
+
"rate": 0.83
|
|
1120
|
+
}
|
|
1101
1121
|
},
|
|
1102
1122
|
"reply-option": {
|
|
1103
|
-
"claude-haiku-5-5":
|
|
1123
|
+
"claude-haiku-5-5": {
|
|
1124
|
+
"rate": 0.87,
|
|
1125
|
+
"judgeHash": "75d15bd34fb5907209ae421ff529574447cda9eff461173f2d8ad6771e2ddd47"
|
|
1126
|
+
}
|
|
1104
1127
|
},
|
|
1105
1128
|
"repeat-pair": {
|
|
1106
|
-
"claude-haiku-5-5":
|
|
1129
|
+
"claude-haiku-5-5": {
|
|
1130
|
+
"rate": 1,
|
|
1131
|
+
"judgeHash": "fceeafd3501e0597d2c460ce2ff88021a1746eee91fdc18adc56a334b711d331"
|
|
1132
|
+
}
|
|
1107
1133
|
},
|
|
1108
1134
|
"taste": {}
|
|
1109
1135
|
},
|
|
@@ -1127,9 +1153,22 @@
|
|
|
1127
1153
|
"minLength": 1
|
|
1128
1154
|
},
|
|
1129
1155
|
"additionalProperties": {
|
|
1130
|
-
"type": "
|
|
1131
|
-
"
|
|
1132
|
-
|
|
1156
|
+
"type": "object",
|
|
1157
|
+
"properties": {
|
|
1158
|
+
"rate": {
|
|
1159
|
+
"type": "number",
|
|
1160
|
+
"minimum": 0,
|
|
1161
|
+
"maximum": 1
|
|
1162
|
+
},
|
|
1163
|
+
"judgeHash": {
|
|
1164
|
+
"type": "string",
|
|
1165
|
+
"pattern": "^[0-9a-f]{64}$"
|
|
1166
|
+
}
|
|
1167
|
+
},
|
|
1168
|
+
"required": [
|
|
1169
|
+
"rate"
|
|
1170
|
+
],
|
|
1171
|
+
"additionalProperties": false
|
|
1133
1172
|
}
|
|
1134
1173
|
},
|
|
1135
1174
|
"required": [
|
|
@@ -1141,7 +1180,7 @@
|
|
|
1141
1180
|
"repeat-pair",
|
|
1142
1181
|
"taste"
|
|
1143
1182
|
],
|
|
1144
|
-
"description": "Per model-judgment boundary,
|
|
1183
|
+
"description": "Per model-judgment boundary, each model id's rate on its labelled set, and the judgeHash of the prompt, role text and answer form its judge read then; a pick from an id below 'minJudgeRate', or not measured, counts as an abstention. A rate whose judge has changed since keeps gating until a new measurement replaces it: the Mac worker measures it again."
|
|
1145
1184
|
},
|
|
1146
1185
|
"minJudgeRate": {
|
|
1147
1186
|
"default": 0.8,
|
|
@@ -1150,6 +1189,13 @@
|
|
|
1150
1189
|
"maximum": 1,
|
|
1151
1190
|
"description": "The lowest measured rate at which hum trusts a model's pick at a boundary; below it the call abstains."
|
|
1152
1191
|
},
|
|
1192
|
+
"evalRuns": {
|
|
1193
|
+
"default": 3,
|
|
1194
|
+
"type": "integer",
|
|
1195
|
+
"exclusiveMinimum": 0,
|
|
1196
|
+
"maximum": 9007199254740991,
|
|
1197
|
+
"description": "Runs pooled per boundary each time hum measures a model: in `hum setup measure`, and when a judge's prompt or role text changed since its rate was measured."
|
|
1198
|
+
},
|
|
1153
1199
|
"questionLimits": {
|
|
1154
1200
|
"default": {
|
|
1155
1201
|
"cells": 80,
|
|
@@ -1485,7 +1531,7 @@
|
|
|
1485
1531
|
"type": "integer",
|
|
1486
1532
|
"exclusiveMinimum": 0,
|
|
1487
1533
|
"maximum": 9007199254740991,
|
|
1488
|
-
"description": "The time between passes that collect review notes, file check items and confirm CI ran their checks."
|
|
1534
|
+
"description": "The time between passes that collect review notes, file check items and confirm CI ran their checks, and between tries to measure again a judge whose prompt or role text changed."
|
|
1489
1535
|
},
|
|
1490
1536
|
"proofTimeoutSeconds": {
|
|
1491
1537
|
"default": 900,
|
package/scripts/eval.ts
CHANGED
|
@@ -8,6 +8,9 @@ import { z } from "zod";
|
|
|
8
8
|
import { type Finished, type Runner, startRunner } from "../src/adapters/claude-p.ts";
|
|
9
9
|
import { gitIn, readGitIdentity } from "../src/adapters/git.ts";
|
|
10
10
|
import { modelCallOver } from "../src/adapters/model-call.ts";
|
|
11
|
+
import { loadConfig, readConfigDir } from "../src/config/load.ts";
|
|
12
|
+
import { migrateConfig } from "../src/config/migrate.ts";
|
|
13
|
+
import type { Result } from "../src/config/refusal.ts";
|
|
11
14
|
import { BOUNDARIES, type Boundary } from "../src/config/schema.ts";
|
|
12
15
|
import { alignmentQuestion, doesRaiseLevel } from "../src/decide/alignment.ts";
|
|
13
16
|
import { replyQuestion } from "../src/decide/answer.ts";
|
|
@@ -19,35 +22,52 @@ import { pairing, prepare } from "../src/loop/signals.ts";
|
|
|
19
22
|
import type { JsonSchema, ProsePurpose } from "../src/decide/prose.ts";
|
|
20
23
|
import { SHIPPED_COMMAND_TIMEOUT_MS } from "../src/exec.ts";
|
|
21
24
|
import { shippedRaw, shippedSettingsAs } from "../src/setup/shipped.ts";
|
|
22
|
-
import { openState } from "../src/state/open.ts";
|
|
25
|
+
import { defaultStateDir, openState } from "../src/state/open.ts";
|
|
23
26
|
import { decideProse } from "../src/worker/prose-run.ts";
|
|
24
|
-
import { PROSE_PROMPTS } from "../src/worker/thin-path.ts";
|
|
27
|
+
import { PROSE_PROMPTS, type RunConfig } from "../src/worker/thin-path.ts";
|
|
25
28
|
import { SECOND_MS } from "../src/time.ts";
|
|
26
29
|
import { rateOf, withRate } from "../src/measure/rate.ts";
|
|
30
|
+
import { judgeHash, SHIPPED_SETS } from "../src/measure/judges.ts";
|
|
31
|
+
import { EVALS_DIR } from "../src/measure/remeasure.ts";
|
|
32
|
+
import { AlignmentCase, MERGED_CASES, ReviewCase } from "../src/measure/cases.ts";
|
|
27
33
|
|
|
28
34
|
/**
|
|
29
35
|
* Measures one model-judgment boundary on its labelled cases, through the same prompts and fences the engine runs,
|
|
30
36
|
* and adds a row to its RESULTS.md. `bun run eval -- --model <id>` measures every boundary on that model id; naming
|
|
31
37
|
* boundaries measures only those. A case the runner held or limited counts in no rate. `--runs <n>` repeats each
|
|
32
|
-
* boundary and pools the runs, since one run proves little
|
|
33
|
-
*
|
|
34
|
-
*
|
|
38
|
+
* boundary and pools the runs, since one run proves little. `--write <folder>` measures the judges as the config in
|
|
39
|
+
* that `.hum/`-shaped folder has them, its role files and kind meanings with them, and records each pooled rate with
|
|
40
|
+
* its judge's hash under `measuredModels` in the folder's `hum.json`, in one write once every boundary is done: that
|
|
41
|
+
* is how setup, and the Mac worker when a judge changed, measure an owner's own models. The alignment and review sets
|
|
42
|
+
* then take the cases the repo's merged items gave them too. Without it the judges and sets are the shipped ones. A
|
|
43
|
+
* boundary where nothing was measured records no rate, and the script exits non-zero.
|
|
35
44
|
*/
|
|
36
45
|
|
|
37
46
|
const { values, positionals } = parseArgs({ allowPositionals: true, options: { model: { type: "string" }, limit: { type: "string" }, runs: { type: "string" }, write: { type: "string" } } });
|
|
38
|
-
/** Boundaries whose labelled cases only the owner can give; they are measured when named, from the cases the owner puts in their evals folder. */
|
|
39
|
-
const OWNER_LABELLED: readonly Boundary[] = ["taste"];
|
|
40
47
|
|
|
41
|
-
const boundaries: readonly Boundary[] = positionals.length > 0 ? z.array(z.enum(BOUNDARIES)).parse(positionals) :
|
|
48
|
+
const boundaries: readonly Boundary[] = positionals.length > 0 ? z.array(z.enum(BOUNDARIES)).parse(positionals) : SHIPPED_SETS;
|
|
42
49
|
const root = join(import.meta.dir, "..");
|
|
43
|
-
const settings = shippedSettingsAs(await readGitIdentity(gitIn(process.cwd(), { timeoutMs: SHIPPED_COMMAND_TIMEOUT_MS })));
|
|
44
50
|
|
|
45
|
-
|
|
46
|
-
|
|
51
|
+
/** The judges as an owner's `.hum/` folder has them, or as hum ships them, to the owner's git identity. */
|
|
52
|
+
const judgesConfig = async (folder: string | undefined): Promise<Result<RunConfig>> => {
|
|
53
|
+
if (folder !== undefined) return loadConfig(readConfigDir(folder), { env: {} });
|
|
54
|
+
|
|
55
|
+
const settings = shippedSettingsAs(await readGitIdentity(gitIn(process.cwd(), { timeoutMs: SHIPPED_COMMAND_TIMEOUT_MS })));
|
|
56
|
+
|
|
57
|
+
return settings.ok ? { ok: true, value: { hum: settings.value, roles: shippedRaw().roles } } : settings;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
const loaded = await judgesConfig(values.write);
|
|
61
|
+
|
|
62
|
+
if (!loaded.ok) {
|
|
63
|
+
console.error(loaded.refusal.message);
|
|
47
64
|
process.exit(1);
|
|
48
65
|
}
|
|
49
66
|
|
|
50
|
-
const config =
|
|
67
|
+
const config = loaded.value;
|
|
68
|
+
|
|
69
|
+
/** The cases merged items gave the judges, kept in the state of the repo whose `.hum/` is measured; the shipped judges take none. */
|
|
70
|
+
const merged = values.write === undefined ? [] : openState(await defaultStateDir(dirname(values.write)), { now: Date.now }).cases.list();
|
|
51
71
|
const model = values.model ?? config.hum.decisionModel;
|
|
52
72
|
const Count = z.coerce.number().int().positive();
|
|
53
73
|
const runs = values.runs === undefined ? 1 : Count.parse(values.runs);
|
|
@@ -124,10 +144,8 @@ const planRepo = (): string => {
|
|
|
124
144
|
};
|
|
125
145
|
|
|
126
146
|
const KindCase = z.object({ case: z.string(), label: z.string(), title: z.string(), body: z.string() });
|
|
127
|
-
const AlignmentCase = z.object({ case: z.string(), label: z.enum(["reverted", "fixed", "clean"]), subject: z.string(), diff: z.string() });
|
|
128
147
|
const PlanCase = z.object({ case: z.string(), label: z.string(), body: z.string() });
|
|
129
148
|
const ReplyCase = z.object({ case: z.string(), label: z.string(), question: z.string(), options: z.array(z.string()), reply: z.string() });
|
|
130
|
-
const ReviewCase = z.object({ case: z.string(), label: z.string(), item: z.string(), diff: z.string() });
|
|
131
149
|
const PairCase = z.object({ case: z.string(), label: z.enum(["same", "different"]), a: z.object({ text: z.string() }), b: z.object({ text: z.string() }) });
|
|
132
150
|
const TasteCase = z.object({ case: z.string(), label: z.enum(["keeps", "breaks"]), rule: z.string(), diff: z.string() });
|
|
133
151
|
|
|
@@ -137,7 +155,7 @@ const AT_ONCE = 4;
|
|
|
137
155
|
const eachOf = <T, R>(cases: readonly T[], measure: (found: T) => Promise<R>): Promise<R[]> => new PQueue({ concurrency: AT_ONCE }).addAll(cases.map((found) => () => measure(found)));
|
|
138
156
|
|
|
139
157
|
const read = async <T>(boundary: Boundary, schema: z.ZodType<T>, file = "cases.jsonl"): Promise<T[]> =>
|
|
140
|
-
(await Bun.file(join(root,
|
|
158
|
+
(await Bun.file(join(root, EVALS_DIR, boundary, file)).text())
|
|
141
159
|
.split("\n")
|
|
142
160
|
.filter(Boolean)
|
|
143
161
|
.map((line) => schema.parse(JSON.parse(line)));
|
|
@@ -169,11 +187,11 @@ const BOUNDARY_RUNS: Record<Boundary, () => Promise<Measured>> = {
|
|
|
169
187
|
};
|
|
170
188
|
},
|
|
171
189
|
alignment: async () => {
|
|
172
|
-
const sample = (await read("alignment", AlignmentCase, "sample.jsonl")).slice(0, limit);
|
|
190
|
+
const sample = [...(await read("alignment", AlignmentCase, "sample.jsonl")), ...MERGED_CASES.alignment(merged).map((found) => AlignmentCase.parse(found))].slice(0, limit);
|
|
173
191
|
|
|
174
192
|
return {
|
|
175
193
|
verdicts: await eachOf(sample, async (found): Promise<Verdict> => {
|
|
176
|
-
const judged = await judge(alignmentQuestion({ title: found.subject, body:
|
|
194
|
+
const judged = await judge(alignmentQuestion({ title: found.subject, body: found.body }, found.diff, config.hum));
|
|
177
195
|
|
|
178
196
|
if (judged.kind === "abstained") return "abstained";
|
|
179
197
|
|
|
@@ -232,9 +250,9 @@ const BOUNDARY_RUNS: Record<Boundary, () => Promise<Measured>> = {
|
|
|
232
250
|
};
|
|
233
251
|
},
|
|
234
252
|
"review-verdict": async () => {
|
|
235
|
-
const cases = (await read("review-verdict", ReviewCase)).slice(0, limit);
|
|
253
|
+
const cases = [...(await read("review-verdict", ReviewCase)), ...MERGED_CASES["review-verdict"](merged).map((found) => ReviewCase.parse(found))].slice(0, limit);
|
|
236
254
|
|
|
237
|
-
return { verdicts: await eachOf(cases, async (found) => scored(await prosePick("review", PROSE_PROMPTS.review(config, { id: found.case, title: found.item, body:
|
|
255
|
+
return { verdicts: await eachOf(cases, async (found) => scored(await prosePick("review", PROSE_PROMPTS.review(config, { id: found.case, title: found.item, body: found.body }, found.diff), found.diff, tmpdir()), found.label)) };
|
|
238
256
|
},
|
|
239
257
|
};
|
|
240
258
|
|
|
@@ -250,15 +268,26 @@ const precisionOf = ({ tp, fp }: PairCounts): number | undefined => rateOf(tp, t
|
|
|
250
268
|
const pairRow = (day: string, pairs: PairCounts, abstained: number, used: number): string =>
|
|
251
269
|
`| ${day} | ${model} | ${pairs.tp + pairs.fp + pairs.fn + pairs.tn} | ${pairs.tp} | ${pairs.fp} | ${pairs.fn} | ${pairs.tn} | ${abstained} | ${pairs.pairedBySignals} | ${rateCell(precisionOf(pairs))} | ${rateCell(rateOf(pairs.tp, pairs.tp + pairs.fn))} | ${used} |`;
|
|
252
270
|
|
|
253
|
-
/**
|
|
254
|
-
|
|
271
|
+
/** One boundary's pooled rate, and the model ids that answered while it was measured. */
|
|
272
|
+
type Pooled = { readonly boundary: Boundary; readonly rate: number; readonly answered: readonly string[] };
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Sets each pooled rate, beside its judge's hash, in the owner's `hum.json` at the current schema, where trust reads
|
|
276
|
+
* it, under the model as named and each id that answered for it; every other rate stays as the config had it.
|
|
277
|
+
*/
|
|
278
|
+
const recordRates = async (folder: string, pooled: readonly Pooled[]): Promise<void> => {
|
|
255
279
|
const file = join(folder, "hum.json");
|
|
256
|
-
const
|
|
280
|
+
const migrated = migrateConfig(JSON.parse(await Bun.file(file).text()));
|
|
257
281
|
|
|
258
|
-
|
|
282
|
+
if (!migrated.ok) throw new Error(migrated.refusal.message);
|
|
283
|
+
|
|
284
|
+
const measuredModels = pooled.reduce((measured, { boundary, rate, answered }) => withRate(measured, { boundary, model, answered, rate, judgeHash: judgeHash(boundary, config) }), config.hum.measuredModels);
|
|
285
|
+
|
|
286
|
+
await Bun.write(file, `${JSON.stringify({ ...migrated.value, measuredModels }, null, "\t")}\n`);
|
|
259
287
|
};
|
|
260
288
|
|
|
261
289
|
const unmeasured: Boundary[] = [];
|
|
290
|
+
const recorded: Pooled[] = [];
|
|
262
291
|
|
|
263
292
|
for (const boundary of boundaries) {
|
|
264
293
|
const pool = { right: 0, measured: 0, pairs: NO_PAIRS };
|
|
@@ -272,7 +301,7 @@ for (const boundary of boundaries) {
|
|
|
272
301
|
const measured = verdicts.length - count("halted");
|
|
273
302
|
const day = new Date().toISOString().slice(0, 10);
|
|
274
303
|
const row = boundary === "repeat-pair" ? pairRow(day, pairs, count("abstained"), calls - before) : `| ${day} | ${model} | ${measured} | ${count("right")} | ${count("wrong")} | ${count("abstained")} | ${rateCell(rateOf(count("right"), measured))} | ${calls - before} |`;
|
|
275
|
-
const results = join(values.write ?? root,
|
|
304
|
+
const results = join(values.write ?? root, EVALS_DIR, boundary, "RESULTS.md");
|
|
276
305
|
const earlier = (await Bun.file(results).exists()) ? await Bun.file(results).text() : boundary === "repeat-pair" ? pairHeader : header;
|
|
277
306
|
|
|
278
307
|
pool.right += count("right");
|
|
@@ -284,15 +313,17 @@ for (const boundary of boundaries) {
|
|
|
284
313
|
|
|
285
314
|
const pooled = boundary === "repeat-pair" ? precisionOf(pool.pairs) : rateOf(pool.right, pool.measured);
|
|
286
315
|
|
|
287
|
-
console.log(`${boundary}: pooled over ${runs} run(s): ${rateCell(pooled)}`);
|
|
316
|
+
console.log(`${boundary}: pooled over ${runs} run(s): ${rateCell(pooled)}, judge ${judgeHash(boundary, config)}`);
|
|
288
317
|
|
|
289
318
|
if (pooled === undefined) unmeasured.push(boundary);
|
|
290
|
-
else
|
|
319
|
+
else recorded.push({ boundary, rate: pooled, answered: [...answered] });
|
|
291
320
|
}
|
|
292
321
|
|
|
322
|
+
if (values.write !== undefined && recorded.length > 0) await recordRates(values.write, recorded);
|
|
323
|
+
|
|
293
324
|
if (unmeasured.length > 0) {
|
|
294
325
|
console.error(`Nothing was measured on ${unmeasured.join(", ")}, so no rate was recorded for ${unmeasured.length === 1 ? "it" : "them"}.`);
|
|
295
326
|
process.exitCode = 1;
|
|
296
327
|
}
|
|
297
328
|
|
|
298
|
-
console.log(`${calls} live calls, the start check included. Record each rate under measuredModels in defaults/hum.json; a model decides a boundary alone only at minJudgeRate or above; repeat-pair takes its precision, against repeatPrecision.`);
|
|
329
|
+
console.log(`${calls} live calls, the start check included. Record each rate as { rate, judgeHash } under measuredModels in defaults/hum.json; a model decides a boundary alone only at minJudgeRate or above; repeat-pair takes its precision, against repeatPrecision.`);
|
package/src/adapters/claude-p.ts
CHANGED
|
@@ -228,6 +228,13 @@ type Ended = RunOutcome | LimitEnded;
|
|
|
228
228
|
|
|
229
229
|
export const hasSucceeded = (outcome: Finished): boolean => outcome.subtype === "success" && !outcome.isError;
|
|
230
230
|
|
|
231
|
+
/** A run that ended with no answer and no token to count: a usage limit, a hold, or a stop. */
|
|
232
|
+
export type Paused = Extract<RunOutcome, { kind: "limited" | "held" | "stopped" }>;
|
|
233
|
+
|
|
234
|
+
const PAUSED_KINDS: readonly string[] = ["limited", "held", "stopped"] satisfies readonly Paused["kind"][];
|
|
235
|
+
|
|
236
|
+
export const isPaused = (outcome: RunOutcome): outcome is Paused => PAUSED_KINDS.includes(outcome.kind);
|
|
237
|
+
|
|
231
238
|
const LOCK_TEXT = /refresh[^\n]*lock|lock[^\n]*refresh/i;
|
|
232
239
|
|
|
233
240
|
const isLimitRefusal = (t: Transcript, result: Result): boolean => result.is_error && (t.rejectedUntil !== undefined || LIMIT_TEXT.test(result.result));
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { tmpdir } from "node:os";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import type { Boundary, HumConfig } from "../config/schema.ts";
|
|
4
|
-
import { isTrusted, jsonSchemaOf } from "../decide/prose.ts";
|
|
4
|
+
import { isTrusted, jsonSchemaOf, type ModelAsk } from "../decide/prose.ts";
|
|
5
5
|
import { SECOND_MS } from "../time.ts";
|
|
6
6
|
import { type RunOutcome, type Runner, hasSucceeded } from "./claude-p.ts";
|
|
7
7
|
|
|
@@ -38,16 +38,20 @@ export type ModelCallOptions = {
|
|
|
38
38
|
readonly onSpent?: (item: string, tokens: number) => void;
|
|
39
39
|
};
|
|
40
40
|
|
|
41
|
+
/** What a model is sent for a question: its prompt, and the schema its answer must match. */
|
|
42
|
+
export const askOf = ({ prompt, answer }: Pick<Question<unknown>, "prompt" | "answer">): ModelAsk => ({ prompt, schema: jsonSchemaOf(answer) });
|
|
43
|
+
|
|
41
44
|
/** The cheap model call through `claude -p`, as hope's judge calls it, with a typed answer by JSON schema. */
|
|
42
45
|
export const modelCallOver =
|
|
43
46
|
(runner: Runner, { model, timeoutMs, trusts, onSpent }: ModelCallOptions): ModelCall =>
|
|
44
|
-
async (
|
|
47
|
+
async (question) => {
|
|
48
|
+
const { boundary, answer, item, reportOnly } = question;
|
|
49
|
+
|
|
45
50
|
const outcome = await runner.run({
|
|
46
|
-
|
|
51
|
+
...askOf(question),
|
|
47
52
|
cwd: tmpdir(),
|
|
48
53
|
model,
|
|
49
54
|
effort: "low",
|
|
50
|
-
schema: jsonSchemaOf(answer),
|
|
51
55
|
call: "judgment",
|
|
52
56
|
signal: AbortSignal.timeout(timeoutMs),
|
|
53
57
|
});
|
|
@@ -18,11 +18,10 @@ export type Pull = {
|
|
|
18
18
|
readonly id: string;
|
|
19
19
|
readonly number: number;
|
|
20
20
|
readonly url: string;
|
|
21
|
-
readonly state: "OPEN" | "MERGED" | "CLOSED";
|
|
22
21
|
readonly autoMerge: boolean;
|
|
23
22
|
/** Its branch is behind the base, which a strict required check makes wait. */
|
|
24
23
|
readonly behind: boolean;
|
|
25
|
-
};
|
|
24
|
+
} & ({ readonly state: "OPEN" | "CLOSED" } | { readonly state: "MERGED"; /** The commit its merge made on the base. */ readonly mergeCommit: string });
|
|
26
25
|
|
|
27
26
|
/** Pull requests on the repo a run's changes land in. */
|
|
28
27
|
export type PullRequests = {
|
|
@@ -62,17 +61,22 @@ const PULL_REQUEST_RULE = "pull_request";
|
|
|
62
61
|
|
|
63
62
|
const SIGNATURES_RULE = "required_signatures";
|
|
64
63
|
|
|
65
|
-
const
|
|
64
|
+
const pullFields = {
|
|
66
65
|
id: z.string(),
|
|
67
66
|
number: z.int(),
|
|
68
67
|
url: z.string(),
|
|
69
|
-
state: z.enum(["OPEN", "MERGED", "CLOSED"]),
|
|
70
68
|
mergeStateStatus: z.string(),
|
|
71
69
|
autoMergeRequest: z.object({ enabledAt: z.string().nullable() }).nullable(),
|
|
72
|
-
}
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
/** GitHub names the commit a merge made once the pull request merged, and none before. */
|
|
73
|
+
const PullNode = z.discriminatedUnion("state", [
|
|
74
|
+
z.object({ ...pullFields, state: z.enum(["OPEN", "CLOSED"]) }),
|
|
75
|
+
z.object({ ...pullFields, state: z.literal("MERGED"), mergeCommit: z.object({ oid: z.string().min(1) }) }),
|
|
76
|
+
]);
|
|
73
77
|
|
|
74
78
|
const PULL_FIELDS =
|
|
75
|
-
"id number url state mergeStateStatus autoMergeRequest { enabledAt }";
|
|
79
|
+
"id number url state mergeStateStatus autoMergeRequest { enabledAt } mergeCommit { oid }";
|
|
76
80
|
|
|
77
81
|
const BEHIND = "BEHIND";
|
|
78
82
|
|
|
@@ -125,9 +129,9 @@ const pullOf = (node: z.infer<typeof PullNode>): Pull => ({
|
|
|
125
129
|
id: node.id,
|
|
126
130
|
number: node.number,
|
|
127
131
|
url: node.url,
|
|
128
|
-
state: node.state,
|
|
129
132
|
autoMerge: node.autoMergeRequest !== null,
|
|
130
133
|
behind: node.mergeStateStatus === BEHIND,
|
|
134
|
+
...(node.state === "MERGED" ? { state: node.state, mergeCommit: node.mergeCommit.oid } : { state: node.state }),
|
|
131
135
|
});
|
|
132
136
|
|
|
133
137
|
/** The method the owner set when the branch's rules allow it, else the first they allow. */
|
package/src/cli/setup.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { join } from "node:path";
|
|
3
2
|
import type { Command } from "commander";
|
|
4
|
-
import { x } from "tinyexec";
|
|
5
3
|
import { z } from "zod";
|
|
6
4
|
import type { Item } from "../adapter-api.ts";
|
|
7
5
|
import { gitIn, readGitIdentity } from "../adapters/git.ts";
|
|
8
|
-
import { HumConfig } from "../config/schema.ts";
|
|
6
|
+
import { HumConfig, SCHEMA_VERSION } from "../config/schema.ts";
|
|
7
|
+
import { runModels } from "../measure/judges.ts";
|
|
8
|
+
import { measureOn } from "../measure/remeasure.ts";
|
|
9
9
|
import { CommandTimedOut, exec, SHIPPED_COMMAND_TIMEOUT_MS } from "../exec.ts";
|
|
10
10
|
import { readSettings } from "../config/settings.ts";
|
|
11
11
|
import { kindCoverage, proposeSetup } from "../setup/propose.ts";
|
|
@@ -28,8 +28,6 @@ const GhIssues = z.array(z.object({ body: z.string().default(""), labels: z.arra
|
|
|
28
28
|
|
|
29
29
|
const GhLabels = z.array(z.object({ name: z.string() }));
|
|
30
30
|
|
|
31
|
-
const EVAL_SCRIPT = join(import.meta.dir, "..", "..", "scripts", "eval.ts");
|
|
32
|
-
|
|
33
31
|
/** What the scan reads from GitHub. Each fact is read alone, and one whose read fails is unknown while the rest stand. */
|
|
34
32
|
type GithubFacts = {
|
|
35
33
|
readonly items: readonly Pick<Item, "body" | "labels">[] | undefined;
|
|
@@ -80,14 +78,9 @@ const githubFacts = async (repo: string): Promise<GithubFacts> => ({
|
|
|
80
78
|
|
|
81
79
|
const printStep = ({ name, status, detail, ms }: Step): string => `${status.padEnd(7)} ${name} (${ms} ms): ${detail}`;
|
|
82
80
|
|
|
83
|
-
const
|
|
84
|
-
|
|
85
|
-
/** The distinct model ids the owner's tiers and cheap call run on. */
|
|
86
|
-
const modelsOf = (repo: string): string[] => {
|
|
87
|
-
const { models, decisionModel } = readSettings(repo, ModelSettings);
|
|
81
|
+
const MeasureSettings = z.looseObject({ models: HumConfig.shape.models, decisionModel: HumConfig.shape.decisionModel, evalRuns: HumConfig.shape.evalRuns });
|
|
88
82
|
|
|
89
|
-
|
|
90
|
-
};
|
|
83
|
+
const Runs = z.coerce.number().int().positive();
|
|
91
84
|
|
|
92
85
|
/** `hum setup scan | write | prove | measure`: the facts and writes behind the /hum skill. */
|
|
93
86
|
export const addSetupCommands = (root: Command, io: Io, done: (code: number) => void): void => {
|
|
@@ -113,7 +106,7 @@ export const addSetupCommands = (root: Command, io: Io, done: (code: number) =>
|
|
|
113
106
|
setup
|
|
114
107
|
.command("write")
|
|
115
108
|
.description("Write .hum/ and the service files from the answers file, or write nothing and say why.")
|
|
116
|
-
.requiredOption("--answers <file>",
|
|
109
|
+
.requiredOption("--answers <file>", `JSON: { "hum": {settings}, "waitHours": { "kind/state": hours }, "schemaVersion": ${SCHEMA_VERSION} }`)
|
|
117
110
|
.option("--home <folder>", "where the service unit goes; default the user's home")
|
|
118
111
|
.action(async ({ answers, home }: { answers: string; home?: string }) => {
|
|
119
112
|
const result = writeSetup(io.cwd, readAnswers(answers), { installedSkills: await installedSkills(), home, gitIdentity: await readGitIdentity(setupGit(io.cwd)), isPublicRepo: await isPublicRepo(io.cwd) });
|
|
@@ -137,13 +130,15 @@ export const addSetupCommands = (root: Command, io: Io, done: (code: number) =>
|
|
|
137
130
|
setup
|
|
138
131
|
.command("measure")
|
|
139
132
|
.description("Measure the owner's models on every labelled set and record the pooled rates in .hum/hum.json.")
|
|
140
|
-
.option("--runs <n>", "runs pooled per boundary
|
|
133
|
+
.option("--runs <n>", "runs pooled per boundary; default evalRuns in hum.json")
|
|
141
134
|
.option("--model <id...>", "model ids to measure; default every model the tiers and the cheap call use")
|
|
142
|
-
.action(async ({ runs, model }: { runs
|
|
143
|
-
|
|
144
|
-
|
|
135
|
+
.action(async ({ runs, model }: { runs?: string; model?: string[] }) => {
|
|
136
|
+
const settings = readSettings(io.cwd, MeasureSettings);
|
|
137
|
+
|
|
138
|
+
for (const id of model ?? runModels(settings)) {
|
|
139
|
+
const measured = await measureOn(io.cwd, { model: id, runs: runs === undefined ? settings.evalRuns : Runs.parse(runs) }, { inherit: true });
|
|
145
140
|
|
|
146
|
-
if (
|
|
141
|
+
if (!measured.ok) return done(EXIT_FAILED);
|
|
147
142
|
}
|
|
148
143
|
|
|
149
144
|
done(EXIT_OK);
|
package/src/config/json.ts
CHANGED
|
@@ -4,3 +4,6 @@ import { z } from "zod";
|
|
|
4
4
|
export const Json = z.json();
|
|
5
5
|
|
|
6
6
|
export type Json = z.infer<typeof Json>;
|
|
7
|
+
|
|
8
|
+
/** A value as the JSON it saves: what `JSON.stringify` writes, checked as Json on the way back. Undefined, which saves nothing, does not type-check. */
|
|
9
|
+
export const asJson = (value: NonNullable<unknown>): Json => Json.parse(JSON.parse(JSON.stringify(value)));
|
package/src/config/migrate.ts
CHANGED
|
@@ -21,10 +21,21 @@ export const climb = <T extends object>(value: T, from: number, to: number, step
|
|
|
21
21
|
/** Per earlier schema version: rewrites a config at that version into the next version's shape. */
|
|
22
22
|
export type Migrations = Steps<ConfigJson>;
|
|
23
23
|
|
|
24
|
+
const RatesAt2 = z.record(z.string(), z.record(z.string(), z.number()));
|
|
25
|
+
|
|
26
|
+
/** A version-2 `measuredModels`, each rate as a score with no judge hash; anything else is left for the schema to refuse. */
|
|
27
|
+
const scoresOf = (measured: Json): Json => {
|
|
28
|
+
const rates = RatesAt2.safeParse(measured);
|
|
29
|
+
|
|
30
|
+
return rates.success ? Object.fromEntries(Object.entries(rates.data).map(([boundary, byModel]) => [boundary, Object.fromEntries(Object.entries(byModel).map(([model, rate]) => [model, { rate }]))])) : measured;
|
|
31
|
+
};
|
|
32
|
+
|
|
24
33
|
/** One entry per earlier schema version. Setup climbs the owner's answers, a part of a config, through the same steps, so a step rewrites only the keys it finds. */
|
|
25
34
|
export const MIGRATIONS = {
|
|
26
35
|
/** 2 holds the pull request token as one of its sources: the secret 1 named becomes `{ from: "secret", secret }`. */
|
|
27
36
|
1: ({ pullRequestTokenSecret, ...config }) => (pullRequestTokenSecret === undefined ? config : { ...config, pullRequestToken: { from: "secret", secret: pullRequestTokenSecret } }),
|
|
37
|
+
/** 3 keeps each measured rate as `{ rate, judgeHash }`; a rate 2 kept has no hash, so its judge counts as changed. */
|
|
38
|
+
2: ({ measuredModels, ...config }) => (measuredModels === undefined ? config : { ...config, measuredModels: scoresOf(measuredModels) }),
|
|
28
39
|
} satisfies Migrations;
|
|
29
40
|
|
|
30
41
|
const Raw = z.record(z.string(), Json);
|