@moo.md/hum 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/scripts/eval.ts +88 -50
- package/src/adapter-api.ts +46 -14
- package/src/adapters/admit.ts +2 -2
- package/src/adapters/argument.ts +1 -1
- package/src/adapters/claim.ts +18 -15
- package/src/adapters/claude-bg.ts +50 -27
- package/src/adapters/claude-p.ts +18 -23
- package/src/adapters/contract.ts +22 -10
- package/src/adapters/each-call.ts +9 -0
- package/src/adapters/fake.ts +6 -6
- package/src/adapters/git.ts +10 -18
- package/src/adapters/github-client.ts +102 -0
- package/src/adapters/github.ts +34 -142
- package/src/adapters/guard.ts +20 -9
- package/src/adapters/history.ts +12 -15
- package/src/adapters/item-file.ts +1 -1
- package/src/adapters/markdown.ts +49 -42
- package/src/adapters/marked.ts +6 -12
- package/src/adapters/model-call.ts +4 -4
- package/src/adapters/once.ts +5 -5
- package/src/adapters/owner-commit.ts +13 -11
- package/src/adapters/pull-requests.ts +11 -13
- package/src/adapters/registry.ts +7 -7
- package/src/adapters/roundup.ts +27 -35
- package/src/adapters/serial.ts +4 -5
- package/src/adapters/transcript.ts +13 -8
- package/src/cli/main.ts +1 -0
- package/src/cli/ops.ts +32 -14
- package/src/cli/service.ts +14 -8
- package/src/cli/setup.ts +8 -3
- package/src/cli/shared-watch.ts +3 -3
- package/src/config/checks.ts +14 -18
- package/src/config/machine.ts +5 -7
- package/src/config/refusal.ts +7 -0
- package/src/config/schema.ts +7 -7
- package/src/config/settings.ts +17 -2
- package/src/config/state-map.ts +5 -3
- package/src/decide/alignment.ts +4 -4
- package/src/decide/answer.ts +24 -22
- package/src/decide/kind.ts +1 -6
- package/src/decide/labels.ts +9 -9
- package/src/decide/merge.ts +6 -7
- package/src/decide/prose.ts +4 -4
- package/src/decide/question.ts +23 -20
- package/src/decide/review.ts +4 -2
- package/src/decide/risk.ts +25 -19
- package/src/decide/taste.ts +3 -3
- package/src/exec.ts +4 -3
- package/src/index.ts +14 -13
- package/src/loop/check-item.ts +3 -3
- package/src/loop/ci.ts +1 -1
- package/src/loop/github.ts +2 -2
- package/src/loop/groups.ts +13 -6
- package/src/loop/memory.ts +40 -28
- package/src/loop/notes.ts +5 -5
- package/src/loop/pair.ts +1 -1
- package/src/loop/pass.ts +18 -12
- package/src/loop/proof.ts +9 -7
- package/src/loop/prove-item.ts +1 -1
- package/src/loop/records.ts +12 -13
- package/src/loop/signals.ts +6 -6
- package/src/measure/inputs.ts +1 -1
- package/src/measure/measure.ts +4 -4
- package/src/measure/rate.ts +2 -0
- package/src/service/ci-tick.ts +2 -2
- package/src/service/supervise.ts +3 -2
- package/src/service/token.ts +3 -3
- package/src/service/wake.ts +2 -2
- package/src/service/workflows.ts +2 -2
- package/src/setup/propose.ts +4 -4
- package/src/setup/prove.ts +18 -12
- package/src/setup/scan.ts +31 -24
- package/src/setup/shipped.ts +8 -2
- package/src/setup/tables.ts +4 -3
- package/src/setup/waits.ts +9 -11
- package/src/setup/write.ts +17 -10
- package/src/slots/pool.ts +32 -24
- package/src/state/events.ts +1 -1
- package/src/state/faults.ts +13 -5
- package/src/state/format.ts +2 -2
- package/src/state/items.ts +9 -10
- package/src/state/open.ts +2 -2
- package/src/state/ops.ts +16 -10
- package/src/state/records.ts +5 -5
- package/src/state/runs.ts +33 -26
- package/src/state/saved.ts +44 -32
- package/src/status/file.ts +20 -1
- package/src/status/format.ts +47 -57
- package/src/status/sections.ts +2 -2
- package/src/worker/alignment-report.ts +4 -5
- package/src/worker/answers.ts +26 -24
- package/src/worker/base-check.ts +2 -2
- package/src/worker/kind.ts +23 -19
- package/src/worker/land.ts +114 -126
- package/src/worker/merged.ts +9 -7
- package/src/worker/open.ts +14 -12
- package/src/worker/prose-run.ts +3 -3
- package/src/worker/questions.ts +15 -12
- package/src/worker/scheduler.ts +4 -4
- package/src/worker/signature.ts +3 -3
- package/src/worker/tally.ts +0 -2
- package/src/worker/taste.ts +6 -15
- package/src/worker/thin-path.ts +148 -121
- package/src/worker/thread.ts +25 -0
- package/src/worker/worker.ts +77 -47
- package/surface.json +632 -2197
- package/src/slots/scan.ts +0 -26
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@moo.md/hum",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "The hum engine: work items become Claude Code agent runs, the repo's checks decide what they can, and only what nothing settles becomes a question to the owner.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": {
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"surface.json"
|
|
33
33
|
],
|
|
34
34
|
"scripts": {
|
|
35
|
-
"typecheck": "tsc --noEmit",
|
|
35
|
+
"typecheck": "test -d ../hum/node_modules || { echo \"hum-engine: the view tests import hum/mod, whose types come from hum's packages; run bun install in hum first\" >&2; exit 1; }; tsc --noEmit",
|
|
36
36
|
"schema": "bun scripts/write-schema.ts",
|
|
37
37
|
"eval": "bun scripts/eval.ts",
|
|
38
38
|
"release:check": "bun tools/release/main.ts check",
|
package/scripts/eval.ts
CHANGED
|
@@ -6,10 +6,10 @@ import { AgentDecisionExhaustedError } from "@statelyai/agent";
|
|
|
6
6
|
import PQueue from "p-queue";
|
|
7
7
|
import { z } from "zod";
|
|
8
8
|
import { type Finished, type Runner, startRunner } from "../src/adapters/claude-p.ts";
|
|
9
|
+
import { gitIn, readGitIdentity } from "../src/adapters/git.ts";
|
|
9
10
|
import { modelCallOver } from "../src/adapters/model-call.ts";
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import { alignmentQuestion, raisesLevel } from "../src/decide/alignment.ts";
|
|
11
|
+
import { BOUNDARIES, type Boundary } from "../src/config/schema.ts";
|
|
12
|
+
import { alignmentQuestion, doesRaiseLevel } from "../src/decide/alignment.ts";
|
|
13
13
|
import { replyQuestion } from "../src/decide/answer.ts";
|
|
14
14
|
import { kindQuestion } from "../src/decide/kind.ts";
|
|
15
15
|
import { tasteQuestion } from "../src/decide/taste.ts";
|
|
@@ -17,17 +17,21 @@ import { pairQuestion } from "../src/loop/pair.ts";
|
|
|
17
17
|
import type { ReviewNote } from "../src/loop/records.ts";
|
|
18
18
|
import { pairing, prepare } from "../src/loop/signals.ts";
|
|
19
19
|
import type { JsonSchema, ProsePurpose } from "../src/decide/prose.ts";
|
|
20
|
+
import { SHIPPED_COMMAND_TIMEOUT_MS } from "../src/exec.ts";
|
|
21
|
+
import { shippedRaw, shippedSettingsAs } from "../src/setup/shipped.ts";
|
|
20
22
|
import { openState } from "../src/state/open.ts";
|
|
21
23
|
import { decideProse } from "../src/worker/prose-run.ts";
|
|
22
24
|
import { PROSE_PROMPTS } from "../src/worker/thin-path.ts";
|
|
23
25
|
import { SECOND_MS } from "../src/time.ts";
|
|
26
|
+
import { rateOf } from "../src/measure/rate.ts";
|
|
24
27
|
|
|
25
28
|
/**
|
|
26
29
|
* Measures one model-judgment boundary on its labelled cases, through the same prompts and fences the engine runs,
|
|
27
30
|
* and adds a row to its RESULTS.md. `bun run eval -- --model <id>` measures every boundary on that model id; naming
|
|
28
31
|
* boundaries measures only those. A case the runner held or limited counts in no rate. `--runs <n>` repeats each
|
|
29
32
|
* boundary and pools the runs, since one run proves little; `--write <folder>` records each pooled rate under
|
|
30
|
-
* `measuredModels` in that folder's `hum.json`, which is how setup measures an owner's own models.
|
|
33
|
+
* `measuredModels` in that folder's `hum.json`, which is how setup measures an owner's own models. A boundary where
|
|
34
|
+
* nothing was measured records no rate, and the script exits non-zero.
|
|
31
35
|
*/
|
|
32
36
|
|
|
33
37
|
const { values, positionals } = parseArgs({ allowPositionals: true, options: { model: { type: "string" }, limit: { type: "string" }, runs: { type: "string" }, write: { type: "string" } } });
|
|
@@ -36,16 +40,23 @@ const OWNER_LABELLED: readonly Boundary[] = ["taste"];
|
|
|
36
40
|
|
|
37
41
|
const boundaries: readonly Boundary[] = positionals.length > 0 ? z.array(z.enum(BOUNDARIES)).parse(positionals) : BOUNDARIES.filter((boundary) => !OWNER_LABELLED.includes(boundary));
|
|
38
42
|
const root = join(import.meta.dir, "..");
|
|
39
|
-
const
|
|
40
|
-
|
|
43
|
+
const settings = shippedSettingsAs(await readGitIdentity(gitIn(process.cwd(), { timeoutMs: SHIPPED_COMMAND_TIMEOUT_MS })));
|
|
44
|
+
|
|
45
|
+
if (!settings.ok) {
|
|
46
|
+
console.error(settings.refusal.message);
|
|
47
|
+
process.exit(1);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const config = { hum: settings.value, roles: shippedRaw().roles };
|
|
41
51
|
const model = values.model ?? config.hum.decisionModel;
|
|
42
|
-
const
|
|
43
|
-
const
|
|
52
|
+
const Count = z.coerce.number().int().positive();
|
|
53
|
+
const runs = values.runs === undefined ? 1 : Count.parse(values.runs);
|
|
54
|
+
const limit = values.limit === undefined ? Number.POSITIVE_INFINITY : Count.parse(values.limit);
|
|
44
55
|
|
|
45
56
|
type Verdict = "right" | "wrong" | "abstained" | "halted";
|
|
46
57
|
|
|
47
|
-
const state = openState(mkdtempSync(join(tmpdir(), "hum-eval-")));
|
|
48
|
-
const started = await startRunner({ state, model });
|
|
58
|
+
const state = openState(mkdtempSync(join(tmpdir(), "hum-eval-")), { now: Date.now });
|
|
59
|
+
const started = await startRunner({ state, model, now: Date.now, env: process.env });
|
|
49
60
|
|
|
50
61
|
if (started.check.kind !== "finished") throw new Error(`the start check ended as ${started.check.kind}`);
|
|
51
62
|
|
|
@@ -109,7 +120,7 @@ const TasteCase = z.object({ case: z.string(), label: z.enum(["keeps", "breaks"]
|
|
|
109
120
|
/** Calls at once: enough to finish a set in minutes, few enough not to trip a rate limit. */
|
|
110
121
|
const AT_ONCE = 4;
|
|
111
122
|
|
|
112
|
-
const eachOf = <T>(cases: readonly T[], measure: (found: T) => Promise<
|
|
123
|
+
const eachOf = <T, R>(cases: readonly T[], measure: (found: T) => Promise<R>): Promise<R[]> => new PQueue({ concurrency: AT_ONCE }).addAll(cases.map((found) => () => measure(found)));
|
|
113
124
|
|
|
114
125
|
const read = async <T>(boundary: Boundary, schema: z.ZodType<T>, file = "cases.jsonl"): Promise<T[]> =>
|
|
115
126
|
(await Bun.file(join(root, "evals", boundary, file)).text())
|
|
@@ -118,78 +129,98 @@ const read = async <T>(boundary: Boundary, schema: z.ZodType<T>, file = "cases.j
|
|
|
118
129
|
.map((line) => schema.parse(JSON.parse(line)));
|
|
119
130
|
|
|
120
131
|
/** The confusion counts of the review-note boundary: its rate is precision, and recall is recorded beside it. */
|
|
121
|
-
|
|
132
|
+
type PairCounts = { readonly tp: number; readonly fp: number; readonly fn: number; readonly tn: number; readonly pairedBySignals: number };
|
|
133
|
+
|
|
134
|
+
type Cell = Exclude<keyof PairCounts, "pairedBySignals">;
|
|
135
|
+
|
|
136
|
+
const NO_PAIRS: PairCounts = { tp: 0, fp: 0, fn: 0, tn: 0, pairedBySignals: 0 };
|
|
137
|
+
|
|
138
|
+
const addPairs = (a: PairCounts, b: PairCounts): PairCounts => ({ tp: a.tp + b.tp, fp: a.fp + b.fp, fn: a.fn + b.fn, tn: a.tn + b.tn, pairedBySignals: a.pairedBySignals + b.pairedBySignals });
|
|
139
|
+
|
|
140
|
+
/** One run of a boundary: each case's verdict, and for the review-note boundary its confusion counts. */
|
|
141
|
+
type Measured = { readonly verdicts: readonly Verdict[]; readonly pairs?: PairCounts };
|
|
122
142
|
|
|
123
143
|
const noteOf = (text: string, sourceId: string): ReviewNote => ({ source: "review-comment", sourceId, change: sourceId, hunk: "", text, at: 0 });
|
|
124
144
|
|
|
125
|
-
const BOUNDARY_RUNS: Record<Boundary, () => Promise<
|
|
145
|
+
const BOUNDARY_RUNS: Record<Boundary, () => Promise<Measured>> = {
|
|
126
146
|
kind: async () => {
|
|
127
147
|
const cases = (await read("kind", KindCase)).slice(0, limit);
|
|
128
148
|
|
|
129
|
-
return
|
|
149
|
+
return {
|
|
150
|
+
verdicts: await eachOf(cases, async (found) => {
|
|
130
151
|
const judged = await judge(kindQuestion(found, config.hum));
|
|
131
152
|
|
|
132
153
|
return judged.kind === "abstained" ? "abstained" : scored(judged.value.kind, found.label);
|
|
133
|
-
|
|
154
|
+
}),
|
|
155
|
+
};
|
|
134
156
|
},
|
|
135
157
|
alignment: async () => {
|
|
136
158
|
const sample = (await read("alignment", AlignmentCase, "sample.jsonl")).slice(0, limit);
|
|
137
159
|
|
|
138
|
-
return
|
|
160
|
+
return {
|
|
161
|
+
verdicts: await eachOf(sample, async (found): Promise<Verdict> => {
|
|
139
162
|
const judged = await judge(alignmentQuestion({ title: found.subject, body: "" }, found.diff, config.hum));
|
|
140
163
|
|
|
141
164
|
if (judged.kind === "abstained") return "abstained";
|
|
142
165
|
|
|
143
|
-
return
|
|
144
|
-
|
|
166
|
+
return doesRaiseLevel({ rank: 0, levels: config.hum.levels.length, answer: judged.value, config: config.hum }) === (found.label !== "clean") ? "right" : "wrong";
|
|
167
|
+
}),
|
|
168
|
+
};
|
|
145
169
|
},
|
|
146
170
|
"plan-outcome": async () => {
|
|
147
171
|
const repo = planRepo();
|
|
148
172
|
const cases = (await read("plan-outcome", PlanCase)).slice(0, limit);
|
|
149
173
|
|
|
150
|
-
return eachOf(cases, async (found) => scored(await prosePick("plan", PROSE_PROMPTS.plan(config, { id: found.case, title: found.body.split("\n")[0] ?? "", body: found.body }), "", repo), found.label));
|
|
174
|
+
return { verdicts: await eachOf(cases, async (found) => scored(await prosePick("plan", PROSE_PROMPTS.plan(config, { id: found.case, title: found.body.split("\n")[0] ?? "", body: found.body }), "", repo), found.label)) };
|
|
151
175
|
},
|
|
152
176
|
"reply-option": async () => {
|
|
153
177
|
const cases = (await read("reply-option", ReplyCase)).slice(0, limit);
|
|
154
178
|
|
|
155
|
-
return
|
|
156
|
-
|
|
179
|
+
return {
|
|
180
|
+
verdicts: await eachOf(cases, async (found) => {
|
|
181
|
+
const judged = await judge(replyQuestion({ question: found.question, options: found.options, reply: found.reply }));
|
|
157
182
|
|
|
158
|
-
|
|
159
|
-
|
|
183
|
+
return judged.kind === "abstained" ? "abstained" : scored(judged.value.option, found.label);
|
|
184
|
+
}),
|
|
185
|
+
};
|
|
160
186
|
},
|
|
161
187
|
"repeat-pair": async () => {
|
|
162
188
|
const cases = (await read("repeat-pair", PairCase)).slice(0, limit);
|
|
163
189
|
const open = cases.filter((found) => pairing(prepare(noteOf(found.a.text, "a")), prepare(noteOf(found.b.text, "b"))).signal === undefined);
|
|
164
190
|
|
|
165
|
-
|
|
191
|
+
const judged = await eachOf(open, async (found): Promise<{ verdict: Verdict; cell?: Cell }> => {
|
|
192
|
+
const pair = await judge(pairQuestion(noteOf(found.a.text, "a"), noteOf(found.b.text, "b")));
|
|
166
193
|
|
|
167
|
-
|
|
168
|
-
const judged = await judge(pairQuestion(noteOf(found.a.text, "a"), noteOf(found.b.text, "b")));
|
|
194
|
+
if (pair.kind === "abstained") return { verdict: "abstained" };
|
|
169
195
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
const said = judged.value.same;
|
|
196
|
+
const said = pair.value.same;
|
|
173
197
|
const truth = found.label === "same";
|
|
174
198
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
return said === truth ? "right" : "wrong";
|
|
199
|
+
return { verdict: said === truth ? "right" : "wrong", cell: said ? (truth ? "tp" : "fp") : truth ? "fn" : "tn" };
|
|
178
200
|
});
|
|
201
|
+
|
|
202
|
+
const countOf = (cell: Cell) => judged.filter((found) => found.cell === cell).length;
|
|
203
|
+
|
|
204
|
+
return {
|
|
205
|
+
verdicts: judged.map(({ verdict }) => verdict),
|
|
206
|
+
pairs: { tp: countOf("tp"), fp: countOf("fp"), fn: countOf("fn"), tn: countOf("tn"), pairedBySignals: cases.length - open.length },
|
|
207
|
+
};
|
|
179
208
|
},
|
|
180
209
|
taste: async () => {
|
|
181
210
|
const cases = (await read("taste", TasteCase)).slice(0, limit);
|
|
182
211
|
|
|
183
|
-
return
|
|
184
|
-
|
|
212
|
+
return {
|
|
213
|
+
verdicts: await eachOf(cases, async (found) => {
|
|
214
|
+
const judged = await judge(tasteQuestion(found.rule, found.diff, config.hum));
|
|
185
215
|
|
|
186
|
-
|
|
187
|
-
|
|
216
|
+
return judged.kind === "abstained" ? "abstained" : scored(judged.value.keeps ? "keeps" : "breaks", found.label);
|
|
217
|
+
}),
|
|
218
|
+
};
|
|
188
219
|
},
|
|
189
220
|
"review-verdict": async () => {
|
|
190
221
|
const cases = (await read("review-verdict", ReviewCase)).slice(0, limit);
|
|
191
222
|
|
|
192
|
-
return eachOf(cases, async (found) => scored(await prosePick("review", PROSE_PROMPTS.review(config, { id: found.case, title: found.item, body: "" }, found.diff), found.diff, tmpdir()), found.label));
|
|
223
|
+
return { verdicts: await eachOf(cases, async (found) => scored(await prosePick("review", PROSE_PROMPTS.review(config, { id: found.case, title: found.item, body: "" }, found.diff), found.diff, tmpdir()), found.label)) };
|
|
193
224
|
},
|
|
194
225
|
};
|
|
195
226
|
|
|
@@ -197,12 +228,13 @@ const header = "| Date | Model id | n | Right | Wrong | Abstained | Rate | Live
|
|
|
197
228
|
|
|
198
229
|
const pairHeader = "| Date | Model id | n | TP | FP | FN | TN | Abstained | Paired by signals | Precision | Recall | Live calls |\n|---|---|---|---|---|---|---|---|---|---|---|---|\n";
|
|
199
230
|
|
|
200
|
-
|
|
231
|
+
/** A rate as a RESULTS.md cell; a dash where nothing was measured. */
|
|
232
|
+
const rateCell = (rate: number | undefined): string => rate?.toFixed(2) ?? "-";
|
|
201
233
|
|
|
202
|
-
const
|
|
203
|
-
`| ${day} | ${model} | ${pairCounts.tp + pairCounts.fp + pairCounts.fn + pairCounts.tn} | ${pairCounts.tp} | ${pairCounts.fp} | ${pairCounts.fn} | ${pairCounts.tn} | ${abstained} | ${pairCounts.pairedBySignals} | ${ratio(pairCounts.tp, pairCounts.tp + pairCounts.fp)} | ${ratio(pairCounts.tp, pairCounts.tp + pairCounts.fn)} | ${used} |`;
|
|
234
|
+
const precisionOf = ({ tp, fp }: PairCounts): number | undefined => rateOf(tp, tp + fp);
|
|
204
235
|
|
|
205
|
-
const
|
|
236
|
+
const pairRow = (day: string, pairs: PairCounts, abstained: number, used: number): string =>
|
|
237
|
+
`| ${day} | ${model} | ${pairs.tp + pairs.fp + pairs.fn + pairs.tn} | ${pairs.tp} | ${pairs.fp} | ${pairs.fn} | ${pairs.tn} | ${abstained} | ${pairs.pairedBySignals} | ${rateCell(precisionOf(pairs))} | ${rateCell(rateOf(pairs.tp, pairs.tp + pairs.fn))} | ${used} |`;
|
|
206
238
|
|
|
207
239
|
/** Sets one pooled rate in the owner's `hum.json`, where trust reads it. */
|
|
208
240
|
const recordRate = async (folder: string, boundary: Boundary, rate: number): Promise<void> => {
|
|
@@ -212,33 +244,39 @@ const recordRate = async (folder: string, boundary: Boundary, rate: number): Pro
|
|
|
212
244
|
await Bun.write(file, `${JSON.stringify({ ...written, measuredModels: { ...written.measuredModels, [boundary]: { ...written.measuredModels[boundary], [model]: Number(rate.toFixed(2)) } } }, null, "\t")}\n`);
|
|
213
245
|
};
|
|
214
246
|
|
|
215
|
-
|
|
216
|
-
const pool = { ...EMPTY_POOL };
|
|
247
|
+
const unmeasured: Boundary[] = [];
|
|
217
248
|
|
|
218
|
-
|
|
249
|
+
for (const boundary of boundaries) {
|
|
250
|
+
const pool = { right: 0, measured: 0, pairs: NO_PAIRS };
|
|
219
251
|
|
|
220
252
|
for (let run = 1; run <= runs; run++) {
|
|
221
253
|
const before = calls;
|
|
222
|
-
const verdicts = await BOUNDARY_RUNS[boundary]();
|
|
254
|
+
const { verdicts, pairs = NO_PAIRS } = await BOUNDARY_RUNS[boundary]();
|
|
223
255
|
const count = (verdict: Verdict) => verdicts.filter((found) => found === verdict).length;
|
|
224
256
|
const measured = verdicts.length - count("halted");
|
|
225
|
-
const rate = measured === 0 ? 0 : count("right") / measured;
|
|
226
257
|
const day = new Date().toISOString().slice(0, 10);
|
|
227
|
-
const row = boundary === "repeat-pair" ? pairRow(day, count("abstained"), calls - before) : `| ${day} | ${model} | ${measured} | ${count("right")} | ${count("wrong")} | ${count("abstained")} | ${
|
|
258
|
+
const row = boundary === "repeat-pair" ? pairRow(day, pairs, count("abstained"), calls - before) : `| ${day} | ${model} | ${measured} | ${count("right")} | ${count("wrong")} | ${count("abstained")} | ${rateCell(rateOf(count("right"), measured))} | ${calls - before} |`;
|
|
228
259
|
const results = join(values.write ?? root, "evals", boundary, "RESULTS.md");
|
|
229
260
|
const earlier = (await Bun.file(results).exists()) ? await Bun.file(results).text() : boundary === "repeat-pair" ? pairHeader : header;
|
|
230
261
|
|
|
231
262
|
pool.right += count("right");
|
|
232
263
|
pool.measured += measured;
|
|
264
|
+
pool.pairs = addPairs(pool.pairs, pairs);
|
|
233
265
|
await Bun.write(results, `${earlier}${row}\n`);
|
|
234
266
|
console.log(`${boundary}: ${row}`);
|
|
235
267
|
}
|
|
236
268
|
|
|
237
|
-
const pooled = boundary === "repeat-pair" ?
|
|
269
|
+
const pooled = boundary === "repeat-pair" ? precisionOf(pool.pairs) : rateOf(pool.right, pool.measured);
|
|
238
270
|
|
|
239
|
-
console.log(`${boundary}: pooled over ${runs} run(s): ${pooled
|
|
271
|
+
console.log(`${boundary}: pooled over ${runs} run(s): ${rateCell(pooled)}`);
|
|
272
|
+
|
|
273
|
+
if (pooled === undefined) unmeasured.push(boundary);
|
|
274
|
+
else if (values.write !== undefined) await recordRate(values.write, boundary, pooled);
|
|
275
|
+
}
|
|
240
276
|
|
|
241
|
-
|
|
277
|
+
if (unmeasured.length > 0) {
|
|
278
|
+
console.error(`Nothing was measured on ${unmeasured.join(", ")}, so no rate was recorded for ${unmeasured.length === 1 ? "it" : "them"}.`);
|
|
279
|
+
process.exitCode = 1;
|
|
242
280
|
}
|
|
243
281
|
|
|
244
282
|
console.log(`${calls} live calls, the start check included. Record each rate under measuredModels in defaults/hum.json; a model decides a boundary alone only at minJudgeRate or above; repeat-pair takes its precision, against repeatPrecision.`);
|
package/src/adapter-api.ts
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import type { Env } from "./exec.ts";
|
|
2
3
|
import type { FaultInput } from "./state/faults.ts";
|
|
3
4
|
|
|
4
5
|
export type { FaultInput };
|
|
5
6
|
|
|
6
7
|
/** Bumped on every change to the API an adapter implements; a breaking change bumps the engine's major version. */
|
|
7
|
-
export const ADAPTER_API_VERSION =
|
|
8
|
+
export const ADAPTER_API_VERSION = 2;
|
|
8
9
|
|
|
9
|
-
/** Failure that retrying cannot fix: a reply of the wrong shape, a refused push. The engine surfaces it at once. */
|
|
10
|
+
/** Failure that retrying cannot fix: a reply of the wrong shape, a refused push. The engine surfaces it at once, as it does a `failed: "now"` answer. */
|
|
10
11
|
export class PermanentError extends Error {
|
|
11
12
|
override name: string = "PermanentError";
|
|
12
13
|
}
|
|
@@ -21,10 +22,19 @@ export const ItemId = z
|
|
|
21
22
|
.string()
|
|
22
23
|
.max(ITEM_ID_LENGTH, ITEM_ID_MESSAGE)
|
|
23
24
|
.regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/, ITEM_ID_MESSAGE)
|
|
24
|
-
.refine((id) => !id.includes("..") && !id.endsWith(".lock"), "an item id has no '..' and does not end in '.lock'")
|
|
25
|
+
.refine((id) => !id.includes("..") && !id.endsWith(".lock"), "an item id has no '..' and does not end in '.lock'")
|
|
26
|
+
.brand<"ItemId">();
|
|
25
27
|
|
|
26
28
|
export type ItemId = z.infer<typeof ItemId>;
|
|
27
29
|
|
|
30
|
+
/** Who holds a claim: a plain name, never an item id. */
|
|
31
|
+
export const Holder = z
|
|
32
|
+
.string()
|
|
33
|
+
.regex(/^[A-Za-z0-9][\w.@:-]{0,99}$/, "a holder is a plain name: letters, digits and . _ @ : -")
|
|
34
|
+
.brand<"Holder">();
|
|
35
|
+
|
|
36
|
+
export type Holder = z.infer<typeof Holder>;
|
|
37
|
+
|
|
28
38
|
const text = z.string().min(1);
|
|
29
39
|
const time = z.int().nonnegative();
|
|
30
40
|
|
|
@@ -60,7 +70,7 @@ export const Item = ItemRef.extend({
|
|
|
60
70
|
|
|
61
71
|
export type Item = z.infer<typeof Item>;
|
|
62
72
|
|
|
63
|
-
export const Claim = z.strictObject({ item: ItemId, holder:
|
|
73
|
+
export const Claim = z.strictObject({ item: ItemId, holder: Holder, expiresAt: time });
|
|
64
74
|
|
|
65
75
|
export type Claim = z.infer<typeof Claim>;
|
|
66
76
|
|
|
@@ -84,39 +94,59 @@ export type FollowUp = { readonly title: string; readonly body: string };
|
|
|
84
94
|
/** The work source: the queue, the claims and the item threads. */
|
|
85
95
|
export interface WorkSource {
|
|
86
96
|
listReady(): Promise<ItemRef[]>;
|
|
87
|
-
read(id:
|
|
97
|
+
read(id: ItemId): Promise<Item>;
|
|
88
98
|
/** An atomic take. An unclaimed or expired item goes to the first caller; the same holder renews. */
|
|
89
|
-
claim(id:
|
|
99
|
+
claim(id: ItemId, holder: Holder, ttlMs: number): Promise<ClaimResult>;
|
|
90
100
|
/** Extends the holder's own claim; `won: false` when another worker holds it now. */
|
|
91
|
-
renew(id:
|
|
92
|
-
comment(id:
|
|
101
|
+
renew(id: ItemId, holder: Holder, ttlMs: number): Promise<ClaimResult>;
|
|
102
|
+
comment(id: ItemId, text: string): Promise<void>;
|
|
93
103
|
/** Files a ready item. Filing the same `title` from the same `from` twice returns the first item. */
|
|
94
|
-
fileFollowUp(from:
|
|
104
|
+
fileFollowUp(from: ItemId, followUp: FollowUp): Promise<ItemRef>;
|
|
95
105
|
}
|
|
96
106
|
|
|
97
107
|
/** Where a run's questions and the choices made without the owner land. */
|
|
98
108
|
export interface Questions {
|
|
99
|
-
ask(id:
|
|
109
|
+
ask(id: ItemId, question: Question): Promise<void>;
|
|
100
110
|
/** Records a choice made without the owner. */
|
|
101
|
-
decide(id:
|
|
111
|
+
decide(id: ItemId, text: string): Promise<void>;
|
|
102
112
|
}
|
|
103
113
|
|
|
104
114
|
/** Where the owner's answers arrive. */
|
|
105
115
|
export interface Answers {
|
|
106
|
-
answer(id:
|
|
116
|
+
answer(id: ItemId, answer: Answer): Promise<void>;
|
|
107
117
|
/** The one cheap read: has anything changed since `since`. An idle call costs no rate limit. */
|
|
108
118
|
watch(since?: string): Promise<Watch>;
|
|
109
119
|
}
|
|
110
120
|
|
|
111
121
|
export interface Release {
|
|
112
122
|
/** Puts the item's artifact where reviewers see it. */
|
|
113
|
-
publish(id:
|
|
123
|
+
publish(id: ItemId, ref: string): Promise<void>;
|
|
114
124
|
/** Ships a published artifact. */
|
|
115
|
-
release(id:
|
|
125
|
+
release(id: ItemId, ref: string): Promise<void>;
|
|
116
126
|
}
|
|
117
127
|
|
|
128
|
+
/**
|
|
129
|
+
* A call an adapter could not complete, answered in place of its value. `retry`: a later attempt may work, and the
|
|
130
|
+
* engine tries again within the provider timeout. `now`: no attempt will, and the engine raises one card and stops.
|
|
131
|
+
*/
|
|
132
|
+
export const Failed = z.strictObject({ failed: z.enum(["retry", "now"]), why: text });
|
|
133
|
+
|
|
134
|
+
export type Failed = z.infer<typeof Failed>;
|
|
135
|
+
|
|
136
|
+
/** An integration as an adapter implements it: every call may answer `Failed` in place of its value. */
|
|
137
|
+
export type Fallible<I> = { readonly [K in keyof I]: I[K] extends (...args: infer A) => Promise<infer R> ? (...args: A) => Promise<R | Failed> : I[K] };
|
|
138
|
+
|
|
118
139
|
/** An adapter declares the API version it implements and the integrations it provides. */
|
|
119
140
|
export type Adapter = {
|
|
141
|
+
readonly apiVersion: number;
|
|
142
|
+
readonly workSource?: Fallible<WorkSource>;
|
|
143
|
+
readonly questions?: Fallible<Questions>;
|
|
144
|
+
readonly answers?: Fallible<Answers>;
|
|
145
|
+
readonly release?: Fallible<Release>;
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
/** The integrations with plain replies: the engine calls them so behind the guard, which retries or throws on `Failed`, and an adapter that never answers `Failed` may implement them so. */
|
|
149
|
+
export type Integrations = {
|
|
120
150
|
readonly apiVersion: number;
|
|
121
151
|
readonly workSource?: WorkSource;
|
|
122
152
|
readonly questions?: Questions;
|
|
@@ -128,6 +158,8 @@ export type AdapterContext = {
|
|
|
128
158
|
/** Raise a fault card, scoped to one item or provider. */
|
|
129
159
|
readonly report: (fault: FaultInput) => void;
|
|
130
160
|
readonly now: () => number;
|
|
161
|
+
/** The process environment, for the tokens and addresses an adapter reads. */
|
|
162
|
+
readonly env: Env;
|
|
131
163
|
};
|
|
132
164
|
|
|
133
165
|
/** The default export of an owner's `adapters/<name>.ts`. */
|
package/src/adapters/admit.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
2
|
import { extname, relative } from "node:path";
|
|
3
|
-
import type { Adapter } from "../adapter-api.ts";
|
|
3
|
+
import type { Adapter, Integrations } from "../adapter-api.ts";
|
|
4
4
|
import type { Faults } from "../state/faults.ts";
|
|
5
5
|
import type { Card } from "../status/format.ts";
|
|
6
6
|
import { type GuardOptions, guarded } from "./guard.ts";
|
|
@@ -47,7 +47,7 @@ export const engineImporters = (files: readonly string[]): { file: string; modul
|
|
|
47
47
|
export const sourceFiles = (dir: string): string[] =>
|
|
48
48
|
existsSync(dir) ? [...SOURCE_GLOB.scanSync({ cwd: dir, absolute: true })].filter(isOutsideDependencies).sort() : [];
|
|
49
49
|
|
|
50
|
-
export type Admission = { readonly ok: true; readonly adapter:
|
|
50
|
+
export type Admission = { readonly ok: true; readonly adapter: Integrations } | { readonly ok: false; readonly cards: Card[] };
|
|
51
51
|
|
|
52
52
|
export type AdmitInput = {
|
|
53
53
|
readonly name: string;
|
package/src/adapters/argument.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { z } from "zod";
|
|
|
2
2
|
import { PermanentError } from "../adapter-api.ts";
|
|
3
3
|
|
|
4
4
|
/** A caller's argument, parsed. One that fails is the caller's mistake, which no retry fixes. */
|
|
5
|
-
export const parseArgument = <
|
|
5
|
+
export const parseArgument = <S extends z.ZodType>(schema: S, value: z.input<S>): z.output<S> => {
|
|
6
6
|
const parsed = schema.safeParse(value);
|
|
7
7
|
|
|
8
8
|
if (!parsed.success) throw new PermanentError(z.prettifyError(parsed.error));
|
package/src/adapters/claim.ts
CHANGED
|
@@ -1,23 +1,21 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { z } from "zod";
|
|
3
|
-
import { type Claim, type ClaimResult, PermanentError } from "../adapter-api.ts";
|
|
3
|
+
import { type Claim, type ClaimResult, Holder, type ItemId, PermanentError } from "../adapter-api.ts";
|
|
4
4
|
import { parseArgument } from "./argument.ts";
|
|
5
5
|
|
|
6
|
-
const Holder = z.string().regex(/^[A-Za-z0-9][\w.@:-]{0,99}$/, "a holder is a plain name: letters, digits and . _ @ : -");
|
|
7
|
-
|
|
8
6
|
const Ttl = z.int().positive("a claim lasts a positive whole number of milliseconds");
|
|
9
7
|
|
|
10
8
|
const CLAIM_SUBJECT = /^hum-claim (\S+) (\d+)$/;
|
|
11
9
|
|
|
12
10
|
export type ClaimMode = "claim" | "renew";
|
|
13
11
|
|
|
14
|
-
/** A provider's claims. `S` is the standing claim with whatever the provider needs to write over it. */
|
|
15
|
-
export type ClaimStore<S extends { readonly claim: Claim }> = {
|
|
16
|
-
readonly standing: (id:
|
|
12
|
+
/** A provider's claims. `S` is the standing claim with whatever the provider needs to write over it; `F` is what its write answers when it fails. */
|
|
13
|
+
export type ClaimStore<S extends { readonly claim: Claim }, F = never> = {
|
|
14
|
+
readonly standing: (id: ItemId) => Promise<S | undefined>;
|
|
17
15
|
/** Asked only before a first claim: a standing one already proves the item was taken while ready. */
|
|
18
|
-
readonly isReady: (id:
|
|
19
|
-
/** Writes `claim` over `over`; false when another writer moved first. */
|
|
20
|
-
readonly write: (claim: Claim, over: S | undefined) => Promise<boolean
|
|
16
|
+
readonly isReady: (id: ItemId) => Promise<boolean>;
|
|
17
|
+
/** Writes `claim` over `over`; false when another writer moved first, `F` when the write failed. */
|
|
18
|
+
readonly write: (claim: Claim, over: S | undefined) => Promise<boolean | NoInfer<F>>;
|
|
21
19
|
};
|
|
22
20
|
|
|
23
21
|
export type ClaimDecision =
|
|
@@ -34,7 +32,7 @@ const isHeldByAnother = (standing: Claim, holder: string, now: number): boolean
|
|
|
34
32
|
* Who may write a claim, given the one standing. An unclaimed or expired item goes to whoever writes first;
|
|
35
33
|
* the holder claiming again renews; a renewal by anyone else loses to the standing claim.
|
|
36
34
|
*/
|
|
37
|
-
|
|
35
|
+
const decideClaim = (mode: ClaimMode, holder: string, standing: Claim | undefined, now: number): ClaimDecision => {
|
|
38
36
|
if (standing && isHeldByAnother(standing, holder, now)) return { kind: "held", claim: standing };
|
|
39
37
|
|
|
40
38
|
if (mode === "renew" && standing?.holder !== holder) return standing ? { kind: "held", claim: standing } : { kind: "unheld" };
|
|
@@ -44,8 +42,8 @@ export const decideClaim = (mode: ClaimMode, holder: string, standing: Claim | u
|
|
|
44
42
|
|
|
45
43
|
/** `claim` and `renew` over one provider's store: an atomic take whose race the provider's write settles. */
|
|
46
44
|
export const claimer =
|
|
47
|
-
<S extends { readonly claim: Claim }>(store: ClaimStore<S>, now: () => number) =>
|
|
48
|
-
async (mode: ClaimMode, item:
|
|
45
|
+
<S extends { readonly claim: Claim }, F = never>(store: ClaimStore<S, F>, now: () => number) =>
|
|
46
|
+
async (mode: ClaimMode, item: ItemId, holder: Holder, ttlMs: number): Promise<ClaimResult | F> => {
|
|
49
47
|
const who = parseArgument(Holder, holder);
|
|
50
48
|
const ttl = parseArgument(Ttl, ttlMs);
|
|
51
49
|
const standing = await store.standing(item);
|
|
@@ -61,7 +59,11 @@ export const claimer =
|
|
|
61
59
|
|
|
62
60
|
const claim: Claim = { item, holder: who, expiresAt: now() + ttl };
|
|
63
61
|
|
|
64
|
-
|
|
62
|
+
const written = await store.write(claim, standing);
|
|
63
|
+
|
|
64
|
+
if (written === true) return { won: true, claim };
|
|
65
|
+
|
|
66
|
+
if (written !== false) return written;
|
|
65
67
|
|
|
66
68
|
const winner = await store.standing(item);
|
|
67
69
|
|
|
@@ -85,9 +87,10 @@ export const claimCommitMessage = (claim: Claim): string => {
|
|
|
85
87
|
};
|
|
86
88
|
|
|
87
89
|
/** The claim a commit message holds, or undefined for any other commit. */
|
|
88
|
-
export const claimOfCommit = (item:
|
|
90
|
+
export const claimOfCommit = (item: ItemId, message: string): Claim | undefined => {
|
|
89
91
|
const [subject = ""] = message.split("\n");
|
|
90
92
|
const match = CLAIM_SUBJECT.exec(subject);
|
|
93
|
+
const holder = Holder.safeParse(match?.[1]);
|
|
91
94
|
|
|
92
|
-
return
|
|
95
|
+
return holder.success && match?.[2] ? { item, holder: holder.data, expiresAt: Number(match[2]) } : undefined;
|
|
93
96
|
};
|