@moo.md/hum 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/defaults/hum.json +321 -0
- package/defaults/kinds/check.json +183 -0
- package/defaults/kinds/docs.json +151 -0
- package/defaults/kinds/feature.json +202 -0
- package/defaults/kinds/fix.json +183 -0
- package/defaults/kinds/hotfix.json +164 -0
- package/defaults/roles/build.md +3 -0
- package/defaults/roles/fix.md +1 -0
- package/defaults/roles/plan.md +3 -0
- package/defaults/roles/review.md +3 -0
- package/evals/alignment/RESULTS.md +8 -0
- package/evals/alignment/cases.jsonl +128 -0
- package/evals/alignment/sample.jsonl +20 -0
- package/evals/kind/RESULTS.md +5 -0
- package/evals/kind/cases.jsonl +24 -0
- package/evals/plan-outcome/RESULTS.md +4 -0
- package/evals/plan-outcome/cases.jsonl +12 -0
- package/evals/repeat-pair/RESULTS.md +4 -0
- package/evals/repeat-pair/cases.jsonl +51 -0
- package/evals/reply-option/RESULTS.md +3 -0
- package/evals/reply-option/cases.jsonl +45 -0
- package/evals/review-verdict/RESULTS.md +4 -0
- package/evals/review-verdict/cases.jsonl +12 -0
- package/package.json +88 -4
- package/schema/hum.schema.json +1355 -0
- package/scripts/eval.ts +244 -0
- package/src/adapter-api.ts +134 -0
- package/src/adapters/admit.ts +111 -0
- package/src/adapters/argument.ts +11 -0
- package/src/adapters/claim.ts +93 -0
- package/src/adapters/claude-bg.ts +392 -0
- package/src/adapters/claude-p.ts +599 -0
- package/src/adapters/contract.ts +231 -0
- package/src/adapters/fake.ts +157 -0
- package/src/adapters/git.ts +156 -0
- package/src/adapters/github.ts +479 -0
- package/src/adapters/guard.ts +152 -0
- package/src/adapters/history.ts +215 -0
- package/src/adapters/item-file.ts +102 -0
- package/src/adapters/limit.ts +67 -0
- package/src/adapters/markdown.ts +365 -0
- package/src/adapters/marked.ts +127 -0
- package/src/adapters/model-call.ts +75 -0
- package/src/adapters/once.ts +45 -0
- package/src/adapters/owner-commit.ts +60 -0
- package/src/adapters/pull-requests.ts +253 -0
- package/src/adapters/registry.ts +156 -0
- package/src/adapters/roundup.ts +293 -0
- package/src/adapters/serial.ts +22 -0
- package/src/adapters/transcript.ts +99 -0
- package/src/cli/commands.ts +46 -0
- package/src/cli/main.ts +8 -0
- package/src/cli/ops.ts +224 -0
- package/src/cli/service.ts +156 -0
- package/src/cli/setup.ts +141 -0
- package/src/cli/shared-watch.ts +127 -0
- package/src/config/change.ts +54 -0
- package/src/config/checks.ts +217 -0
- package/src/config/git-ref-name.d.ts +3 -0
- package/src/config/hash.ts +6 -0
- package/src/config/json.ts +6 -0
- package/src/config/load.ts +88 -0
- package/src/config/machine.ts +20 -0
- package/src/config/migrate.ts +61 -0
- package/src/config/refusal.ts +58 -0
- package/src/config/schema.ts +465 -0
- package/src/config/settings.ts +6 -0
- package/src/config/state-map.ts +76 -0
- package/src/decide/alignment.ts +79 -0
- package/src/decide/answer.ts +131 -0
- package/src/decide/kind.ts +39 -0
- package/src/decide/labels.ts +137 -0
- package/src/decide/merge.ts +102 -0
- package/src/decide/prose.ts +109 -0
- package/src/decide/question.ts +159 -0
- package/src/decide/review.ts +10 -0
- package/src/decide/risk.ts +216 -0
- package/src/decide/taste.ts +37 -0
- package/src/exec.ts +85 -0
- package/src/index.ts +200 -0
- package/src/loop/breaks.ts +8 -0
- package/src/loop/check-item.ts +76 -0
- package/src/loop/ci.ts +8 -0
- package/src/loop/github.ts +81 -0
- package/src/loop/groups.ts +108 -0
- package/src/loop/memory.ts +70 -0
- package/src/loop/notes.ts +76 -0
- package/src/loop/pair.ts +39 -0
- package/src/loop/pass.ts +99 -0
- package/src/loop/proof.ts +85 -0
- package/src/loop/prove-item.ts +33 -0
- package/src/loop/records.ts +75 -0
- package/src/loop/signals.ts +130 -0
- package/src/measure/inputs.ts +28 -0
- package/src/measure/measure.ts +68 -0
- package/src/service/cards.ts +24 -0
- package/src/service/ci-tick.ts +28 -0
- package/src/service/launchd.ts +16 -0
- package/src/service/supervise.ts +145 -0
- package/src/service/systemd.ts +44 -0
- package/src/service/token.ts +21 -0
- package/src/service/unit.ts +29 -0
- package/src/service/wake.ts +30 -0
- package/src/service/workflows.ts +97 -0
- package/src/setup/propose.ts +120 -0
- package/src/setup/prove.ts +216 -0
- package/src/setup/scan.ts +186 -0
- package/src/setup/scratch.ts +81 -0
- package/src/setup/shipped.ts +15 -0
- package/src/setup/tables.ts +101 -0
- package/src/setup/waits.ts +58 -0
- package/src/setup/write.ts +161 -0
- package/src/slots/env.ts +42 -0
- package/src/slots/pool.ts +329 -0
- package/src/slots/scan.ts +26 -0
- package/src/slots/worktree.ts +17 -0
- package/src/state/events.ts +52 -0
- package/src/state/faults.ts +73 -0
- package/src/state/format.ts +50 -0
- package/src/state/fs.ts +100 -0
- package/src/state/items.ts +195 -0
- package/src/state/open.ts +80 -0
- package/src/state/ops.ts +76 -0
- package/src/state/records.ts +35 -0
- package/src/state/runs.ts +99 -0
- package/src/state/saved.ts +119 -0
- package/src/status/file.ts +49 -0
- package/src/status/format.ts +132 -0
- package/src/status/sections.ts +14 -0
- package/src/status/version.ts +2 -0
- package/src/time.ts +7 -0
- package/src/worker/alignment-report.ts +18 -0
- package/src/worker/answers.ts +153 -0
- package/src/worker/base-check.ts +45 -0
- package/src/worker/checks.ts +40 -0
- package/src/worker/kind.ts +68 -0
- package/src/worker/land.ts +509 -0
- package/src/worker/lifecycle.ts +97 -0
- package/src/worker/merged.ts +29 -0
- package/src/worker/open.ts +100 -0
- package/src/worker/prose-run.ts +56 -0
- package/src/worker/questions.ts +89 -0
- package/src/worker/scheduler.ts +135 -0
- package/src/worker/signature.ts +41 -0
- package/src/worker/tally.ts +25 -0
- package/src/worker/taste.ts +38 -0
- package/src/worker/thin-path.ts +886 -0
- package/src/worker/worker.ts +750 -0
- package/surface.json +8787 -0
- package/README.md +0 -3
package/scripts/eval.ts
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import { parseArgs } from "node:util";
|
|
5
|
+
import { AgentDecisionExhaustedError } from "@statelyai/agent";
|
|
6
|
+
import PQueue from "p-queue";
|
|
7
|
+
import { z } from "zod";
|
|
8
|
+
import { type Finished, type Runner, startRunner } from "../src/adapters/claude-p.ts";
|
|
9
|
+
import { modelCallOver } from "../src/adapters/model-call.ts";
|
|
10
|
+
import { readConfigDir } from "../src/config/load.ts";
|
|
11
|
+
import { BOUNDARIES, type Boundary, HumConfig } from "../src/config/schema.ts";
|
|
12
|
+
import { alignmentQuestion, raisesLevel } from "../src/decide/alignment.ts";
|
|
13
|
+
import { replyQuestion } from "../src/decide/answer.ts";
|
|
14
|
+
import { kindQuestion } from "../src/decide/kind.ts";
|
|
15
|
+
import { tasteQuestion } from "../src/decide/taste.ts";
|
|
16
|
+
import { pairQuestion } from "../src/loop/pair.ts";
|
|
17
|
+
import type { ReviewNote } from "../src/loop/records.ts";
|
|
18
|
+
import { pairing, prepare } from "../src/loop/signals.ts";
|
|
19
|
+
import type { JsonSchema, ProsePurpose } from "../src/decide/prose.ts";
|
|
20
|
+
import { openState } from "../src/state/open.ts";
|
|
21
|
+
import { decideProse } from "../src/worker/prose-run.ts";
|
|
22
|
+
import { PROSE_PROMPTS } from "../src/worker/thin-path.ts";
|
|
23
|
+
import { SECOND_MS } from "../src/time.ts";
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Measures one model-judgment boundary on its labelled cases, through the same prompts and fences the engine runs,
|
|
27
|
+
* and adds a row to its RESULTS.md. `bun run eval -- --model <id>` measures every boundary on that model id; naming
|
|
28
|
+
* boundaries measures only those. A case the runner held or limited counts in no rate. `--runs <n>` repeats each
|
|
29
|
+
* boundary and pools the runs, since one run proves little; `--write <folder>` records each pooled rate under
|
|
30
|
+
* `measuredModels` in that folder's `hum.json`, which is how setup measures an owner's own models.
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
const { values, positionals } = parseArgs({ allowPositionals: true, options: { model: { type: "string" }, limit: { type: "string" }, runs: { type: "string" }, write: { type: "string" } } });
|
|
34
|
+
/** Boundaries whose labelled cases only the owner can give; they are measured when named, from the cases the owner puts in their evals folder. */
|
|
35
|
+
const OWNER_LABELLED: readonly Boundary[] = ["taste"];
|
|
36
|
+
|
|
37
|
+
const boundaries: readonly Boundary[] = positionals.length > 0 ? z.array(z.enum(BOUNDARIES)).parse(positionals) : BOUNDARIES.filter((boundary) => !OWNER_LABELLED.includes(boundary));
|
|
38
|
+
const root = join(import.meta.dir, "..");
|
|
39
|
+
const raw = readConfigDir(join(root, "defaults"));
|
|
40
|
+
const config = { hum: HumConfig.parse(raw.hum), roles: raw.roles };
|
|
41
|
+
const model = values.model ?? config.hum.decisionModel;
|
|
42
|
+
const runs = values.runs === undefined ? 1 : Number(values.runs);
|
|
43
|
+
const limit = values.limit === undefined ? Number.POSITIVE_INFINITY : Number(values.limit);
|
|
44
|
+
|
|
45
|
+
type Verdict = "right" | "wrong" | "abstained" | "halted";
|
|
46
|
+
|
|
47
|
+
const state = openState(mkdtempSync(join(tmpdir(), "hum-eval-")));
|
|
48
|
+
const started = await startRunner({ state, model });
|
|
49
|
+
|
|
50
|
+
if (started.check.kind !== "finished") throw new Error(`the start check ended as ${started.check.kind}`);
|
|
51
|
+
|
|
52
|
+
let calls = 1;
|
|
53
|
+
const runner: Runner = { ...started.runner, run: (request) => (calls++, started.runner.run(request)) };
|
|
54
|
+
const judge = modelCallOver(runner, { model, timeoutMs: config.hum.modelCallTimeoutSeconds * SECOND_MS, trusts: () => true });
|
|
55
|
+
|
|
56
|
+
/** A run that ended without an answer: a limit or a hold. It says nothing about the boundary. */
|
|
57
|
+
class Halted extends Error {}
|
|
58
|
+
|
|
59
|
+
const answerRun = (cwd: string) => async (prompt: string, schema: JsonSchema): Promise<Finished> => {
|
|
60
|
+
const outcome = await runner.run({ prompt, cwd, model, effort: "low", schema });
|
|
61
|
+
|
|
62
|
+
if (outcome.kind !== "finished") throw new Halted(outcome.kind);
|
|
63
|
+
|
|
64
|
+
return outcome;
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
/** The pick a plan or review run settles on, as the machine would take it. */
|
|
68
|
+
const prosePick = async (purpose: ProsePurpose, prompt: string, diff: string, cwd: string): Promise<string | Verdict> => {
|
|
69
|
+
try {
|
|
70
|
+
return (await decideProse({ purpose, prompt, diff, model, run: answerRun(cwd) })).pick.event;
|
|
71
|
+
} catch (error) {
|
|
72
|
+
if (error instanceof Halted) return "halted";
|
|
73
|
+
|
|
74
|
+
if (error instanceof AgentDecisionExhaustedError) return "abstained";
|
|
75
|
+
|
|
76
|
+
throw error;
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
const scored = (picked: string | Verdict, label: string): Verdict => (picked === "halted" || picked === "abstained" ? picked : picked === label ? "right" : "wrong");
|
|
81
|
+
|
|
82
|
+
/** A small repo the plan cases name, so a plan run has files to read. */
|
|
83
|
+
const planRepo = (): string => {
|
|
84
|
+
const dir = mkdtempSync(join(tmpdir(), "hum-eval-repo-"));
|
|
85
|
+
const files = {
|
|
86
|
+
"package.json": '{ "name": "acme", "version": "1.4.0", "bin": { "acme": "src/cli.ts" } }\n',
|
|
87
|
+
"README.md": "# acme\n\nThe CLI will recieve a file and print it.\n\n| Plan | Price |\n|---|---|\n| Pro | $10 |\n",
|
|
88
|
+
"src/cli.ts": 'import { fmtDate } from "./dates.ts";\n\nconsole.log("\\u001b[32macme\\u001b[0m", fmtDate(new Date()));\n',
|
|
89
|
+
"src/dates.ts": "export const fmtDate = (date: Date): string => date.toISOString().slice(0, 10);\n",
|
|
90
|
+
"src/limits.ts": "export const clamp = (value: number, min: number, max: number): number => Math.min(max, Math.max(min, value));\n",
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
for (const [file, text] of Object.entries(files)) {
|
|
94
|
+
mkdirSync(dirname(join(dir, file)), { recursive: true });
|
|
95
|
+
writeFileSync(join(dir, file), text);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
return dir;
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
const KindCase = z.object({ case: z.string(), label: z.string(), title: z.string(), body: z.string() });
|
|
102
|
+
const AlignmentCase = z.object({ case: z.string(), label: z.enum(["reverted", "fixed", "clean"]), subject: z.string(), diff: z.string() });
|
|
103
|
+
const PlanCase = z.object({ case: z.string(), label: z.string(), body: z.string() });
|
|
104
|
+
const ReplyCase = z.object({ case: z.string(), label: z.string(), question: z.string(), options: z.array(z.string()), reply: z.string() });
|
|
105
|
+
const ReviewCase = z.object({ case: z.string(), label: z.string(), item: z.string(), diff: z.string() });
|
|
106
|
+
const PairCase = z.object({ case: z.string(), label: z.enum(["same", "different"]), a: z.object({ text: z.string() }), b: z.object({ text: z.string() }) });
|
|
107
|
+
const TasteCase = z.object({ case: z.string(), label: z.enum(["keeps", "breaks"]), rule: z.string(), diff: z.string() });
|
|
108
|
+
|
|
109
|
+
/** Calls at once: enough to finish a set in minutes, few enough not to trip a rate limit. */
|
|
110
|
+
const AT_ONCE = 4;
|
|
111
|
+
|
|
112
|
+
const eachOf = <T>(cases: readonly T[], measure: (found: T) => Promise<Verdict>): Promise<Verdict[]> => new PQueue({ concurrency: AT_ONCE }).addAll(cases.map((found) => () => measure(found)));
|
|
113
|
+
|
|
114
|
+
const read = async <T>(boundary: Boundary, schema: z.ZodType<T>, file = "cases.jsonl"): Promise<T[]> =>
|
|
115
|
+
(await Bun.file(join(root, "evals", boundary, file)).text())
|
|
116
|
+
.split("\n")
|
|
117
|
+
.filter(Boolean)
|
|
118
|
+
.map((line) => schema.parse(JSON.parse(line)));
|
|
119
|
+
|
|
120
|
+
/** The confusion counts of the review-note boundary: its rate is precision, and recall is recorded beside it. */
|
|
121
|
+
const pairCounts = { tp: 0, fp: 0, fn: 0, tn: 0, pairedBySignals: 0 };
|
|
122
|
+
|
|
123
|
+
const noteOf = (text: string, sourceId: string): ReviewNote => ({ source: "review-comment", sourceId, change: sourceId, hunk: "", text, at: 0 });
|
|
124
|
+
|
|
125
|
+
const BOUNDARY_RUNS: Record<Boundary, () => Promise<Verdict[]>> = {
|
|
126
|
+
kind: async () => {
|
|
127
|
+
const cases = (await read("kind", KindCase)).slice(0, limit);
|
|
128
|
+
|
|
129
|
+
return eachOf(cases, async (found) => {
|
|
130
|
+
const judged = await judge(kindQuestion(found, config.hum));
|
|
131
|
+
|
|
132
|
+
return judged.kind === "abstained" ? "abstained" : scored(judged.value.kind, found.label);
|
|
133
|
+
});
|
|
134
|
+
},
|
|
135
|
+
alignment: async () => {
|
|
136
|
+
const sample = (await read("alignment", AlignmentCase, "sample.jsonl")).slice(0, limit);
|
|
137
|
+
|
|
138
|
+
return eachOf(sample, async (found) => {
|
|
139
|
+
const judged = await judge(alignmentQuestion({ title: found.subject, body: "" }, found.diff, config.hum));
|
|
140
|
+
|
|
141
|
+
if (judged.kind === "abstained") return "abstained";
|
|
142
|
+
|
|
143
|
+
return raisesLevel(0, config.hum.levels.length, judged.value, config.hum) === (found.label !== "clean") ? "right" : "wrong";
|
|
144
|
+
});
|
|
145
|
+
},
|
|
146
|
+
"plan-outcome": async () => {
|
|
147
|
+
const repo = planRepo();
|
|
148
|
+
const cases = (await read("plan-outcome", PlanCase)).slice(0, limit);
|
|
149
|
+
|
|
150
|
+
return eachOf(cases, async (found) => scored(await prosePick("plan", PROSE_PROMPTS.plan(config, { id: found.case, title: found.body.split("\n")[0] ?? "", body: found.body }), "", repo), found.label));
|
|
151
|
+
},
|
|
152
|
+
"reply-option": async () => {
|
|
153
|
+
const cases = (await read("reply-option", ReplyCase)).slice(0, limit);
|
|
154
|
+
|
|
155
|
+
return eachOf(cases, async (found) => {
|
|
156
|
+
const judged = await judge(replyQuestion(found.question, found.options, found.reply));
|
|
157
|
+
|
|
158
|
+
return judged.kind === "abstained" ? "abstained" : scored(judged.value.option, found.label);
|
|
159
|
+
});
|
|
160
|
+
},
|
|
161
|
+
"repeat-pair": async () => {
|
|
162
|
+
const cases = (await read("repeat-pair", PairCase)).slice(0, limit);
|
|
163
|
+
const open = cases.filter((found) => pairing(prepare(noteOf(found.a.text, "a")), prepare(noteOf(found.b.text, "b"))).signal === undefined);
|
|
164
|
+
|
|
165
|
+
pairCounts.pairedBySignals = cases.length - open.length;
|
|
166
|
+
|
|
167
|
+
return eachOf(open, async (found) => {
|
|
168
|
+
const judged = await judge(pairQuestion(noteOf(found.a.text, "a"), noteOf(found.b.text, "b")));
|
|
169
|
+
|
|
170
|
+
if (judged.kind === "abstained") return "abstained";
|
|
171
|
+
|
|
172
|
+
const said = judged.value.same;
|
|
173
|
+
const truth = found.label === "same";
|
|
174
|
+
|
|
175
|
+
pairCounts[said ? (truth ? "tp" : "fp") : truth ? "fn" : "tn"]++;
|
|
176
|
+
|
|
177
|
+
return said === truth ? "right" : "wrong";
|
|
178
|
+
});
|
|
179
|
+
},
|
|
180
|
+
taste: async () => {
|
|
181
|
+
const cases = (await read("taste", TasteCase)).slice(0, limit);
|
|
182
|
+
|
|
183
|
+
return eachOf(cases, async (found) => {
|
|
184
|
+
const judged = await judge(tasteQuestion(found.rule, found.diff, config.hum));
|
|
185
|
+
|
|
186
|
+
return judged.kind === "abstained" ? "abstained" : scored(judged.value.keeps ? "keeps" : "breaks", found.label);
|
|
187
|
+
});
|
|
188
|
+
},
|
|
189
|
+
"review-verdict": async () => {
|
|
190
|
+
const cases = (await read("review-verdict", ReviewCase)).slice(0, limit);
|
|
191
|
+
|
|
192
|
+
return eachOf(cases, async (found) => scored(await prosePick("review", PROSE_PROMPTS.review(config, { id: found.case, title: found.item, body: "" }, found.diff), found.diff, tmpdir()), found.label));
|
|
193
|
+
},
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
const header = "| Date | Model id | n | Right | Wrong | Abstained | Rate | Live calls |\n|---|---|---|---|---|---|---|---|\n";
|
|
197
|
+
|
|
198
|
+
const pairHeader = "| Date | Model id | n | TP | FP | FN | TN | Abstained | Paired by signals | Precision | Recall | Live calls |\n|---|---|---|---|---|---|---|---|---|---|---|---|\n";
|
|
199
|
+
|
|
200
|
+
const ratio = (part: number, whole: number): string => (whole === 0 ? "0.00" : (part / whole).toFixed(2));
|
|
201
|
+
|
|
202
|
+
const pairRow = (day: string, abstained: number, used: number): string =>
|
|
203
|
+
`| ${day} | ${model} | ${pairCounts.tp + pairCounts.fp + pairCounts.fn + pairCounts.tn} | ${pairCounts.tp} | ${pairCounts.fp} | ${pairCounts.fn} | ${pairCounts.tn} | ${abstained} | ${pairCounts.pairedBySignals} | ${ratio(pairCounts.tp, pairCounts.tp + pairCounts.fp)} | ${ratio(pairCounts.tp, pairCounts.tp + pairCounts.fn)} | ${used} |`;
|
|
204
|
+
|
|
205
|
+
const EMPTY_POOL = { right: 0, measured: 0 };
|
|
206
|
+
|
|
207
|
+
/** Sets one pooled rate in the owner's `hum.json`, where trust reads it. */
|
|
208
|
+
const recordRate = async (folder: string, boundary: Boundary, rate: number): Promise<void> => {
|
|
209
|
+
const file = join(folder, "hum.json");
|
|
210
|
+
const written = z.looseObject({ measuredModels: z.record(z.string(), z.record(z.string(), z.number())).default({}) }).parse(JSON.parse(await Bun.file(file).text()));
|
|
211
|
+
|
|
212
|
+
await Bun.write(file, `${JSON.stringify({ ...written, measuredModels: { ...written.measuredModels, [boundary]: { ...written.measuredModels[boundary], [model]: Number(rate.toFixed(2)) } } }, null, "\t")}\n`);
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
for (const boundary of boundaries) {
|
|
216
|
+
const pool = { ...EMPTY_POOL };
|
|
217
|
+
|
|
218
|
+
Object.assign(pairCounts, { tp: 0, fp: 0, fn: 0, tn: 0, pairedBySignals: 0 });
|
|
219
|
+
|
|
220
|
+
for (let run = 1; run <= runs; run++) {
|
|
221
|
+
const before = calls;
|
|
222
|
+
const verdicts = await BOUNDARY_RUNS[boundary]();
|
|
223
|
+
const count = (verdict: Verdict) => verdicts.filter((found) => found === verdict).length;
|
|
224
|
+
const measured = verdicts.length - count("halted");
|
|
225
|
+
const rate = measured === 0 ? 0 : count("right") / measured;
|
|
226
|
+
const day = new Date().toISOString().slice(0, 10);
|
|
227
|
+
const row = boundary === "repeat-pair" ? pairRow(day, count("abstained"), calls - before) : `| ${day} | ${model} | ${measured} | ${count("right")} | ${count("wrong")} | ${count("abstained")} | ${rate.toFixed(2)} | ${calls - before} |`;
|
|
228
|
+
const results = join(values.write ?? root, "evals", boundary, "RESULTS.md");
|
|
229
|
+
const earlier = (await Bun.file(results).exists()) ? await Bun.file(results).text() : boundary === "repeat-pair" ? pairHeader : header;
|
|
230
|
+
|
|
231
|
+
pool.right += count("right");
|
|
232
|
+
pool.measured += measured;
|
|
233
|
+
await Bun.write(results, `${earlier}${row}\n`);
|
|
234
|
+
console.log(`${boundary}: ${row}`);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
const pooled = boundary === "repeat-pair" ? pairCounts.tp / (pairCounts.tp + pairCounts.fp || 1) : pool.right / (pool.measured || 1);
|
|
238
|
+
|
|
239
|
+
console.log(`${boundary}: pooled over ${runs} run(s): ${pooled.toFixed(2)}`);
|
|
240
|
+
|
|
241
|
+
if (values.write !== undefined) await recordRate(values.write, boundary, pooled);
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
console.log(`${calls} live calls, the start check included. Record each rate under measuredModels in defaults/hum.json; a model decides a boundary alone only at minJudgeRate or above; repeat-pair takes its precision, against repeatPrecision.`);
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { FaultInput } from "./state/faults.ts";
|
|
3
|
+
|
|
4
|
+
export type { FaultInput };
|
|
5
|
+
|
|
6
|
+
/** Bumped on every change to the API an adapter implements; a breaking change bumps the engine's major version. */
|
|
7
|
+
export const ADAPTER_API_VERSION = 1;
|
|
8
|
+
|
|
9
|
+
/** Failure that retrying cannot fix: a reply of the wrong shape, a refused push. The engine surfaces it at once. */
|
|
10
|
+
export class PermanentError extends Error {
|
|
11
|
+
override name: string = "PermanentError";
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** The most characters an item id holds. */
|
|
15
|
+
export const ITEM_ID_LENGTH = 100;
|
|
16
|
+
|
|
17
|
+
const ITEM_ID_MESSAGE = "an item id is letters, digits, dot, dash or underscore";
|
|
18
|
+
|
|
19
|
+
/** An item id is safe as a file name and as a git ref component. */
|
|
20
|
+
export const ItemId = z
|
|
21
|
+
.string()
|
|
22
|
+
.max(ITEM_ID_LENGTH, ITEM_ID_MESSAGE)
|
|
23
|
+
.regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/, ITEM_ID_MESSAGE)
|
|
24
|
+
.refine((id) => !id.includes("..") && !id.endsWith(".lock"), "an item id has no '..' and does not end in '.lock'");
|
|
25
|
+
|
|
26
|
+
export type ItemId = z.infer<typeof ItemId>;
|
|
27
|
+
|
|
28
|
+
const text = z.string().min(1);
|
|
29
|
+
const time = z.int().nonnegative();
|
|
30
|
+
|
|
31
|
+
export const THREAD_KINDS = ["comment", "question", "answer", "decision"] as const;
|
|
32
|
+
|
|
33
|
+
/** Who wrote a thread entry: hum, which marks its comments, or a person typing without a marker. */
|
|
34
|
+
export const THREAD_AUTHORS = ["hum", "person"] as const;
|
|
35
|
+
|
|
36
|
+
/** One entry on an item's thread, oldest first. */
|
|
37
|
+
export const ThreadEntry = z.strictObject({
|
|
38
|
+
kind: z.enum(THREAD_KINDS),
|
|
39
|
+
text,
|
|
40
|
+
at: time,
|
|
41
|
+
option: text.optional(),
|
|
42
|
+
/** A source that cannot tell leaves it out, and the entry reads as hum's. */
|
|
43
|
+
author: z.enum(THREAD_AUTHORS).optional(),
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
export type ThreadEntry = z.infer<typeof ThreadEntry>;
|
|
47
|
+
|
|
48
|
+
export const ItemRef = z.strictObject({ id: ItemId, title: text });
|
|
49
|
+
|
|
50
|
+
export type ItemRef = z.infer<typeof ItemRef>;
|
|
51
|
+
|
|
52
|
+
export const Item = ItemRef.extend({
|
|
53
|
+
body: z.string(),
|
|
54
|
+
/** True while the source lists the item as ready to claim. */
|
|
55
|
+
ready: z.boolean(),
|
|
56
|
+
thread: z.array(ThreadEntry),
|
|
57
|
+
/** Names the source tags the item with: its labels, or a markdown item's `kind`. One that names a kind sets the item's kind. */
|
|
58
|
+
labels: z.array(text).optional(),
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
export type Item = z.infer<typeof Item>;
|
|
62
|
+
|
|
63
|
+
export const Claim = z.strictObject({ item: ItemId, holder: text, expiresAt: time });
|
|
64
|
+
|
|
65
|
+
export type Claim = z.infer<typeof Claim>;
|
|
66
|
+
|
|
67
|
+
/** `claim` holds the winner: the caller on `won`, the current holder otherwise. */
|
|
68
|
+
export const ClaimResult = z.strictObject({ won: z.boolean(), claim: Claim });
|
|
69
|
+
|
|
70
|
+
export type ClaimResult = z.infer<typeof ClaimResult>;
|
|
71
|
+
|
|
72
|
+
/** `changed` is true when `token` differs from the one the caller passed in. */
|
|
73
|
+
export const Watch = z.strictObject({ token: text, changed: z.boolean() });
|
|
74
|
+
|
|
75
|
+
export type Watch = z.infer<typeof Watch>;
|
|
76
|
+
|
|
77
|
+
/** `detail` rides folded on the thread, for agents; the owner reads `text` alone. */
|
|
78
|
+
export type Question = { readonly text: string; readonly options?: readonly string[]; readonly detail?: string };
|
|
79
|
+
|
|
80
|
+
export type Answer = { readonly text: string; readonly option?: string };
|
|
81
|
+
|
|
82
|
+
export type FollowUp = { readonly title: string; readonly body: string };
|
|
83
|
+
|
|
84
|
+
/** The work source: the queue, the claims and the item threads. */
|
|
85
|
+
export interface WorkSource {
|
|
86
|
+
listReady(): Promise<ItemRef[]>;
|
|
87
|
+
read(id: string): Promise<Item>;
|
|
88
|
+
/** An atomic take. An unclaimed or expired item goes to the first caller; the same holder renews. */
|
|
89
|
+
claim(id: string, holder: string, ttlMs: number): Promise<ClaimResult>;
|
|
90
|
+
/** Extends the holder's own claim; `won: false` when another worker holds it now. */
|
|
91
|
+
renew(id: string, holder: string, ttlMs: number): Promise<ClaimResult>;
|
|
92
|
+
comment(id: string, text: string): Promise<void>;
|
|
93
|
+
/** Files a ready item. Filing the same `title` from the same `from` twice returns the first item. */
|
|
94
|
+
fileFollowUp(from: string, followUp: FollowUp): Promise<ItemRef>;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Where a run's questions and the choices made without the owner land. */
|
|
98
|
+
export interface Questions {
|
|
99
|
+
ask(id: string, question: Question): Promise<void>;
|
|
100
|
+
/** Records a choice made without the owner. */
|
|
101
|
+
decide(id: string, text: string): Promise<void>;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Where the owner's answers arrive. */
|
|
105
|
+
export interface Answers {
|
|
106
|
+
answer(id: string, answer: Answer): Promise<void>;
|
|
107
|
+
/** The one cheap read: has anything changed since `since`. An idle call costs no rate limit. */
|
|
108
|
+
watch(since?: string): Promise<Watch>;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export interface Release {
|
|
112
|
+
/** Puts the item's artifact where reviewers see it. */
|
|
113
|
+
publish(id: string, ref: string): Promise<void>;
|
|
114
|
+
/** Ships a published artifact. */
|
|
115
|
+
release(id: string, ref: string): Promise<void>;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** An adapter declares the API version it implements and the integrations it provides. */
|
|
119
|
+
export type Adapter = {
|
|
120
|
+
readonly apiVersion: number;
|
|
121
|
+
readonly workSource?: WorkSource;
|
|
122
|
+
readonly questions?: Questions;
|
|
123
|
+
readonly answers?: Answers;
|
|
124
|
+
readonly release?: Release;
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
export type AdapterContext = {
|
|
128
|
+
/** Raise a fault card, scoped to one item or provider. */
|
|
129
|
+
readonly report: (fault: FaultInput) => void;
|
|
130
|
+
readonly now: () => number;
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
/** The default export of an owner's `adapters/<name>.ts`. */
|
|
134
|
+
export type AdapterFactory = (context: AdapterContext) => Adapter | Promise<Adapter>;
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { extname, relative } from "node:path";
|
|
3
|
+
import type { Adapter } from "../adapter-api.ts";
|
|
4
|
+
import type { Faults } from "../state/faults.ts";
|
|
5
|
+
import type { Card } from "../status/format.ts";
|
|
6
|
+
import { type GuardOptions, guarded } from "./guard.ts";
|
|
7
|
+
import { refuseOtherVersion } from "./registry.ts";
|
|
8
|
+
|
|
9
|
+
/** Modules an adapter may not import: the state machine stays inside the engine. */
|
|
10
|
+
const ENGINE_MODULES = ["xstate", "@statelyai/agent"] as const;
|
|
11
|
+
|
|
12
|
+
const SOURCE_GLOB = new Bun.Glob("**/*.{ts,mts,cts,tsx,js,mjs,cjs,jsx}");
|
|
13
|
+
|
|
14
|
+
/** The parser for each source extension. */
|
|
15
|
+
const LOADERS = {
|
|
16
|
+
".ts": "ts",
|
|
17
|
+
".mts": "ts",
|
|
18
|
+
".cts": "ts",
|
|
19
|
+
".tsx": "tsx",
|
|
20
|
+
".js": "js",
|
|
21
|
+
".mjs": "js",
|
|
22
|
+
".cjs": "js",
|
|
23
|
+
".jsx": "jsx",
|
|
24
|
+
} as const satisfies Record<string, "ts" | "tsx" | "js" | "jsx">;
|
|
25
|
+
|
|
26
|
+
const isLoaderExtension = (extension: string): extension is keyof typeof LOADERS => extension in LOADERS;
|
|
27
|
+
|
|
28
|
+
const engineModuleOf = (path: string): string | undefined => ENGINE_MODULES.find((module) => path === module || path.startsWith(`${module}/`));
|
|
29
|
+
|
|
30
|
+
const isOutsideDependencies = (path: string): boolean => !path.split("/").includes("node_modules");
|
|
31
|
+
|
|
32
|
+
/** The files among `files` that import an engine module, with the module each imports. */
|
|
33
|
+
export const engineImporters = (files: readonly string[]): { file: string; module: string }[] =>
|
|
34
|
+
files.flatMap((file) => {
|
|
35
|
+
const extension = extname(file);
|
|
36
|
+
const loader = isLoaderExtension(extension) ? LOADERS[extension] : "ts";
|
|
37
|
+
|
|
38
|
+
const module = new Bun.Transpiler({ loader })
|
|
39
|
+
.scanImports(readFileSync(file, "utf8"))
|
|
40
|
+
.map((imported) => engineModuleOf(imported.path))
|
|
41
|
+
.find((found) => found !== undefined);
|
|
42
|
+
|
|
43
|
+
return module ? [{ file, module }] : [];
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
/** Every source file under `dir`, sorted; none when `dir` does not exist. */
|
|
47
|
+
export const sourceFiles = (dir: string): string[] =>
|
|
48
|
+
existsSync(dir) ? [...SOURCE_GLOB.scanSync({ cwd: dir, absolute: true })].filter(isOutsideDependencies).sort() : [];
|
|
49
|
+
|
|
50
|
+
export type Admission = { readonly ok: true; readonly adapter: Adapter } | { readonly ok: false; readonly cards: Card[] };
|
|
51
|
+
|
|
52
|
+
export type AdmitInput = {
|
|
53
|
+
readonly name: string;
|
|
54
|
+
readonly adapter: Adapter;
|
|
55
|
+
/** The adapter's own source files, scanned for engine imports. */
|
|
56
|
+
readonly files?: readonly string[];
|
|
57
|
+
/** Paths in cards are shown relative to this folder. */
|
|
58
|
+
readonly root?: string;
|
|
59
|
+
readonly faults: Faults;
|
|
60
|
+
readonly guard: Omit<GuardOptions, "provider" | "faults">;
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* The start check for one adapter. It refuses an adapter, with one card, when it implements another API version,
|
|
65
|
+
* when its source imports an engine module, or when its replies no longer have the shape the API defines.
|
|
66
|
+
* A refusal holds only that adapter; the caller runs the rest of hum.
|
|
67
|
+
*/
|
|
68
|
+
export const admit = async ({ name, adapter, files = [], root = process.cwd(), faults, guard }: AdmitInput): Promise<Admission> => {
|
|
69
|
+
const shown = (file: string) => relative(root, file) || file;
|
|
70
|
+
const versionCard = refuseOtherVersion(name, adapter, faults);
|
|
71
|
+
|
|
72
|
+
const importCards = engineImporters(files).map(({ file, module }) =>
|
|
73
|
+
faults.raise({
|
|
74
|
+
code: "adapter-imports-engine",
|
|
75
|
+
item: shown(file),
|
|
76
|
+
cause: `${shown(file)} imports "${module}", which an adapter may not.`,
|
|
77
|
+
fix: "Import only the adapter API; keep the state machine out of the adapter.",
|
|
78
|
+
}),
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
const refusals = [...(versionCard ? [versionCard] : []), ...importCards];
|
|
82
|
+
|
|
83
|
+
if (refusals.length > 0) return { ok: false, cards: refusals };
|
|
84
|
+
|
|
85
|
+
const wrapped = guarded(adapter, { ...guard, provider: name, faults });
|
|
86
|
+
|
|
87
|
+
try {
|
|
88
|
+
const [first] = (await wrapped.workSource?.listReady()) ?? [];
|
|
89
|
+
|
|
90
|
+
if (first) await wrapped.workSource?.read(first.id);
|
|
91
|
+
} catch (error) {
|
|
92
|
+
const raised = faults.list().filter((card) => card.item === name);
|
|
93
|
+
|
|
94
|
+
return {
|
|
95
|
+
ok: false,
|
|
96
|
+
cards:
|
|
97
|
+
raised.length > 0
|
|
98
|
+
? raised
|
|
99
|
+
: [
|
|
100
|
+
faults.raise({
|
|
101
|
+
code: "integration-probe-failed",
|
|
102
|
+
item: name,
|
|
103
|
+
cause: `The start check could not read ${name}: ${error instanceof Error ? error.message : String(error)}`.split("\n")[0] ?? "",
|
|
104
|
+
fix: `Check ${name}'s setup with /hum.`,
|
|
105
|
+
}),
|
|
106
|
+
],
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
return { ok: true, adapter: wrapped };
|
|
111
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { PermanentError } from "../adapter-api.ts";
|
|
3
|
+
|
|
4
|
+
/** A caller's argument, parsed. One that fails is the caller's mistake, which no retry fixes. */
|
|
5
|
+
export const parseArgument = <Out, In>(schema: z.ZodType<Out, In>, value: In): Out => {
|
|
6
|
+
const parsed = schema.safeParse(value);
|
|
7
|
+
|
|
8
|
+
if (!parsed.success) throw new PermanentError(z.prettifyError(parsed.error));
|
|
9
|
+
|
|
10
|
+
return parsed.data;
|
|
11
|
+
};
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { type Claim, type ClaimResult, PermanentError } from "../adapter-api.ts";
|
|
4
|
+
import { parseArgument } from "./argument.ts";
|
|
5
|
+
|
|
6
|
+
const Holder = z.string().regex(/^[A-Za-z0-9][\w.@:-]{0,99}$/, "a holder is a plain name: letters, digits and . _ @ : -");
|
|
7
|
+
|
|
8
|
+
const Ttl = z.int().positive("a claim lasts a positive whole number of milliseconds");
|
|
9
|
+
|
|
10
|
+
const CLAIM_SUBJECT = /^hum-claim (\S+) (\d+)$/;
|
|
11
|
+
|
|
12
|
+
export type ClaimMode = "claim" | "renew";
|
|
13
|
+
|
|
14
|
+
/** A provider's claims. `S` is the standing claim with whatever the provider needs to write over it. */
|
|
15
|
+
export type ClaimStore<S extends { readonly claim: Claim }> = {
|
|
16
|
+
readonly standing: (id: string) => Promise<S | undefined>;
|
|
17
|
+
/** Asked only before a first claim: a standing one already proves the item was taken while ready. */
|
|
18
|
+
readonly isReady: (id: string) => Promise<boolean>;
|
|
19
|
+
/** Writes `claim` over `over`; false when another writer moved first. */
|
|
20
|
+
readonly write: (claim: Claim, over: S | undefined) => Promise<boolean>;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export type ClaimDecision =
|
|
24
|
+
| { readonly kind: "held"; readonly claim: Claim }
|
|
25
|
+
| { readonly kind: "unheld" }
|
|
26
|
+
| { readonly kind: "write"; readonly first: boolean };
|
|
27
|
+
|
|
28
|
+
/** A claim stands until it expires. */
|
|
29
|
+
export const isLive = (claim: Claim | undefined, now: number): boolean => claim !== undefined && claim.expiresAt > now;
|
|
30
|
+
|
|
31
|
+
const isHeldByAnother = (standing: Claim, holder: string, now: number): boolean => isLive(standing, now) && standing.holder !== holder;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Who may write a claim, given the one standing. An unclaimed or expired item goes to whoever writes first;
|
|
35
|
+
* the holder claiming again renews; a renewal by anyone else loses to the standing claim.
|
|
36
|
+
*/
|
|
37
|
+
export const decideClaim = (mode: ClaimMode, holder: string, standing: Claim | undefined, now: number): ClaimDecision => {
|
|
38
|
+
if (standing && isHeldByAnother(standing, holder, now)) return { kind: "held", claim: standing };
|
|
39
|
+
|
|
40
|
+
if (mode === "renew" && standing?.holder !== holder) return standing ? { kind: "held", claim: standing } : { kind: "unheld" };
|
|
41
|
+
|
|
42
|
+
return { kind: "write", first: standing === undefined };
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
/** `claim` and `renew` over one provider's store: an atomic take whose race the provider's write settles. */
|
|
46
|
+
export const claimer =
|
|
47
|
+
<S extends { readonly claim: Claim }>(store: ClaimStore<S>, now: () => number) =>
|
|
48
|
+
async (mode: ClaimMode, item: string, holder: string, ttlMs: number): Promise<ClaimResult> => {
|
|
49
|
+
const who = parseArgument(Holder, holder);
|
|
50
|
+
const ttl = parseArgument(Ttl, ttlMs);
|
|
51
|
+
const standing = await store.standing(item);
|
|
52
|
+
const decision = decideClaim(mode, who, standing?.claim, now());
|
|
53
|
+
|
|
54
|
+
switch (decision.kind) {
|
|
55
|
+
case "held":
|
|
56
|
+
return { won: false, claim: decision.claim };
|
|
57
|
+
case "unheld":
|
|
58
|
+
throw new PermanentError(`"${who}" holds no claim on "${item}"`);
|
|
59
|
+
case "write": {
|
|
60
|
+
if (decision.first && !(await store.isReady(item))) throw new PermanentError(`item "${item}" is not ready to claim`);
|
|
61
|
+
|
|
62
|
+
const claim: Claim = { item, holder: who, expiresAt: now() + ttl };
|
|
63
|
+
|
|
64
|
+
if (await store.write(claim, standing)) return { won: true, claim };
|
|
65
|
+
|
|
66
|
+
const winner = await store.standing(item);
|
|
67
|
+
|
|
68
|
+
if (!winner) throw new PermanentError(`the claim on "${item}" lost a race to one that has since vanished`);
|
|
69
|
+
|
|
70
|
+
return { won: false, claim: winner.claim };
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
/** A claim written as a commit. The nonce keeps two writers' commits distinct even when all else matches. */
|
|
76
|
+
export const claimCommit = ({ holder, expiresAt }: Claim) => ({
|
|
77
|
+
headline: `hum-claim ${holder} ${expiresAt}`,
|
|
78
|
+
body: `nonce: ${randomUUID()}`,
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
export const claimCommitMessage = (claim: Claim): string => {
|
|
82
|
+
const { headline, body } = claimCommit(claim);
|
|
83
|
+
|
|
84
|
+
return `${headline}\n\n${body}`;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
/** The claim a commit message holds, or undefined for any other commit. */
|
|
88
|
+
export const claimOfCommit = (item: string, message: string): Claim | undefined => {
|
|
89
|
+
const [subject = ""] = message.split("\n");
|
|
90
|
+
const match = CLAIM_SUBJECT.exec(subject);
|
|
91
|
+
|
|
92
|
+
return match?.[1] && match[2] ? { item, holder: match[1], expiresAt: Number(match[2]) } : undefined;
|
|
93
|
+
};
|