@moo.md/hum 0.4.1 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/defaults/hum.json +39 -12
  2. package/npm-shrinkwrap.json +2 -2
  3. package/package.json +2 -1
  4. package/release.json +1 -1
  5. package/schema/hum.schema.json +63 -17
  6. package/scripts/eval.ts +58 -27
  7. package/src/adapters/claude-p.ts +7 -0
  8. package/src/adapters/model-call.ts +8 -4
  9. package/src/adapters/pull-requests.ts +11 -7
  10. package/src/cli/setup.ts +13 -18
  11. package/src/config/json.ts +3 -0
  12. package/src/config/migrate.ts +11 -0
  13. package/src/config/schema.ts +21 -6
  14. package/src/decide/alignment.ts +1 -1
  15. package/src/decide/merge.ts +21 -1
  16. package/src/decide/prose.ts +12 -2
  17. package/src/decide/risk.ts +1 -0
  18. package/src/loop/breaks.ts +17 -0
  19. package/src/loop/notes.ts +2 -2
  20. package/src/loop/pair.ts +2 -2
  21. package/src/loop/pass.ts +6 -3
  22. package/src/loop/records.ts +24 -1
  23. package/src/loop/signals.ts +2 -2
  24. package/src/measure/cases.ts +102 -0
  25. package/src/measure/inputs.ts +15 -1
  26. package/src/measure/judges.ts +52 -0
  27. package/src/measure/measure.ts +22 -3
  28. package/src/measure/rate.ts +9 -6
  29. package/src/measure/remeasure.ts +31 -0
  30. package/src/publish/handoff.ts +1 -1
  31. package/src/service/workflows.ts +4 -2
  32. package/src/setup/write.ts +2 -4
  33. package/src/state/events.ts +201 -19
  34. package/src/state/format.ts +9 -2
  35. package/src/state/item-log.ts +133 -0
  36. package/src/state/items.ts +6 -5
  37. package/src/state/open.ts +7 -2
  38. package/src/state/runs.ts +5 -0
  39. package/src/state/saved.ts +7 -2
  40. package/src/status/format.ts +12 -3
  41. package/src/worker/land.ts +36 -15
  42. package/src/worker/merged.ts +18 -8
  43. package/src/worker/open.ts +5 -1
  44. package/src/worker/prose-run.ts +6 -2
  45. package/src/worker/signature.ts +11 -5
  46. package/src/worker/thin-path.ts +69 -53
  47. package/src/worker/worker.ts +82 -12
  48. package/surface.json +969 -151
@@ -4,10 +4,10 @@ import { z } from "zod";
4
4
  import shippedConfig from "../../defaults/hum.json" with { type: "json" };
5
5
 
6
6
  /** The `hum.json` schema version this engine writes. */
7
- export const SCHEMA_VERSION = 2;
7
+ export const SCHEMA_VERSION = 3;
8
8
 
9
9
  /** The schema versions this engine reads, each migrated up to `SCHEMA_VERSION` by `migrateConfig`. */
10
- export const READABLE_SCHEMA_VERSIONS: readonly number[] = [1, 2];
10
+ export const READABLE_SCHEMA_VERSIONS: readonly number[] = [1, 2, 3];
11
11
 
12
12
  /** Parts the engine ships; the owner adds roles, never these. */
13
13
  export const FIXED_PARTS = [
@@ -127,6 +127,17 @@ const backoff = z
127
127
 
128
128
  const fraction = z.number().gt(0).lt(1);
129
129
 
130
+ const SHA256_HEX = /^[0-9a-f]{64}$/;
131
+
132
+ /** A model's measured rate at one boundary, and the hash of what the judge read when it was measured. */
133
+ const Score = z.strictObject({
134
+ rate: z.number().min(0).max(1),
135
+ /** Absent on a rate measured before hum kept the hash: what the judge read then is unknown. */
136
+ judgeHash: z.string().regex(SHA256_HEX, "a judge hash is a SHA-256 in hex").optional(),
137
+ });
138
+
139
+ export type Score = z.infer<typeof Score>;
140
+
130
141
  /** A name GitHub stores a secret under: a workflow reads it as `secrets.<name>`, and GitHub keeps the `GITHUB_` prefix for its own. */
131
142
  const secretName = z
132
143
  .string()
@@ -187,7 +198,7 @@ const Shipped = z.object({
187
198
  latestVersionCacheSeconds: seconds.describe("How long the newest published engine version is kept before it is read again."),
188
199
  watchCacheSeconds: seconds.describe("How long one read of what changed serves every `hum watch` in the repo before the next one reads again."),
189
200
  watchLockSeconds: seconds.describe("How long the lock of the one `hum watch` that reads stays held before another may take it over."),
190
- maxFixRuns: z.int().nonnegative().describe("Fix runs an item gets after failed checks before it ends stuck."),
201
+ maxFixRuns: z.int().nonnegative().describe("Fix runs an item gets after failed checks, counted since the owner last answered, before it ends stuck."),
191
202
  serviceLabel: z.string().regex(SERVICE_LABEL, "a service label is letters, digits, dot, dash and underscore").describe("The service unit's label, and the name its file takes."),
192
203
  maxCrashes: count.describe("Worker crashes within crashWindowSeconds that stop the service's restarts."),
193
204
  crashWindowSeconds: seconds.describe("The window in which maxCrashes crashes stop the service's restarts."),
@@ -256,9 +267,12 @@ const Shipped = z.object({
256
267
  decisionModel: name.describe("The model id of the cheap model call, made through claude -p."),
257
268
  modelCallTimeoutSeconds: seconds.describe("How long a cheap model call may take before it counts as an abstention."),
258
269
  measuredModels: z
259
- .record(z.enum(BOUNDARIES), z.record(name, z.number().min(0).max(1)))
260
- .describe("Per model-judgment boundary, the rate each model id scored on its labelled set; a pick from an id below 'minJudgeRate', or not measured, counts as an abstention."),
270
+ .record(z.enum(BOUNDARIES), z.record(name, Score))
271
+ .describe(
272
+ "Per model-judgment boundary, each model id's rate on its labelled set, and the judgeHash of the prompt, role text and answer form its judge read then; a pick from an id below 'minJudgeRate', or not measured, counts as an abstention. A rate whose judge has changed since keeps gating until a new measurement replaces it: the Mac worker measures it again.",
273
+ ),
261
274
  minJudgeRate: z.number().min(0).max(1).describe("The lowest measured rate at which hum trusts a model's pick at a boundary; below it the call abstains."),
275
+ evalRuns: count.describe("Runs pooled per boundary each time hum measures a model: in `hum setup measure`, and when a judge's prompt or role text changed since its rate was measured."),
262
276
  questionLimits: z
263
277
  .strictObject({ cells: count, wordCells: count })
264
278
  .describe("The most terminal cells a question holds, and the most any one word of it holds; a longer question is replaced by a short one and its text rides folded."),
@@ -297,7 +311,7 @@ const Shipped = z.object({
297
311
  memoryMaxBytes: count.describe("The most bytes memory.md holds; a new entry past it is refused."),
298
312
  memoryUnusedRuns: count.describe("Runs without use after which a memory.md entry is listed for the owner."),
299
313
  measureWindow: count.describe("Merged changes the per-merge numbers average over, and the window per path."),
300
- loopIntervalSeconds: seconds.describe("The time between passes that collect review notes, file check items and confirm CI ran their checks."),
314
+ loopIntervalSeconds: seconds.describe("The time between passes that collect review notes, file check items and confirm CI ran their checks, and between tries to measure again a judge whose prompt or role text changed."),
301
315
  proofTimeoutSeconds: seconds.describe("How long the repo's checks may run, on a run's branch, on the base after a merge or as a new check on one note's commit, before they are stopped and fail."),
302
316
  kindFloors: z.record(name, name).describe("A kind's lowest risk level: its changes take at least this level, as in { \"check\": \"high\" }."),
303
317
  checkKind: name.describe("The kind of the work item filed for a rule that review notes repeat: it turns the notes into one check."),
@@ -513,6 +527,7 @@ export const OWNER_CHANGEABLE = {
513
527
  "model call timeout": "modelCallTimeoutSeconds",
514
528
  "measured models": "measuredModels",
515
529
  "judge rate bar": "minJudgeRate",
530
+ "eval runs": "evalRuns",
516
531
  "merge-risk weights": "mergeRisk",
517
532
  "question limits": "questionLimits",
518
533
  "question timeout": "questionTimeoutHours",
@@ -32,7 +32,7 @@ export const alignmentQuestion = (item: Pick<Item, "title" | "body">, diff: stri
32
32
 
33
33
  /** The pull request comment for a call below the bar: its grades, or why it gave none. It blocks nothing. */
34
34
  export const alignmentComment = (alignment: { readonly graded: AlignmentAnswer } | { readonly abstained: string }, config: Pick<HumConfig, "decisionModel" | "measuredModels" | "minJudgeRate">): string => {
35
- const rate = config.measuredModels.alignment?.[config.decisionModel];
35
+ const rate = config.measuredModels.alignment?.[config.decisionModel]?.rate;
36
36
  const measured = rate === undefined ? "not measured" : `measured at ${rate.toFixed(2)}`;
37
37
  const view = "graded" in alignment ? `alignment ${alignment.graded.alignment}, verification ${alignment.graded.verification}` : `no view (${alignment.abstained})`;
38
38
 
@@ -26,9 +26,29 @@ export type Landing =
26
26
 
27
27
  const STRICTEST_PROOF = "the owner's word that the change is proven";
28
28
 
29
- /** Who has approved a change: a fresh review run, and the owner. */
29
+ /** Who has approved a change: a review run, whichever step ran it, and the owner. */
30
30
  export type Approvals = { readonly review: boolean; readonly owner: boolean };
31
31
 
32
+ /** How an item reached done: its checks alone, a review run's approval of the head that landed on top of them, the owner's approval, or a plan run that split it into new items. */
33
+ export const DONE_BY = ["checks", "review", "owner", "split"] as const;
34
+
35
+ export type DoneBy = (typeof DONE_BY)[number];
36
+
37
+ /** The review verdict that lets a change merge. */
38
+ export const REVIEW_APPROVAL = "approve";
39
+
40
+ /** Whether a review run's verdict approves the change that ends at `head`: one on another head approves nothing. */
41
+ export const doesApprove = (verdict: { readonly head: string; readonly event: string } | undefined, head: string): boolean => verdict?.head === head && verdict.event === REVIEW_APPROVAL;
42
+
43
+ /** The approvals that clear a change, the owner's first; each is the way the change it clears reaches done. */
44
+ const APPROVALS_OWNER_FIRST = ["owner", "review"] as const satisfies readonly (keyof Approvals & DoneBy)[];
45
+
46
+ /** How a cleared change reaches done: by the first approval it holds, else by its checks alone. */
47
+ const clearedBy = (approvals: Approvals): DoneBy => APPROVALS_OWNER_FIRST.find((approval) => approvals[approval]) ?? "checks";
48
+
49
+ /** How a change reaches done if it merges as landing finds it: as its approvals clear it, else by the person who merges it while it waits. */
50
+ export const reachesDoneBy = (approvals: Approvals, cleared: boolean): DoneBy => (cleared ? clearedBy(approvals) : "owner");
51
+
32
52
  /** What clears a clear change without the owner, by its level's review. The owner's approval clears any change. */
33
53
  const CLEARED_BY = {
34
54
  none: () => true,
@@ -24,9 +24,13 @@ export const fenced = (label: string, text: string): string => {
24
24
  return `<${tag}>\n${clean}\n</${tag}>`;
25
25
  };
26
26
 
27
+ /** Whether every model id that answered has a measured rate at the boundary of at least `bar`; none answering meets no bar. */
28
+ export const doesMeetBar = (models: readonly string[], boundary: Boundary, measuredModels: HumConfig["measuredModels"], bar: number): boolean =>
29
+ models.length > 0 && models.every((model) => (measuredModels[boundary]?.[model]?.rate ?? 0) >= bar);
30
+
27
31
  /** A model's pick counts only when every model id that answered has a measured rate at the boundary that meets the bar. */
28
32
  export const isTrusted = (models: readonly string[], boundary: Boundary, { measuredModels, minJudgeRate }: Pick<HumConfig, "measuredModels" | "minJudgeRate">): boolean =>
29
- models.length > 0 && models.every((model) => (measuredModels[boundary]?.[model] ?? 0) >= minJudgeRate);
33
+ doesMeetBar(models, boundary, measuredModels, minJudgeRate);
30
34
 
31
35
  export const Cite = z.strictObject({ file: z.string().min(1), line: z.int().positive() });
32
36
 
@@ -43,9 +47,12 @@ export type PlanFile = z.infer<typeof PlanFile>;
43
47
  /** A plan's identity: a later run reads the plan with this hash, and any other text is not the approved plan. */
44
48
  export const planHash = (plan: PlanFile): string => hashJson(plan);
45
49
 
50
+ /** The event by which a plan run splits its item into parts. */
51
+ export const SPLIT = "needs-split";
52
+
46
53
  /** The two states whose outcome is prose: the events each may pick, and the safer pick an unsure run takes. */
47
54
  export const PROSE_STATES = {
48
- plan: { boundary: "plan-outcome", events: ["plan-ready", "needs-split", "plan-blocked"], unsure: "plan-blocked" },
55
+ plan: { boundary: "plan-outcome", events: ["plan-ready", SPLIT, "plan-blocked"], unsure: "plan-blocked" },
49
56
  review: { boundary: "review-verdict", events: ["approve", "request-changes", "needs-owner"], unsure: "request-changes" },
50
57
  } as const satisfies Record<string, { boundary: Boundary; events: readonly [string, ...string[]]; unsure: string }>;
51
58
 
@@ -66,6 +73,9 @@ export type EventPick = z.infer<typeof EventPick>;
66
73
 
67
74
  export type JsonSchema = z.core.JSONSchema.BaseSchema;
68
75
 
76
+ /** What a model is sent to judge: the prompt, and the schema its answer must match. */
77
+ export type ModelAsk = { readonly prompt: string; readonly schema: JsonSchema };
78
+
69
79
  /** A schema as `claude --json-schema` reads it: draft-07, since its validator knows no later draft. */
70
80
  export const jsonSchemaOf = (schema: z.ZodType): JsonSchema => z.toJSONSchema(schema, { target: "draft-7" });
71
81
 
@@ -20,6 +20,7 @@ export const Change = z
20
20
 
21
21
  export type Change = z.infer<typeof Change>;
22
22
 
23
+ /** Worst first: a change that several commits held takes the first of these that any of them has. */
23
24
  export const LABELS = ["reverted", "fixed", "clean"] as const;
24
25
 
25
26
  export type Label = (typeof LABELS)[number];
@@ -1,4 +1,7 @@
1
+ import { labelChanges, type LabelRules, type PastChange } from "../decide/labels.ts";
2
+ import { type Label, LABELS } from "../decide/risk.ts";
1
3
  import type { State } from "../state/open.ts";
4
+ import { commitsOf, type Merge } from "./records.ts";
2
5
 
3
6
  /** Paths blocked by a break whose level blocks and whose follow-up has not merged. */
4
7
  export const blockedPaths = (state: State): string[] =>
@@ -6,3 +9,17 @@ export const blockedPaths = (state: State): string[] =>
6
9
 
7
10
  /** The commits whose merge broke the base: the fix limit counts them beside the breaks git labels. */
8
11
  export const brokenCommits = (state: State): ReadonlySet<string> => new Set(state.breaks.list().map(({ sha }) => sha));
12
+
13
+ /** The worst label the loop saved for any commit of a merged change; none where it saved none. */
14
+ export const brokenLabelOf = (state: State, merge: Pick<Merge, "sha" | "commits">): Exclude<Label, "clean"> | undefined => {
15
+ const found = new Set<Label>(commitsOf(merge).flatMap((sha) => state.labels.get(sha)?.label ?? []));
16
+
17
+ return LABELS.find((label): label is Exclude<Label, "clean"> => label !== "clean" && found.has(label));
18
+ };
19
+
20
+ /** Saves the label git's rule gives each change in `changes` that broke, the base's own breaks counted; the measure reads a merge's label from its commits'. */
21
+ export const saveLabels = (state: State, changes: readonly PastChange[], rules: LabelRules): void => {
22
+ for (const [sha, label] of labelChanges(changes, rules, brokenCommits(state))) {
23
+ if (label !== "clean" && state.labels.get(sha)?.label !== label) state.labels.put(sha, { sha, label });
24
+ }
25
+ };
package/src/loop/notes.ts CHANGED
@@ -2,7 +2,7 @@ import type { PastChange } from "../decide/labels.ts";
2
2
  import { breakOf, type LabelRules } from "../decide/labels.ts";
3
3
  import { optionOf, type SettledAnswer } from "../state/saved.ts";
4
4
  import type { State } from "../state/open.ts";
5
- import { type NoteSource, noteKey, type ReviewNote } from "./records.ts";
5
+ import { BREAK_NOTES, type NoteSource, noteKey, type ReviewNote } from "./records.ts";
6
6
 
7
7
  /** What an integration reads from the review surface, before the owner's own are told from the rest. */
8
8
  export type RawComment = {
@@ -52,7 +52,7 @@ export const breakNotes = (changes: readonly PastChange[], rules: LabelRules, hu
52
52
  return [
53
53
  hunkOf(found.by.sha, file).then(
54
54
  (hunk): ReviewNote => ({
55
- source: found.label === "reverted" ? "revert" : "fix",
55
+ source: BREAK_NOTES[found.label],
56
56
  sourceId: found.by.sha,
57
57
  change: change.sha,
58
58
  file,
package/src/loop/pair.ts CHANGED
@@ -2,7 +2,7 @@ import { z } from "zod";
2
2
  import type { Question } from "../adapters/model-call.ts";
3
3
  import type { ModelCall } from "../adapters/model-call.ts";
4
4
  import type { HumConfig } from "../config/schema.ts";
5
- import { DATA_RULE, fenced } from "../decide/prose.ts";
5
+ import { DATA_RULE, doesMeetBar, fenced } from "../decide/prose.ts";
6
6
  import { LOOP_SPEND_PREFIX } from "../measure/inputs.ts";
7
7
  import type { Proposer } from "./groups.ts";
8
8
  import { noteKey, type ReviewNote } from "./records.ts";
@@ -36,4 +36,4 @@ export const modelProposer =
36
36
 
37
37
  /** Whether every model id behind a proposal reached the owner's precision bar on labelled pairs. */
38
38
  export const doesTrustRepeats = (models: readonly string[], { measuredModels, repeatPrecision }: Pick<HumConfig, "measuredModels" | "repeatPrecision">): boolean =>
39
- models.length > 0 && models.every((model) => (measuredModels["repeat-pair"]?.[model] ?? 0) >= repeatPrecision);
39
+ doesMeetBar(models, "repeat-pair", measuredModels, repeatPrecision);
package/src/loop/pass.ts CHANGED
@@ -7,6 +7,7 @@ import { checkNameOf, fileChecks, filedFrom } from "./check-item.ts";
7
7
  import { type CiLogs, doesLogName } from "./ci.ts";
8
8
  import { type Proposer, groupNotes } from "./groups.ts";
9
9
  import { withoutCovered } from "./memory.ts";
10
+ import { saveLabels } from "./breaks.ts";
10
11
  import { breakNotes, ownerNotes, planChangingNotes, type RawComment, saveNotes } from "./notes.ts";
11
12
  import { type RepeatGroup, type ReviewNote, noteKey } from "./records.ts";
12
13
 
@@ -42,11 +43,11 @@ const unappliedCard = (state: State) => (extension: string, signals: readonly st
42
43
  };
43
44
 
44
45
  /** Step 1: the owner's notes from each source, saved once. Returns the ones this pass added. */
45
- const collectNotes = async ({ state, config, comments, history, hunkOf }: LoopPass): Promise<ReviewNote[]> =>
46
+ const collectNotes = async ({ state, config, comments, hunkOf }: LoopPass, changes: readonly PastChange[]): Promise<ReviewNote[]> =>
46
47
  saveNotes(state, [
47
48
  ...ownerNotes(await comments(sinceOfLastComment(state)), config.hum.ownerLogins),
48
49
  ...planChangingNotes(state.answers.list()),
49
- ...(await breakNotes(await history(), config.hum.changeLabels, hunkOf)),
50
+ ...(await breakNotes(changes, config.hum.changeLabels, hunkOf)),
50
51
  ]);
51
52
 
52
53
  /** Step 6: a landed check item whose name the next CI run's log shows. */
@@ -97,8 +98,10 @@ export const closeLoop = async ({ state, workSource, memoryPath }: Pick<LoopPass
97
98
  /** One pass of the loop: collect, find repeats, file checks, confirm CI ran them, close the loop. */
98
99
  export const runLoopPass = async (pass: LoopPass): Promise<void> => {
99
100
  const { state, config, workSource, propose, trusts } = pass;
101
+ const changes = await pass.history();
100
102
 
101
- await groupNotes({ state, propose, trusts, onUnapplied: unappliedCard(state) }, await collectNotes(pass));
103
+ saveLabels(state, changes, config.hum.changeLabels);
104
+ await groupNotes({ state, propose, trusts, onUnapplied: unappliedCard(state) }, await collectNotes(pass, changes));
102
105
  await fileChecks({ state, workSource, repeatAt: config.hum.repeatAt, checkKind: config.hum.checkKind });
103
106
  await confirmChecks(pass);
104
107
  await closeLoop(pass);
@@ -1,4 +1,5 @@
1
1
  import { z } from "zod";
2
+ import { type Label, LABELS } from "../decide/risk.ts";
2
3
 
3
4
  const name = z.string().min(1);
4
5
  const time = z.int().nonnegative();
@@ -8,6 +9,9 @@ const NOTE_SOURCES = ["review-comment", "review-body", "plan-answer", "revert",
8
9
 
9
10
  export type NoteSource = (typeof NOTE_SOURCES)[number];
10
11
 
12
+ /** The note a broken change gets, by how a later change broke it. */
13
+ export const BREAK_NOTES = { reverted: "revert", fixed: "fix" } as const satisfies Record<Exclude<Label, "clean">, NoteSource>;
14
+
11
15
  /** One correction the owner made to hum's work, saved before anything acts on it and keyed by its source and source id. */
12
16
  export const ReviewNote = z.object({
13
17
  source: z.enum(NOTE_SOURCES),
@@ -50,18 +54,37 @@ export const PossibleRepeat = z.object({ a: name, b: name, models: z.array(name)
50
54
 
51
55
  export type PossibleRepeat = z.infer<typeof PossibleRepeat>;
52
56
 
53
- /** A merged change: what the measure counts. */
57
+ /**
58
+ * A merged change: what the measure counts, and, with the picks its item's log keeps, what a judge can be scored again
59
+ * on once its prompt changes. The item's text is the work source's, and the change the judges read is the base's diff
60
+ * across its commits on the base's first-parent line, hum's done mark aside.
61
+ */
54
62
  export const Merge = z.object({
55
63
  item: name,
64
+ /** The base's commit once the change is on it. */
56
65
  sha: name,
57
66
  at: time,
58
67
  paths: z.array(name),
59
68
  /** The repo's checks have run on the base since it merged; a merge saved before hum checked the base counts as checked. */
60
69
  baseChecked: z.boolean().default(true),
70
+ /**
71
+ * Every commit that held the change, oldest first: the run branch's own, then, for a pull request, the commit its
72
+ * merge made on the base. A note, revert or fix that names one is about this item. Absent where hum did not see
73
+ * them, as on a merge saved before it kept them; `sha` alone then stands for them.
74
+ */
75
+ commits: z.array(name).optional(),
61
76
  });
62
77
 
63
78
  export type Merge = z.infer<typeof Merge>;
64
79
 
80
+ /** The commits that held a merged change; its `sha` alone where hum did not see them. */
81
+ export const commitsOf = ({ sha, commits }: Pick<Merge, "sha" | "commits">): readonly string[] => commits ?? [sha];
82
+
83
+ /** A commit on the base that git's label rule finds a later change reverted or fixed, or that a test of the base failed after. */
84
+ export const BrokenCommit = z.object({ sha: name, label: z.enum(LABELS).exclude(["clean"]) });
85
+
86
+ export type BrokenCommit = z.infer<typeof BrokenCommit>;
87
+
65
88
  /** A test of the base that failed after an item's change merged. */
66
89
  export const Break = z.object({ item: name, sha: name, at: time, paths: z.array(name), level: name, failure: z.string(), onBreak: z.enum(["follow-up", "block"]), followUp: name });
67
90
 
@@ -1,6 +1,6 @@
1
1
  import { dirname, extname } from "node:path";
2
2
  import { Lang, parse } from "@ast-grep/napi";
3
- import type { ReviewNote } from "./records.ts";
3
+ import { BREAK_NOTES, type ReviewNote } from "./records.ts";
4
4
 
5
5
  /** The languages whose hunks the parser reads. A hunk in any other language leaves the signals that need a parse unapplied. */
6
6
  const PARSED_LANGUAGES = new Map<string, Lang>([
@@ -96,7 +96,7 @@ const doShare = <T>(a: Set<T> | undefined, b: Set<T> | undefined): boolean => a
96
96
  /** Two notes on one change are about one hunk; only notes across changes repeat. */
97
97
  const isAnotherChange = (a: Prepared, b: Prepared): boolean => a.note.change !== b.note.change;
98
98
 
99
- const BREAK_SOURCES: readonly ReviewNote["source"][] = ["revert", "fix"];
99
+ const BREAK_SOURCES: readonly ReviewNote["source"][] = Object.values(BREAK_NOTES);
100
100
 
101
101
  type Signal = {
102
102
  readonly name: string;
@@ -0,0 +1,102 @@
1
+ import { z } from "zod";
2
+ import type { HumConfig } from "../config/schema.ts";
3
+ import type { PastChange } from "../decide/labels.ts";
4
+ import { type Label, LABELS } from "../decide/risk.ts";
5
+ import { brokenLabelOf } from "../loop/breaks.ts";
6
+ import { commitsOf, type Merge } from "../loop/records.ts";
7
+ import { foldItem, type Picked } from "../state/item-log.ts";
8
+ import type { State } from "../state/open.ts";
9
+
10
+ /** A case of the alignment call's set: what the change answers and its diff, labelled by what git later did to it. */
11
+ export const AlignmentCase = z.object({ case: z.string(), label: z.enum(LABELS), subject: z.string(), body: z.string().default(""), diff: z.string() });
12
+
13
+ /** A case of the review run's set: an item and the diff that answers it, labelled by the verdict it deserved. */
14
+ export const ReviewCase = z.object({ case: z.string(), label: z.string(), item: z.string(), body: z.string().default(""), diff: z.string() });
15
+
16
+ /** The judges whose sets a merged item joins: those that judge a change, each with the name its pick has in the item's log. */
17
+ const JUDGED = ["alignment", "review-verdict"] as const;
18
+
19
+ type Judged = (typeof JUDGED)[number];
20
+
21
+ const PICK_OF = { alignment: "alignment", "review-verdict": "review" } as const satisfies Record<Judged, Picked["judge"]>;
22
+
23
+ /** What a merged item gives the judges its change was put to: what they read, and the label git settled on the change. */
24
+ export const MergedCase = z.object({
25
+ item: z.string().min(1),
26
+ title: z.string(),
27
+ body: z.string(),
28
+ diff: z.string(),
29
+ label: z.enum(LABELS),
30
+ judges: z.array(z.enum(JUDGED)),
31
+ });
32
+
33
+ export type MergedCase = z.infer<typeof MergedCase>;
34
+
35
+ const casesFor =
36
+ <C>(judge: Judged, asCase: (found: MergedCase) => C) =>
37
+ (merged: readonly MergedCase[]): C[] =>
38
+ merged.flatMap((found) => (found.judges.includes(judge) ? [asCase(found)] : []));
39
+
40
+ /** The merged items' cases of each judge's set, in its shape: the alignment call's take the change's label, the review run's approve for a clean change and request-changes for one reverted or fixed. */
41
+ export const MERGED_CASES = {
42
+ alignment: casesFor("alignment", ({ item, title, body, diff, label }): z.input<typeof AlignmentCase> => ({ case: item, label, subject: title, body, diff })),
43
+ "review-verdict": casesFor("review-verdict", ({ item, title, body, diff, label }): z.input<typeof ReviewCase> => ({ case: item, label: label === "clean" ? "approve" : "request-changes", item: title, body, diff })),
44
+ } as const satisfies Record<Judged, (merged: readonly MergedCase[]) => object[]>;
45
+
46
+ /** Where a merged item's text and its change's diff come from: the work source, and git. */
47
+ export type CaseSources = {
48
+ readonly itemText: (item: string) => Promise<{ readonly title: string; readonly body: string } | undefined>;
49
+ readonly diff: (from: string, to: string) => Promise<string>;
50
+ };
51
+
52
+ /**
53
+ * A merged change's label once a fix can no longer change it: reverted or fixed as the loop saved it for any of its
54
+ * commits, clean once `fixWindow` later changes on the base touched one of its paths; undefined before then, or where
55
+ * the history no longer reaches the change.
56
+ */
57
+ const settledLabel = (state: State, merge: Merge, changes: readonly PastChange[], fixWindow: number): Label | undefined => {
58
+ const at = changes.findIndex(({ sha }) => sha === merge.sha);
59
+ const touching = at === -1 ? [] : changes.slice(at + 1).filter(({ files }) => files.some((file) => merge.paths.includes(file)));
60
+
61
+ return brokenLabelOf(state, merge) ?? (touching.length >= fixWindow ? "clean" : undefined);
62
+ };
63
+
64
+ /** The base's commit just before the change: walking back from its commit on the base, the first that is not one of its own. */
65
+ const baseBefore = (merge: Merge, changes: readonly PastChange[]): string | undefined => {
66
+ const own = new Set(commitsOf(merge));
67
+ const at = changes.findIndex(({ sha }) => sha === merge.sha);
68
+
69
+ return at === -1 ? undefined : changes.slice(0, at).findLast(({ sha }) => !own.has(sha))?.sha;
70
+ };
71
+
72
+ /**
73
+ * Keeps a case for each merged item whose change a judge was put to, once git settles its label; a case kept before
74
+ * takes a label that changed since, as a later revert gives. The case holds the item's text and the base's diff across
75
+ * the change, as much as a judge reads. An item the work source no longer gives, or a change whose base before it the
76
+ * history no longer reaches, gives no case.
77
+ */
78
+ export const keepMergedCases = async (state: State, changes: readonly PastChange[], { changeLabels, diffLimit }: Pick<HumConfig, "changeLabels" | "diffLimit">, sources: CaseSources): Promise<void> => {
79
+ for (const merge of state.merges.list()) {
80
+ const log = state.events.read(merge.item);
81
+ const label = settledLabel(state, merge, changes, changeLabels.fixWindow);
82
+
83
+ if (!log.ok || label === undefined) continue;
84
+
85
+ const { picks } = foldItem(log.value);
86
+ const judges = JUDGED.filter((judge) => picks[PICK_OF[judge]] !== undefined);
87
+
88
+ if (judges.length === 0) continue;
89
+
90
+ const kept = state.cases.get(merge.item);
91
+ const from = baseBefore(merge, changes);
92
+
93
+ if (kept !== undefined) {
94
+ state.cases.put(merge.item, { ...kept, label, judges });
95
+ continue;
96
+ }
97
+
98
+ const text = from === undefined ? undefined : await sources.itemText(merge.item);
99
+
100
+ if (from !== undefined && text !== undefined) state.cases.put(merge.item, { item: merge.item, title: text.title, body: text.body, diff: (await sources.diff(from, merge.sha)).slice(0, diffLimit), label, judges });
101
+ }
102
+ };
@@ -1,3 +1,7 @@
1
+ import type { Label } from "../decide/risk.ts";
2
+ import { brokenLabelOf } from "../loop/breaks.ts";
3
+ import { commitsOf, type Merge } from "../loop/records.ts";
4
+ import { doneByOf, foldItem } from "../state/item-log.ts";
1
5
  import type { State } from "../state/open.ts";
2
6
  import type { Counted } from "./measure.ts";
3
7
 
@@ -20,8 +24,18 @@ export const measureInputsOf = (state: State) => {
20
24
  tokens: spent.find((found) => found.item === item)?.total ?? 0,
21
25
  });
22
26
 
27
+ /** A merged change's label: the worst the loop saved for any of its commits, as git's label rule gave it; clean where it saved none. */
28
+ const labelOf = (merge: Merge): Label => brokenLabelOf(state, merge) ?? "clean";
29
+
30
+ /** How the item reached done, from its log; unknown where a newer engine wrote some of it, which raises the item's card. */
31
+ const reachedDoneBy = (item: string) => {
32
+ const log = state.events.read(item);
33
+
34
+ return log.ok ? doneByOf(foldItem(log.value)) : undefined;
35
+ };
36
+
23
37
  return {
24
- merged: merges.map(({ item, sha, at, paths }) => ({ item, at, paths, ...costOf(item, [sha]) })),
38
+ merged: merges.map((merge) => ({ item: merge.item, at: merge.at, paths: merge.paths, doneBy: reachedDoneBy(merge.item), label: labelOf(merge), ...costOf(merge.item, commitsOf(merge)) })),
25
39
  abandoned: state.parked.list().flatMap(({ item, why, since }) => (why === "stuck" && !merges.some((merge) => merge.item === item) ? [{ item, since, paths: [], ...costOf(item, []) }] : [])),
26
40
  loopTokens: spent.filter(({ item }) => item.startsWith(LOOP_SPEND_PREFIX)).reduce((total, { total: tokens }) => total + tokens, 0),
27
41
  };
@@ -0,0 +1,52 @@
1
+ import { askOf } from "../adapters/model-call.ts";
2
+ import { hashJson } from "../config/hash.ts";
3
+ import { asJson } from "../config/json.ts";
4
+ import { BOUNDARIES, type Boundary, type HumConfig, type Score } from "../config/schema.ts";
5
+ import { alignmentQuestion } from "../decide/alignment.ts";
6
+ import { replyQuestion } from "../decide/answer.ts";
7
+ import { kindQuestion } from "../decide/kind.ts";
8
+ import { type ModelAsk, PROSE_STATES } from "../decide/prose.ts";
9
+ import { tasteQuestion } from "../decide/taste.ts";
10
+ import { pairQuestion } from "../loop/pair.ts";
11
+ import type { ReviewNote } from "../loop/records.ts";
12
+ import { proseAsk } from "../worker/prose-run.ts";
13
+ import { PROSE_PROMPTS, type RunConfig } from "../worker/thin-path.ts";
14
+
15
+ const BLANK_ITEM = { id: "", title: "", body: "" };
16
+
17
+ const BLANK_NOTE: ReviewNote = { source: "review-comment", sourceId: "", change: "", hunk: "", text: "", at: 0 };
18
+
19
+ /** What each judge is sent around an empty case: its prompt, with the role text and skills of a plan or review run, and the answer form. */
20
+ const JUDGE_ASKS = {
21
+ kind: ({ hum }) => askOf(kindQuestion(BLANK_ITEM, hum)),
22
+ alignment: ({ hum }) => askOf(alignmentQuestion(BLANK_ITEM, "", hum)),
23
+ "plan-outcome": (config) => proseAsk(PROSE_PROMPTS.plan(config, BLANK_ITEM), PROSE_STATES.plan.events),
24
+ "review-verdict": (config) => proseAsk(PROSE_PROMPTS.review(config, BLANK_ITEM, ""), PROSE_STATES.review.events),
25
+ "reply-option": () => askOf(replyQuestion({ question: "", options: [], reply: "" })),
26
+ "repeat-pair": () => askOf(pairQuestion(BLANK_NOTE, BLANK_NOTE)),
27
+ taste: ({ hum }) => askOf(tasteQuestion("", "", hum)),
28
+ } as const satisfies Record<Boundary, (config: RunConfig) => ModelAsk>;
29
+
30
+ /** The hash of what a judge reads besides its case: a change to its prompt, role text, skills or answer form changes it. */
31
+ export const judgeHash = (boundary: Boundary, config: RunConfig): string => hashJson(asJson(JUDGE_ASKS[boundary](config)));
32
+
33
+ /** The boundaries measured on the labelled sets hum ships; taste is measured on cases only the owner can label. */
34
+ export const SHIPPED_SETS: readonly Boundary[] = BOUNDARIES.filter((boundary) => boundary !== "taste");
35
+
36
+ /** The model ids hum runs: each tier's and the cheap call's. */
37
+ export const runModels = ({ models, decisionModel }: Pick<HumConfig, "models" | "decisionModel">): string[] => [...new Set([...Object.values(models), decisionModel])];
38
+
39
+ /** A model's score at a boundary whose judge no longer reads what it read when the score was measured. Its rate gates until a new measurement replaces it. */
40
+ export type StaleScore = { readonly boundary: Boundary; readonly model: string; readonly score: Score };
41
+
42
+ /** Each score of a model hum runs, at a boundary measured on hum's own sets, whose judge has changed since, or whose judge was never recorded. */
43
+ export const staleScores = (config: RunConfig): StaleScore[] =>
44
+ SHIPPED_SETS.flatMap((boundary) => {
45
+ const now = judgeHash(boundary, config);
46
+
47
+ return runModels(config.hum).flatMap((model) => {
48
+ const score = config.hum.measuredModels[boundary]?.[model];
49
+
50
+ return score !== undefined && score.judgeHash !== now ? [{ boundary, model, score }] : [];
51
+ });
52
+ });
@@ -1,7 +1,11 @@
1
+ import { DONE_BY, type DoneBy } from "../decide/merge.ts";
2
+ import type { Label } from "../decide/risk.ts";
3
+
1
4
  /** What an item cost the owner: review notes, questions, and model tokens. */
2
5
  export type Counted = { readonly notes: number; readonly questions: number; readonly tokens: number };
3
6
 
4
- export type MergedChange = Counted & { readonly item: string; readonly at: number; readonly paths: readonly string[] };
7
+ /** A merged change: its cost, how its item reached done where hum kept it, and whether a later change reverted or fixed it. */
8
+ export type MergedChange = Counted & { readonly item: string; readonly at: number; readonly paths: readonly string[]; readonly doneBy: DoneBy | undefined; readonly label: Label };
5
9
 
6
10
  /** An item that never merged. Its cost counts all the same, so dropping work never lowers a number. */
7
11
  export type AbandonedItem = Counted & { readonly item: string; readonly since: number; readonly paths: readonly string[] };
@@ -18,9 +22,13 @@ export type MeasureInput = {
18
22
  readonly globs: readonly string[];
19
23
  };
20
24
 
25
+ /** The merged changes that reached done one way, over that way's own last `window`, and how many of them a later change reverted or fixed. */
26
+ type DoneByBreaks = { readonly doneBy: DoneBy; readonly merges: number; readonly reverted: number; readonly fixed: number };
27
+
21
28
  export type Measured = PerMerge & {
22
29
  readonly window: number;
23
30
  readonly byPath: ({ readonly glob: string } & PerMerge)[];
31
+ readonly byDoneBy: DoneByBreaks[];
24
32
  };
25
33
 
26
34
  const sum = (counted: readonly Counted[], key: keyof Counted): number => counted.reduce((total, found) => total + found[key], 0);
@@ -49,9 +57,20 @@ const abandonedSince = (abandoned: readonly AbandonedItem[], inWindow: readonly
49
57
 
50
58
  const doesTouch = (glob: string, paths: readonly string[]): boolean => paths.some((path) => new Bun.Glob(glob).match(path));
51
59
 
60
+ const labelled = (merged: readonly MergedChange[], label: Label): number => merged.filter((found) => found.label === label).length;
61
+
62
+ /** Each way of reaching done that any merged change took: its own last `window` merged changes, and how many of them broke. A merge saved before hum kept its way counts in none. */
63
+ const breaksByDoneBy = (merged: readonly MergedChange[], window: number): DoneByBreaks[] =>
64
+ DONE_BY.flatMap((doneBy) => {
65
+ const done = newestFirst(merged.filter((found) => found.doneBy === doneBy), window);
66
+
67
+ return done.length === 0 ? [] : [{ doneBy, merges: done.length, reverted: labelled(done, "reverted"), fixed: labelled(done, "fixed") }];
68
+ });
69
+
52
70
  /**
53
71
  * The three numbers per merged change over the last `window` merged changes, and over each path's own last
54
- * `window`. `undefined` while nothing has merged, when a per-merge number has no meaning.
72
+ * `window`, and the breaks of each way of reaching done. `undefined` while nothing has merged, when a per-merge
73
+ * number has no meaning.
55
74
  */
56
75
  export const measureOf = ({ merged, abandoned, loopTokens, window, globs }: MeasureInput): Measured | undefined => {
57
76
  const recent = newestFirst(merged, window);
@@ -64,5 +83,5 @@ export const measureOf = ({ merged, abandoned, loopTokens, window, globs }: Meas
64
83
  return onPath.length === 0 ? [] : [{ glob, ...perMerge(onPath, abandonedSince(abandoned, onPath).filter(({ paths }) => doesTouch(glob, paths)), 0) }];
65
84
  });
66
85
 
67
- return { window, ...perMerge(recent, abandonedSince(abandoned, recent), loopTokens), byPath };
86
+ return { window, ...perMerge(recent, abandonedSince(abandoned, recent), loopTokens), byPath, byDoneBy: breaksByDoneBy(merged, window) };
68
87
  };
@@ -12,13 +12,16 @@ type MeasuredModels = HumConfig["measuredModels"];
12
12
  const UNMEASURED = Object.fromEntries(BOUNDARIES.map((boundary) => [boundary, {}])) as MeasuredModels;
13
13
 
14
14
  /**
15
- * The owner's measured rates with one pooled rate set at `boundary`, under the model as the owner names it and under
16
- * each model id that answered while it was measured, and every boundary still unmeasured kept with no rate. Trust
17
- * compares the ids a call's answer names, which an alias such as `haiku` never is, and the pre-call checks read the
18
- * name in the owner's settings.
15
+ * The owner's measured rates with one pooled rate set at `boundary`, beside the hash of what its judge read, under the
16
+ * model as the owner names it and under each model id that answered while it was measured, and every boundary still
17
+ * unmeasured kept with no rate. Trust compares the ids a call's answer names, which an alias such as `haiku` never
18
+ * is, and the pre-call checks read the name in the owner's settings.
19
19
  */
20
- export const withRate = (measured: Partial<MeasuredModels>, { boundary, model, answered, rate }: { readonly boundary: Boundary; readonly model: string; readonly answered: readonly string[]; readonly rate: number }): MeasuredModels => ({
20
+ export const withRate = (
21
+ measured: Partial<MeasuredModels>,
22
+ { boundary, model, answered, rate, judgeHash }: { readonly boundary: Boundary; readonly model: string; readonly answered: readonly string[]; readonly rate: number; readonly judgeHash: string },
23
+ ): MeasuredModels => ({
21
24
  ...UNMEASURED,
22
25
  ...measured,
23
- [boundary]: { ...measured[boundary], ...Object.fromEntries([model, ...answered].map((id) => [id, Number(rate.toFixed(RATE_DIGITS))])) },
26
+ [boundary]: { ...measured[boundary], ...Object.fromEntries([model, ...answered].map((id) => [id, { rate: Number(rate.toFixed(RATE_DIGITS)), judgeHash }])) },
24
27
  });
@@ -0,0 +1,31 @@
1
+ import { join } from "node:path";
2
+ import { x } from "tinyexec";
3
+ import type { Boundary } from "../config/schema.ts";
4
+
5
+ const EVAL_SCRIPT = join(import.meta.dir, "..", "..", "scripts", "eval.ts");
6
+
7
+ /** The folder of each boundary's labelled set and results: in the package, and in an owner's `.hum/` for the results of their own models. */
8
+ export const EVALS_DIR = "evals";
9
+
10
+ const OUTPUT_TAIL = 300;
11
+
12
+ /** A model to measure on the owner's judges, at the boundaries named, or at every one hum ships a set for when none are. */
13
+ export type Measurement = { readonly model: string; readonly runs: number; readonly boundaries?: readonly Boundary[] };
14
+
15
+ /** Whether a measurement recorded a rate at every boundary it was asked for, and the end of what it printed when it did not. */
16
+ export type Measured = { readonly ok: true } | { readonly ok: false; readonly why: string };
17
+
18
+ /**
19
+ * Measures a model on the judges of the config in `repo`'s `.hum/`, through `scripts/eval.ts`, which records each
20
+ * pooled rate with its judge's hash in `.hum/hum.json` once it is done. `inherit` shows its progress on this
21
+ * process's terminal; otherwise its output is kept to say why it failed.
22
+ */
23
+ export const measureOn = async (repo: string, { model, runs, boundaries = [] }: Measurement, { inherit, signal }: { readonly inherit: boolean; readonly signal?: AbortSignal }): Promise<Measured> => {
24
+ const ran = await x(process.execPath, [EVAL_SCRIPT, "--model", model, "--runs", String(runs), "--write", join(repo, ".hum"), ...boundaries], {
25
+ throwOnError: false,
26
+ signal,
27
+ nodeOptions: { cwd: repo, stdio: inherit ? "inherit" : "pipe" },
28
+ });
29
+
30
+ return ran.exitCode === 0 ? { ok: true } : { ok: false, why: `${ran.stderr.trim() || ran.stdout.trim() || `it exited ${ran.exitCode ?? "on a signal"}`}`.slice(-OUTPUT_TAIL) };
31
+ };