@moo.md/hum 0.4.1 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/defaults/hum.json +39 -12
  2. package/npm-shrinkwrap.json +2 -2
  3. package/package.json +2 -1
  4. package/release.json +1 -1
  5. package/schema/hum.schema.json +63 -17
  6. package/scripts/eval.ts +58 -27
  7. package/src/adapters/claude-p.ts +7 -0
  8. package/src/adapters/model-call.ts +8 -4
  9. package/src/adapters/pull-requests.ts +11 -7
  10. package/src/cli/setup.ts +13 -18
  11. package/src/config/json.ts +3 -0
  12. package/src/config/migrate.ts +11 -0
  13. package/src/config/schema.ts +21 -6
  14. package/src/decide/alignment.ts +1 -1
  15. package/src/decide/merge.ts +21 -1
  16. package/src/decide/prose.ts +12 -2
  17. package/src/decide/risk.ts +1 -0
  18. package/src/loop/breaks.ts +17 -0
  19. package/src/loop/notes.ts +2 -2
  20. package/src/loop/pair.ts +2 -2
  21. package/src/loop/pass.ts +6 -3
  22. package/src/loop/records.ts +24 -1
  23. package/src/loop/signals.ts +2 -2
  24. package/src/measure/cases.ts +102 -0
  25. package/src/measure/inputs.ts +15 -1
  26. package/src/measure/judges.ts +52 -0
  27. package/src/measure/measure.ts +22 -3
  28. package/src/measure/rate.ts +9 -6
  29. package/src/measure/remeasure.ts +31 -0
  30. package/src/publish/handoff.ts +1 -1
  31. package/src/service/workflows.ts +4 -2
  32. package/src/setup/write.ts +2 -4
  33. package/src/state/events.ts +201 -19
  34. package/src/state/format.ts +9 -2
  35. package/src/state/item-log.ts +133 -0
  36. package/src/state/items.ts +6 -5
  37. package/src/state/open.ts +7 -2
  38. package/src/state/runs.ts +5 -0
  39. package/src/state/saved.ts +7 -2
  40. package/src/status/format.ts +12 -3
  41. package/src/worker/land.ts +36 -15
  42. package/src/worker/merged.ts +18 -8
  43. package/src/worker/open.ts +5 -1
  44. package/src/worker/prose-run.ts +6 -2
  45. package/src/worker/signature.ts +11 -5
  46. package/src/worker/thin-path.ts +69 -53
  47. package/src/worker/worker.ts +82 -12
  48. package/surface.json +969 -151
@@ -28,6 +28,9 @@ import { runLoopPass, type LoopPass } from "../loop/pass.ts";
28
28
  import { modelProposer, doesTrustRepeats } from "../loop/pair.ts";
29
29
  import { measureOf } from "../measure/measure.ts";
30
30
  import { measureInputsOf } from "../measure/inputs.ts";
31
+ import { keepMergedCases } from "../measure/cases.ts";
32
+ import { type StaleScore, staleScores } from "../measure/judges.ts";
33
+ import type { Measured, Measurement } from "../measure/remeasure.ts";
31
34
  import { emptyStatus, readStatus, writeHeartbeat, writeStatus } from "../status/file.ts";
32
35
  import type { Status } from "../status/format.ts";
33
36
  import { newestChoices, openQuestions } from "../status/sections.ts";
@@ -93,6 +96,8 @@ export type WorkerOptions = {
93
96
  readonly loop?: Partial<Pick<LoopPass, "comments" | "ciLogs">>;
94
97
  /** The CI job this worker is, where a wake splits into jobs; unset, a tick does all a wake does. */
95
98
  readonly ciJob?: CiJob;
99
+ /** Measures a model again on the owner's judges, writing its new scores in `.hum/hum.json`; absent where that file is thrown away, as in CI. */
100
+ readonly remeasure?: (measurement: Measurement, signal: AbortSignal) => Promise<Measured>;
96
101
  };
97
102
 
98
103
  /**
@@ -125,6 +130,15 @@ const ERROR_EXCERPT = 300;
125
130
 
126
131
  const UNREAD_CODE = "item-unread";
127
132
 
133
+ /** The card for scores that gate on a judge that changed since they were measured. */
134
+ const JUDGE_CODE = "judge-changed";
135
+
136
+ const staleText = (stale: readonly StaleScore[]): string => stale.map(({ boundary, model, score }) => `${boundary} on ${model} at ${score.rate.toFixed(2)}`).join(", ");
137
+
138
+ /** The stale scores, one measurement per model, of the boundaries whose judge changed for it. */
139
+ const byModel = (stale: readonly StaleScore[]): { readonly model: string; readonly boundaries: StaleScore["boundary"][] }[] =>
140
+ [...new Set(stale.map(({ model }) => model))].map((model) => ({ model, boundaries: stale.flatMap((found) => (found.model === model ? [found.boundary] : [])) }));
141
+
128
142
  const linesText = (lines: readonly number[]): string => `${lines.length === 1 ? "line" : "lines"} ${lines.join(", ")}`;
129
143
 
130
144
  /** The card for each reason memory.md lines are refused. */
@@ -196,6 +210,11 @@ export class Worker {
196
210
  #watchToken: string | undefined;
197
211
  /** Set while the worker has seen an update; no change lands until a worker on the new config takes it. */
198
212
  #updating: string | undefined;
213
+ /** The scores that gate on a judge that changed since they were measured: the config is the worker's for its life. */
214
+ readonly #stale: readonly StaleScore[];
215
+ /** Stops the measurement of changed judges while one runs. */
216
+ #remeasuring: AbortController | undefined;
217
+ #remeasureAt = Number.NEGATIVE_INFINITY;
199
218
 
200
219
  get config(): Config {
201
220
  return this.#o.config;
@@ -210,6 +229,7 @@ export class Worker {
210
229
  this.#latest = readEvery(options.latestVersion, { everyMs: options.config.hum.latestVersionCacheSeconds * SECOND_MS, timeoutMs });
211
230
  this.#ciTick = readEvery(options.ciTick, { everyMs: options.config.hum.ciTickCacheSeconds * SECOND_MS, timeoutMs });
212
231
  this.#start = signatureOf({ repo: options.repo, fallbackVersion: options.engineVersion ?? ENGINE_VERSION, itemsDir: options.config.hum.itemsDir });
232
+ this.#stale = staleScores(options.config);
213
233
  this.#judge = options.judge ?? configuredModelCall(options.runner, options.config.hum, (item, tokens) => tallyOf(options.state, item).spend(tokens));
214
234
  rememberKinds(options.state, options.config);
215
235
  this.#recheckLandings();
@@ -416,16 +436,7 @@ export class Worker {
416
436
 
417
437
  const hasNewAnswer = waiting.has(id) && answerTo(item) !== undefined && lastAnswerAt !== undefined;
418
438
 
419
- if (hasNewAnswer) {
420
- state.events.add({
421
- source: "thread",
422
- sourceId: `answer:${lastAnswerAt}`,
423
- itemId: id,
424
- type: "answered",
425
- payload: {},
426
- });
427
- this.#answered.add(id);
428
- }
439
+ if (hasNewAnswer) this.#answered.add(id);
429
440
  }),
430
441
  ),
431
442
  );
@@ -586,6 +597,8 @@ export class Worker {
586
597
 
587
598
  if (pulls) await settleMergedPulls(state, { pulls, workSource: adapters.workSource, git, remote }, this.#now());
588
599
 
600
+ const changes = await readHistory(git, tip, config.hum);
601
+
589
602
  await runLoopPass({
590
603
  state,
591
604
  config,
@@ -593,12 +606,66 @@ export class Worker {
593
606
  now: this.#now,
594
607
  comments: loop?.comments ?? (async () => []),
595
608
  ciLogs: loop?.ciLogs ?? (async () => []),
596
- history: () => readHistory(git, tip, config.hum),
609
+ history: async () => changes,
597
610
  hunkOf: async (sha, file) => (await git.run(["show", ...PATCH, "-U3", "--format=", sha, ...(file === undefined ? [] : ["--", file])])).stdout,
598
611
  propose: modelProposer(call),
599
612
  trusts: (models) => doesTrustRepeats(models, config.hum),
600
613
  memoryPath: join(repo, ".hum", "memory.md"),
601
614
  });
615
+
616
+ await keepMergedCases(state, changes, config.hum, {
617
+ itemText: (item) => adapters.workSource.read(ItemId.parse(item)).then(({ title, body }) => ({ title, body }), () => undefined),
618
+ diff: (from, to) => git.raw(["diff", ...PATCH, from, to]),
619
+ });
620
+ }
621
+
622
+ /** Raises the judge card with this cause, in place of one that names another. */
623
+ #showJudgeCard(cause: string, fix: string): void {
624
+ const { faults } = this.#o.state;
625
+
626
+ if (faults.list().some((card) => card.code === JUDGE_CODE && card.cause !== cause)) faults.clear({ code: JUDGE_CODE });
627
+
628
+ faults.raise({ code: JUDGE_CODE, cause, fix });
629
+ }
630
+
631
+ /**
632
+ * A score whose judge changed since it was measured keeps gating. Where hum can keep a measurement, it measures
633
+ * those judges again, once a loop interval and outside a usage limit; the new scores in `.hum/hum.json` restart the
634
+ * worker, which gates on them, so a judge whose new rate misses the bar abstains from then on. Where it cannot, or
635
+ * the measurement fails, a card names the old scores.
636
+ */
637
+ #judgesIfDue(t: number): void {
638
+ const { state, config, runner, remeasure } = this.#o;
639
+ const stale = this.#stale;
640
+
641
+ if (stale.length === 0) return state.faults.clear({ code: JUDGE_CODE });
642
+
643
+ const gating = `${staleText(stale)} gate on rates measured before the judge's prompt or role text last changed.`;
644
+
645
+ if (remeasure === undefined) return this.#showJudgeCard(gating, "Run `bunx hum setup measure` where Claude Code is signed in and commit .hum/hum.json; CI cannot keep a rate it measures.");
646
+
647
+ if (this.#remeasuring !== undefined || t - this.#remeasureAt < config.hum.loopIntervalSeconds * SECOND_MS || runner.limitedUntil() > t) return;
648
+
649
+ const abort = new AbortController();
650
+ const failed = (why: string) => this.#showJudgeCard(`${gating} Measuring them again failed: ${why}`, "hum tries again after loopIntervalSeconds; `bunx hum setup measure` measures them now.");
651
+
652
+ this.#remeasureAt = t;
653
+ this.#remeasuring = abort;
654
+ void this.#pool.add(async () => {
655
+ try {
656
+ for (const measurement of byModel(stale)) {
657
+ if (abort.signal.aborted) return;
658
+
659
+ const measured = await remeasure({ ...measurement, runs: config.hum.evalRuns }, abort.signal);
660
+
661
+ if (!measured.ok && !abort.signal.aborted) failed(`${measurement.model}: ${measured.why}`);
662
+ }
663
+ } catch (error) {
664
+ failed(error instanceof Error ? error.message : String(error));
665
+ } finally {
666
+ this.#remeasuring = undefined;
667
+ }
668
+ });
602
669
  }
603
670
 
604
671
  #failed(id: string, error: Error): void {
@@ -617,6 +684,8 @@ export class Worker {
617
684
  async #stopRuns(): Promise<void> {
618
685
  for (const run of this.#inFlight.values()) run.abort.abort();
619
686
 
687
+ this.#remeasuring?.abort();
688
+
620
689
  await this.idle();
621
690
  }
622
691
 
@@ -769,7 +838,7 @@ export class Worker {
769
838
  if (update !== undefined) {
770
839
  await this.#publish(t, signature, update);
771
840
 
772
- return this.#inFlight.size === 0 ? { kind: "exit", reason: update } : { kind: "draining", reason: update };
841
+ return this.#inFlight.size === 0 && this.#remeasuring === undefined ? { kind: "exit", reason: update } : { kind: "draining", reason: update };
773
842
  }
774
843
 
775
844
  const sources = await this.#readSources();
@@ -780,6 +849,7 @@ export class Worker {
780
849
  });
781
850
 
782
851
  this.#loopIfDue(t);
852
+ this.#judgesIfDue(t);
783
853
  }
784
854
 
785
855
  const ready = await this.#readReady(sources, sleeping);