@moo.md/hum 0.4.1 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/defaults/hum.json +39 -12
- package/npm-shrinkwrap.json +2 -2
- package/package.json +2 -1
- package/release.json +1 -1
- package/schema/hum.schema.json +63 -17
- package/scripts/eval.ts +58 -27
- package/src/adapters/claude-p.ts +7 -0
- package/src/adapters/model-call.ts +8 -4
- package/src/adapters/pull-requests.ts +11 -7
- package/src/cli/setup.ts +13 -18
- package/src/config/json.ts +3 -0
- package/src/config/migrate.ts +11 -0
- package/src/config/schema.ts +21 -6
- package/src/decide/alignment.ts +1 -1
- package/src/decide/merge.ts +21 -1
- package/src/decide/prose.ts +12 -2
- package/src/decide/risk.ts +1 -0
- package/src/loop/breaks.ts +17 -0
- package/src/loop/notes.ts +2 -2
- package/src/loop/pair.ts +2 -2
- package/src/loop/pass.ts +6 -3
- package/src/loop/records.ts +24 -1
- package/src/loop/signals.ts +2 -2
- package/src/measure/cases.ts +102 -0
- package/src/measure/inputs.ts +15 -1
- package/src/measure/judges.ts +52 -0
- package/src/measure/measure.ts +22 -3
- package/src/measure/rate.ts +9 -6
- package/src/measure/remeasure.ts +31 -0
- package/src/publish/handoff.ts +1 -1
- package/src/service/workflows.ts +4 -2
- package/src/setup/write.ts +2 -4
- package/src/state/events.ts +201 -19
- package/src/state/format.ts +9 -2
- package/src/state/item-log.ts +133 -0
- package/src/state/items.ts +6 -5
- package/src/state/open.ts +7 -2
- package/src/state/runs.ts +5 -0
- package/src/state/saved.ts +7 -2
- package/src/status/format.ts +12 -3
- package/src/worker/land.ts +36 -15
- package/src/worker/merged.ts +18 -8
- package/src/worker/open.ts +5 -1
- package/src/worker/prose-run.ts +6 -2
- package/src/worker/signature.ts +11 -5
- package/src/worker/thin-path.ts +69 -53
- package/src/worker/worker.ts +82 -12
- package/surface.json +969 -151
package/src/worker/worker.ts
CHANGED
|
@@ -28,6 +28,9 @@ import { runLoopPass, type LoopPass } from "../loop/pass.ts";
|
|
|
28
28
|
import { modelProposer, doesTrustRepeats } from "../loop/pair.ts";
|
|
29
29
|
import { measureOf } from "../measure/measure.ts";
|
|
30
30
|
import { measureInputsOf } from "../measure/inputs.ts";
|
|
31
|
+
import { keepMergedCases } from "../measure/cases.ts";
|
|
32
|
+
import { type StaleScore, staleScores } from "../measure/judges.ts";
|
|
33
|
+
import type { Measured, Measurement } from "../measure/remeasure.ts";
|
|
31
34
|
import { emptyStatus, readStatus, writeHeartbeat, writeStatus } from "../status/file.ts";
|
|
32
35
|
import type { Status } from "../status/format.ts";
|
|
33
36
|
import { newestChoices, openQuestions } from "../status/sections.ts";
|
|
@@ -93,6 +96,8 @@ export type WorkerOptions = {
|
|
|
93
96
|
readonly loop?: Partial<Pick<LoopPass, "comments" | "ciLogs">>;
|
|
94
97
|
/** The CI job this worker is, where a wake splits into jobs; unset, a tick does all a wake does. */
|
|
95
98
|
readonly ciJob?: CiJob;
|
|
99
|
+
/** Measures a model again on the owner's judges, writing its new scores in `.hum/hum.json`; absent where that file is thrown away, as in CI. */
|
|
100
|
+
readonly remeasure?: (measurement: Measurement, signal: AbortSignal) => Promise<Measured>;
|
|
96
101
|
};
|
|
97
102
|
|
|
98
103
|
/**
|
|
@@ -125,6 +130,15 @@ const ERROR_EXCERPT = 300;
|
|
|
125
130
|
|
|
126
131
|
const UNREAD_CODE = "item-unread";
|
|
127
132
|
|
|
133
|
+
/** The card for scores that gate on a judge that changed since they were measured. */
|
|
134
|
+
const JUDGE_CODE = "judge-changed";
|
|
135
|
+
|
|
136
|
+
const staleText = (stale: readonly StaleScore[]): string => stale.map(({ boundary, model, score }) => `${boundary} on ${model} at ${score.rate.toFixed(2)}`).join(", ");
|
|
137
|
+
|
|
138
|
+
/** The stale scores, one measurement per model, of the boundaries whose judge changed for it. */
|
|
139
|
+
const byModel = (stale: readonly StaleScore[]): { readonly model: string; readonly boundaries: StaleScore["boundary"][] }[] =>
|
|
140
|
+
[...new Set(stale.map(({ model }) => model))].map((model) => ({ model, boundaries: stale.flatMap((found) => (found.model === model ? [found.boundary] : [])) }));
|
|
141
|
+
|
|
128
142
|
const linesText = (lines: readonly number[]): string => `${lines.length === 1 ? "line" : "lines"} ${lines.join(", ")}`;
|
|
129
143
|
|
|
130
144
|
/** The card for each reason memory.md lines are refused. */
|
|
@@ -196,6 +210,11 @@ export class Worker {
|
|
|
196
210
|
#watchToken: string | undefined;
|
|
197
211
|
/** Set while the worker has seen an update; no change lands until a worker on the new config takes it. */
|
|
198
212
|
#updating: string | undefined;
|
|
213
|
+
/** The scores that gate on a judge that changed since they were measured: the config is the worker's for its life. */
|
|
214
|
+
readonly #stale: readonly StaleScore[];
|
|
215
|
+
/** Stops the measurement of changed judges while one runs. */
|
|
216
|
+
#remeasuring: AbortController | undefined;
|
|
217
|
+
#remeasureAt = Number.NEGATIVE_INFINITY;
|
|
199
218
|
|
|
200
219
|
get config(): Config {
|
|
201
220
|
return this.#o.config;
|
|
@@ -210,6 +229,7 @@ export class Worker {
|
|
|
210
229
|
this.#latest = readEvery(options.latestVersion, { everyMs: options.config.hum.latestVersionCacheSeconds * SECOND_MS, timeoutMs });
|
|
211
230
|
this.#ciTick = readEvery(options.ciTick, { everyMs: options.config.hum.ciTickCacheSeconds * SECOND_MS, timeoutMs });
|
|
212
231
|
this.#start = signatureOf({ repo: options.repo, fallbackVersion: options.engineVersion ?? ENGINE_VERSION, itemsDir: options.config.hum.itemsDir });
|
|
232
|
+
this.#stale = staleScores(options.config);
|
|
213
233
|
this.#judge = options.judge ?? configuredModelCall(options.runner, options.config.hum, (item, tokens) => tallyOf(options.state, item).spend(tokens));
|
|
214
234
|
rememberKinds(options.state, options.config);
|
|
215
235
|
this.#recheckLandings();
|
|
@@ -416,16 +436,7 @@ export class Worker {
|
|
|
416
436
|
|
|
417
437
|
const hasNewAnswer = waiting.has(id) && answerTo(item) !== undefined && lastAnswerAt !== undefined;
|
|
418
438
|
|
|
419
|
-
if (hasNewAnswer)
|
|
420
|
-
state.events.add({
|
|
421
|
-
source: "thread",
|
|
422
|
-
sourceId: `answer:${lastAnswerAt}`,
|
|
423
|
-
itemId: id,
|
|
424
|
-
type: "answered",
|
|
425
|
-
payload: {},
|
|
426
|
-
});
|
|
427
|
-
this.#answered.add(id);
|
|
428
|
-
}
|
|
439
|
+
if (hasNewAnswer) this.#answered.add(id);
|
|
429
440
|
}),
|
|
430
441
|
),
|
|
431
442
|
);
|
|
@@ -586,6 +597,8 @@ export class Worker {
|
|
|
586
597
|
|
|
587
598
|
if (pulls) await settleMergedPulls(state, { pulls, workSource: adapters.workSource, git, remote }, this.#now());
|
|
588
599
|
|
|
600
|
+
const changes = await readHistory(git, tip, config.hum);
|
|
601
|
+
|
|
589
602
|
await runLoopPass({
|
|
590
603
|
state,
|
|
591
604
|
config,
|
|
@@ -593,12 +606,66 @@ export class Worker {
|
|
|
593
606
|
now: this.#now,
|
|
594
607
|
comments: loop?.comments ?? (async () => []),
|
|
595
608
|
ciLogs: loop?.ciLogs ?? (async () => []),
|
|
596
|
-
history: () =>
|
|
609
|
+
history: async () => changes,
|
|
597
610
|
hunkOf: async (sha, file) => (await git.run(["show", ...PATCH, "-U3", "--format=", sha, ...(file === undefined ? [] : ["--", file])])).stdout,
|
|
598
611
|
propose: modelProposer(call),
|
|
599
612
|
trusts: (models) => doesTrustRepeats(models, config.hum),
|
|
600
613
|
memoryPath: join(repo, ".hum", "memory.md"),
|
|
601
614
|
});
|
|
615
|
+
|
|
616
|
+
await keepMergedCases(state, changes, config.hum, {
|
|
617
|
+
itemText: (item) => adapters.workSource.read(ItemId.parse(item)).then(({ title, body }) => ({ title, body }), () => undefined),
|
|
618
|
+
diff: (from, to) => git.raw(["diff", ...PATCH, from, to]),
|
|
619
|
+
});
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
/** Raises the judge card with this cause, in place of one that names another. */
|
|
623
|
+
#showJudgeCard(cause: string, fix: string): void {
|
|
624
|
+
const { faults } = this.#o.state;
|
|
625
|
+
|
|
626
|
+
if (faults.list().some((card) => card.code === JUDGE_CODE && card.cause !== cause)) faults.clear({ code: JUDGE_CODE });
|
|
627
|
+
|
|
628
|
+
faults.raise({ code: JUDGE_CODE, cause, fix });
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
/**
|
|
632
|
+
* A score whose judge changed since it was measured keeps gating. Where hum can keep a measurement, it measures
|
|
633
|
+
* those judges again, once a loop interval and outside a usage limit; the new scores in `.hum/hum.json` restart the
|
|
634
|
+
* worker, which gates on them, so a judge whose new rate misses the bar abstains from then on. Where it cannot, or
|
|
635
|
+
* the measurement fails, a card names the old scores.
|
|
636
|
+
*/
|
|
637
|
+
#judgesIfDue(t: number): void {
|
|
638
|
+
const { state, config, runner, remeasure } = this.#o;
|
|
639
|
+
const stale = this.#stale;
|
|
640
|
+
|
|
641
|
+
if (stale.length === 0) return state.faults.clear({ code: JUDGE_CODE });
|
|
642
|
+
|
|
643
|
+
const gating = `${staleText(stale)} gate on rates measured before the judge's prompt or role text last changed.`;
|
|
644
|
+
|
|
645
|
+
if (remeasure === undefined) return this.#showJudgeCard(gating, "Run `bunx hum setup measure` where Claude Code is signed in and commit .hum/hum.json; CI cannot keep a rate it measures.");
|
|
646
|
+
|
|
647
|
+
if (this.#remeasuring !== undefined || t - this.#remeasureAt < config.hum.loopIntervalSeconds * SECOND_MS || runner.limitedUntil() > t) return;
|
|
648
|
+
|
|
649
|
+
const abort = new AbortController();
|
|
650
|
+
const failed = (why: string) => this.#showJudgeCard(`${gating} Measuring them again failed: ${why}`, "hum tries again after loopIntervalSeconds; `bunx hum setup measure` measures them now.");
|
|
651
|
+
|
|
652
|
+
this.#remeasureAt = t;
|
|
653
|
+
this.#remeasuring = abort;
|
|
654
|
+
void this.#pool.add(async () => {
|
|
655
|
+
try {
|
|
656
|
+
for (const measurement of byModel(stale)) {
|
|
657
|
+
if (abort.signal.aborted) return;
|
|
658
|
+
|
|
659
|
+
const measured = await remeasure({ ...measurement, runs: config.hum.evalRuns }, abort.signal);
|
|
660
|
+
|
|
661
|
+
if (!measured.ok && !abort.signal.aborted) failed(`${measurement.model}: ${measured.why}`);
|
|
662
|
+
}
|
|
663
|
+
} catch (error) {
|
|
664
|
+
failed(error instanceof Error ? error.message : String(error));
|
|
665
|
+
} finally {
|
|
666
|
+
this.#remeasuring = undefined;
|
|
667
|
+
}
|
|
668
|
+
});
|
|
602
669
|
}
|
|
603
670
|
|
|
604
671
|
#failed(id: string, error: Error): void {
|
|
@@ -617,6 +684,8 @@ export class Worker {
|
|
|
617
684
|
async #stopRuns(): Promise<void> {
|
|
618
685
|
for (const run of this.#inFlight.values()) run.abort.abort();
|
|
619
686
|
|
|
687
|
+
this.#remeasuring?.abort();
|
|
688
|
+
|
|
620
689
|
await this.idle();
|
|
621
690
|
}
|
|
622
691
|
|
|
@@ -769,7 +838,7 @@ export class Worker {
|
|
|
769
838
|
if (update !== undefined) {
|
|
770
839
|
await this.#publish(t, signature, update);
|
|
771
840
|
|
|
772
|
-
return this.#inFlight.size === 0 ? { kind: "exit", reason: update } : { kind: "draining", reason: update };
|
|
841
|
+
return this.#inFlight.size === 0 && this.#remeasuring === undefined ? { kind: "exit", reason: update } : { kind: "draining", reason: update };
|
|
773
842
|
}
|
|
774
843
|
|
|
775
844
|
const sources = await this.#readSources();
|
|
@@ -780,6 +849,7 @@ export class Worker {
|
|
|
780
849
|
});
|
|
781
850
|
|
|
782
851
|
this.#loopIfDue(t);
|
|
852
|
+
this.#judgesIfDue(t);
|
|
783
853
|
}
|
|
784
854
|
|
|
785
855
|
const ready = await this.#readReady(sources, sleeping);
|