@lchase/kiln 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/assets/serve.css +195 -472
- package/assets/serve.js +121 -6
- package/dist/attention.js +13 -4
- package/dist/board-view.js +688 -0
- package/dist/board.js +0 -97
- package/dist/cli.js +63 -6
- package/dist/config.js +25 -1
- package/dist/driver.js +117 -16
- package/dist/events.js +40 -1
- package/dist/failure.js +140 -0
- package/dist/fix-commit.js +73 -0
- package/dist/ledger.js +11 -3
- package/dist/project.js +20 -4
- package/dist/rails.js +164 -0
- package/dist/reconcile.js +29 -3
- package/dist/reliability.js +53 -25
- package/dist/report.js +19 -5
- package/dist/schemas.js +35 -5
- package/dist/serve-page.js +396 -316
- package/dist/serve.js +252 -13
- package/dist/settings.js +9 -1
- package/package.json +4 -3
- package/workflows/fix.yaml +1 -1
package/dist/board.js
CHANGED
|
@@ -1,102 +1,5 @@
|
|
|
1
1
|
import { existsSync, readFileSync, readdirSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
-
import { scheduledRetry } from "./attention.js";
|
|
4
|
-
const FEATURE_STAGES = ["spec", "build", "review", "approve", "land", "pr-review"];
|
|
5
|
-
const CATALOG = [
|
|
6
|
-
{ id: "queued", label: "Queued", who: "waiting · deps", tone: "queued" },
|
|
7
|
-
{ id: "spec", label: "Spec", who: "spec-writer · opus" },
|
|
8
|
-
{ id: "build", label: "Build", who: "builder · opus" },
|
|
9
|
-
{ id: "review", label: "Review", who: "sec · ux · ui · code" },
|
|
10
|
-
{ id: "approve", label: "Approve", who: "approver · opus" },
|
|
11
|
-
{ id: "explore", label: "Explore", who: "investigator · opus" },
|
|
12
|
-
{ id: "land", label: "Land", who: "git commit" },
|
|
13
|
-
{ id: "pr-review", label: "PR review", who: "pr-sync" },
|
|
14
|
-
{ id: "done", label: "Done", who: "local", tone: "merged" },
|
|
15
|
-
{ id: "merged", label: "Merged", who: "pr", tone: "merged" },
|
|
16
|
-
{ id: "needs-human", label: "Needs human", who: "approve · retry", tone: "human" },
|
|
17
|
-
];
|
|
18
|
-
const STEP_STAGE = {
|
|
19
|
-
spec: "spec",
|
|
20
|
-
build: "build",
|
|
21
|
-
verify: "build",
|
|
22
|
-
security: "review",
|
|
23
|
-
ux: "review",
|
|
24
|
-
ui: "review",
|
|
25
|
-
code: "review",
|
|
26
|
-
approver: "approve",
|
|
27
|
-
approve: "needs-human",
|
|
28
|
-
land: "land",
|
|
29
|
-
explore: "explore",
|
|
30
|
-
};
|
|
31
|
-
const HUMAN_STATES = new Set(["needs-human", "failed", "dead", "invalid", "rejected"]);
|
|
32
|
-
/**
|
|
33
|
-
* Column for one job. Human gates and broken runs sit in Needs human.
|
|
34
|
-
* A failed job with an automatic retry still scheduled stays on the step
|
|
35
|
-
* its last run reached. `scheduledRetry` is that decision.
|
|
36
|
-
*/
|
|
37
|
-
export function stageOf(job) {
|
|
38
|
-
if (scheduledRetry(job))
|
|
39
|
-
return runStage(job);
|
|
40
|
-
if (HUMAN_STATES.has(job.state))
|
|
41
|
-
return "needs-human";
|
|
42
|
-
if (job.state === "in-review")
|
|
43
|
-
return "pr-review";
|
|
44
|
-
if (job.state === "done")
|
|
45
|
-
return job.prUrl || job.prNumber ? "merged" : "done";
|
|
46
|
-
const current = job.steps.find((step) => step.phase === "current" || step.phase === "failed");
|
|
47
|
-
if (job.state === "queued" || job.state === "blocked")
|
|
48
|
-
return "queued";
|
|
49
|
-
if (job.state === "approved") {
|
|
50
|
-
if (current && current.id !== "land" && current.id !== "approve" && STEP_STAGE[current.id])
|
|
51
|
-
return STEP_STAGE[current.id];
|
|
52
|
-
return "land";
|
|
53
|
-
}
|
|
54
|
-
if (current && STEP_STAGE[current.id])
|
|
55
|
-
return STEP_STAGE[current.id];
|
|
56
|
-
return runStage(job);
|
|
57
|
-
}
|
|
58
|
-
/**
|
|
59
|
-
* Stage of the last step the run actually reached.
|
|
60
|
-
* The approve gate's column is Needs human; a scheduled retry is not a gate,
|
|
61
|
-
* so that step stays on Approve. Spec or Explore when the run never started a step.
|
|
62
|
-
*/
|
|
63
|
-
function runStage(job) {
|
|
64
|
-
const last = [...job.steps].reverse().find((step) => step.phase === "current" || step.phase === "failed");
|
|
65
|
-
const stage = last ? STEP_STAGE[last.id] : undefined;
|
|
66
|
-
if (stage && stage !== "needs-human")
|
|
67
|
-
return stage;
|
|
68
|
-
if (stage === "needs-human") {
|
|
69
|
-
const earlier = [...job.steps].reverse().find((step) => {
|
|
70
|
-
const mapped = STEP_STAGE[step.id];
|
|
71
|
-
return step.phase === "done" && mapped !== undefined && mapped !== "needs-human";
|
|
72
|
-
});
|
|
73
|
-
if (earlier)
|
|
74
|
-
return STEP_STAGE[earlier.id] ?? "approve";
|
|
75
|
-
return "approve";
|
|
76
|
-
}
|
|
77
|
-
return job.workflow === "investigation" ? "explore" : "spec";
|
|
78
|
-
}
|
|
79
|
-
/** Stages to render. An empty kiln still shows the feature pipeline, collapsed. */
|
|
80
|
-
export function stagesFor(jobs) {
|
|
81
|
-
const workflows = new Set(jobs.map((job) => job.workflow));
|
|
82
|
-
const wanted = new Set();
|
|
83
|
-
if (jobs.length === 0 || workflows.has("feature") || [...workflows].some((name) => name !== "investigation")) {
|
|
84
|
-
for (const id of FEATURE_STAGES)
|
|
85
|
-
wanted.add(id);
|
|
86
|
-
}
|
|
87
|
-
if (workflows.has("investigation"))
|
|
88
|
-
wanted.add("explore");
|
|
89
|
-
wanted.add("queued");
|
|
90
|
-
wanted.add("needs-human");
|
|
91
|
-
const reached = new Set(jobs.map((job) => stageOf(job)));
|
|
92
|
-
if (reached.has("done") || !reached.has("merged"))
|
|
93
|
-
wanted.add("done");
|
|
94
|
-
if (reached.has("merged"))
|
|
95
|
-
wanted.add("merged");
|
|
96
|
-
for (const id of reached)
|
|
97
|
-
wanted.add(id);
|
|
98
|
-
return CATALOG.filter((stage) => wanted.has(stage.id));
|
|
99
|
-
}
|
|
100
3
|
const REVIEW_FILE = /^(security|ux|ui|code)(?:\.(\d+))?\.json$/;
|
|
101
4
|
const TEXT_LIMIT = 8_000;
|
|
102
5
|
export function readReviewRounds(runDir) {
|
package/dist/cli.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { addRepo, renderRepoList, resolveKilnConfig } from "./config.js";
|
|
3
3
|
import { renderSandboxStatus, runSandboxBuild } from "./sandbox.js";
|
|
4
|
-
import { answerJob, retryJob, runStatus, runTick } from "./driver.js";
|
|
4
|
+
import { answerJob, retryJob, reviseJob, runStatus, runTick } from "./driver.js";
|
|
5
|
+
import { pauseFactory, resumeFactory } from "./rails.js";
|
|
5
6
|
import { parseSince, renderLedgerFile } from "./ledger.js";
|
|
6
7
|
import { startServe } from "./serve.js";
|
|
7
8
|
import { addFromFile } from "./intake/file.js";
|
|
@@ -19,7 +20,10 @@ function usage() {
|
|
|
19
20
|
" kiln status [jobId] [--config <file>]",
|
|
20
21
|
" kiln approve <jobId> [--config <file>]",
|
|
21
22
|
" kiln reject <jobId> [--config <file>]",
|
|
23
|
+
" kiln revise <jobId> --note <text> [--config <file>]",
|
|
22
24
|
" kiln retry <jobId> [--config <file>]",
|
|
25
|
+
" kiln pause [--config <file>]",
|
|
26
|
+
" kiln resume [--config <file>]",
|
|
23
27
|
" kiln ledger [--since 7d] [--repo <name>] [--config <file>]",
|
|
24
28
|
" kiln sandbox [build] [--config <file>]",
|
|
25
29
|
" kiln serve [--port 4100] [--tick <seconds>|--no-tick] [--config <file>]",
|
|
@@ -66,7 +70,13 @@ function usage() {
|
|
|
66
70
|
"or a process that is already gone when tick or serve starts (orphaned) retries",
|
|
67
71
|
"after 10s, 20s, 40s, and so on, capped at 5 minutes, up to maxAttempts",
|
|
68
72
|
"A rejected Claude login (401 or an invalid bearer token) is auth-failed and is not retried.",
|
|
69
|
-
"(default 3).
|
|
73
|
+
"(default 3, overridable per repo). Hitting the cap is a budget failure and the",
|
|
74
|
+
"job needs a person. The same failure fingerprint stops the job after",
|
|
75
|
+
"maxSameFingerprint (default 3). A test, build, or type error is not retried.",
|
|
76
|
+
"A rejected gate is not retried. revise sends the job around again with the",
|
|
77
|
+
"note in the next prompt. pause stops new runs until resume. In-flight runs",
|
|
78
|
+
"finish. A daily dollar cap, when set, does the same once today's recorded",
|
|
79
|
+
"spend reaches it. ledger prints that spend.",
|
|
70
80
|
"Attempts are files under the kiln home, not fields on the job.",
|
|
71
81
|
"Events are one JSON line each in <kiln-home>/events.jsonl. A write",
|
|
72
82
|
"failure is reported and does not stop the job. kiln ledger counts",
|
|
@@ -227,6 +237,13 @@ function parseFlags(argv) {
|
|
|
227
237
|
bag.noTick = true;
|
|
228
238
|
continue;
|
|
229
239
|
}
|
|
240
|
+
if (flag === "--note") {
|
|
241
|
+
if (!next || next.startsWith("--"))
|
|
242
|
+
usage();
|
|
243
|
+
bag.note = next;
|
|
244
|
+
i += 1;
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
230
247
|
if (flag === "--since") {
|
|
231
248
|
if (!next || next.startsWith("--"))
|
|
232
249
|
usage();
|
|
@@ -380,7 +397,7 @@ function sandboxCommand(argv) {
|
|
|
380
397
|
function tickCommand(argv) {
|
|
381
398
|
const flags = parseFlags(argv);
|
|
382
399
|
rejectServeFlags(flags);
|
|
383
|
-
if (flags.positionals.length !== 0 || flags.test || hasIntakeFlags(flags) || flags.deliver || flags.base || flags.stack) {
|
|
400
|
+
if (flags.positionals.length !== 0 || flags.test || flags.note || hasIntakeFlags(flags) || flags.deliver || flags.base || flags.stack) {
|
|
384
401
|
usage();
|
|
385
402
|
}
|
|
386
403
|
const config = resolveKilnConfig({ configPath: flags.configPath });
|
|
@@ -389,7 +406,7 @@ function tickCommand(argv) {
|
|
|
389
406
|
function statusCommand(argv) {
|
|
390
407
|
const flags = parseFlags(argv);
|
|
391
408
|
rejectServeFlags(flags);
|
|
392
|
-
if (flags.positionals.length > 1 || flags.test || hasIntakeFlags(flags) || flags.deliver || flags.base || flags.stack) {
|
|
409
|
+
if (flags.positionals.length > 1 || flags.test || flags.note || hasIntakeFlags(flags) || flags.deliver || flags.base || flags.stack) {
|
|
393
410
|
usage();
|
|
394
411
|
}
|
|
395
412
|
const config = resolveKilnConfig({ configPath: flags.configPath });
|
|
@@ -399,7 +416,7 @@ function retryCommand(argv) {
|
|
|
399
416
|
const flags = parseFlags(argv);
|
|
400
417
|
rejectServeFlags(flags);
|
|
401
418
|
const jobId = flags.positionals[0];
|
|
402
|
-
if (!jobId || flags.positionals.length !== 1 || flags.test || hasIntakeFlags(flags) || flags.deliver || flags.base || flags.stack) {
|
|
419
|
+
if (!jobId || flags.positionals.length !== 1 || flags.test || flags.note || hasIntakeFlags(flags) || flags.deliver || flags.base || flags.stack) {
|
|
403
420
|
usage();
|
|
404
421
|
}
|
|
405
422
|
const config = resolveKilnConfig({ configPath: flags.configPath });
|
|
@@ -429,13 +446,41 @@ function ledgerCommand(argv) {
|
|
|
429
446
|
process.stdout.write(renderLedgerFile(config.kilnDir, {
|
|
430
447
|
...(since ? { since } : {}),
|
|
431
448
|
...(flags.repo ? { repo: flags.repo } : {}),
|
|
449
|
+
...(config.dailyDollarCap !== undefined ? { dailyDollarCap: config.dailyDollarCap } : {}),
|
|
432
450
|
}));
|
|
433
451
|
}
|
|
452
|
+
function reviseCommand(argv) {
|
|
453
|
+
const flags = parseFlags(argv);
|
|
454
|
+
rejectServeFlags(flags);
|
|
455
|
+
const jobId = flags.positionals[0];
|
|
456
|
+
if (!jobId ||
|
|
457
|
+
flags.positionals.length !== 1 ||
|
|
458
|
+
!flags.note ||
|
|
459
|
+
flags.test ||
|
|
460
|
+
hasIntakeFlags(flags) ||
|
|
461
|
+
flags.deliver ||
|
|
462
|
+
flags.base ||
|
|
463
|
+
flags.stack) {
|
|
464
|
+
usage();
|
|
465
|
+
}
|
|
466
|
+
const config = resolveKilnConfig({ configPath: flags.configPath });
|
|
467
|
+
process.stdout.write(reviseJob(config, jobId, flags.note));
|
|
468
|
+
}
|
|
469
|
+
function pauseCommand(argv, pause) {
|
|
470
|
+
const flags = parseFlags(argv);
|
|
471
|
+
rejectServeFlags(flags);
|
|
472
|
+
if (flags.positionals.length !== 0 || flags.test || hasIntakeFlags(flags) || flags.note || flags.deliver || flags.base || flags.stack) {
|
|
473
|
+
usage();
|
|
474
|
+
}
|
|
475
|
+
const config = resolveKilnConfig({ configPath: flags.configPath });
|
|
476
|
+
const changed = pause ? pauseFactory(config.kilnDir) : resumeFactory(config.kilnDir);
|
|
477
|
+
process.stdout.write(pause ? (changed ? "paused\n" : "already paused\n") : changed ? "resumed\n" : "already running\n");
|
|
478
|
+
}
|
|
434
479
|
function gateCommand(choice, argv) {
|
|
435
480
|
const flags = parseFlags(argv);
|
|
436
481
|
rejectServeFlags(flags);
|
|
437
482
|
const jobId = flags.positionals[0];
|
|
438
|
-
if (!jobId || flags.positionals.length !== 1 || flags.test || hasIntakeFlags(flags) || flags.deliver || flags.base || flags.stack) {
|
|
483
|
+
if (!jobId || flags.positionals.length !== 1 || flags.test || flags.note || hasIntakeFlags(flags) || flags.deliver || flags.base || flags.stack) {
|
|
439
484
|
usage();
|
|
440
485
|
}
|
|
441
486
|
const config = resolveKilnConfig({ configPath: flags.configPath });
|
|
@@ -511,6 +556,18 @@ async function main() {
|
|
|
511
556
|
retryCommand(argv.slice(1));
|
|
512
557
|
return;
|
|
513
558
|
}
|
|
559
|
+
if (command === "revise") {
|
|
560
|
+
reviseCommand(argv.slice(1));
|
|
561
|
+
return;
|
|
562
|
+
}
|
|
563
|
+
if (command === "pause") {
|
|
564
|
+
pauseCommand(argv.slice(1), true);
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
567
|
+
if (command === "resume") {
|
|
568
|
+
pauseCommand(argv.slice(1), false);
|
|
569
|
+
return;
|
|
570
|
+
}
|
|
514
571
|
if (command === "ledger") {
|
|
515
572
|
ledgerCommand(argv.slice(1));
|
|
516
573
|
return;
|
package/dist/config.js
CHANGED
|
@@ -2,7 +2,7 @@ import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "no
|
|
|
2
2
|
import { homedir } from "node:os";
|
|
3
3
|
import { dirname, isAbsolute, join, resolve } from "node:path";
|
|
4
4
|
import { KilnConfigError } from "./errors.js";
|
|
5
|
-
import { DEFAULT_MAX_ATTEMPTS, DEFAULT_MAX_FIX_ROUNDS, DEFAULT_STALL_TIMEOUT_MS, KilnConfigFileSchema, REPO_NAME, } from "./schemas.js";
|
|
5
|
+
import { DEFAULT_MAX_ATTEMPTS, DEFAULT_MAX_FIX_ROUNDS, DEFAULT_MAX_SAME_FINGERPRINT, DEFAULT_STALL_TIMEOUT_MS, KilnConfigFileSchema, REPO_NAME, } from "./schemas.js";
|
|
6
6
|
const CONFIG_NAME = "kiln.config.json";
|
|
7
7
|
export function loadKilnConfig(configPath, options = {}) {
|
|
8
8
|
const home = options.home ?? homedir();
|
|
@@ -31,6 +31,8 @@ export function loadKilnConfig(configPath, options = {}) {
|
|
|
31
31
|
...(entry.base ? { base: entry.base } : {}),
|
|
32
32
|
...(entry.prTitle ? { prTitle: entry.prTitle } : {}),
|
|
33
33
|
...(entry.branch ? { branchTemplate: entry.branch } : {}),
|
|
34
|
+
...(entry.maxAttempts !== undefined ? { maxAttempts: entry.maxAttempts } : {}),
|
|
35
|
+
...(entry.maxFixRounds !== undefined ? { maxFixRounds: entry.maxFixRounds } : {}),
|
|
34
36
|
};
|
|
35
37
|
}
|
|
36
38
|
const kilnDir = configDir;
|
|
@@ -46,6 +48,8 @@ export function loadKilnConfig(configPath, options = {}) {
|
|
|
46
48
|
stallTimeoutMs: parsed.data.stallTimeoutMs ?? DEFAULT_STALL_TIMEOUT_MS,
|
|
47
49
|
maxAttempts: parsed.data.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
|
|
48
50
|
maxFixRounds: parsed.data.maxFixRounds ?? DEFAULT_MAX_FIX_ROUNDS,
|
|
51
|
+
maxSameFingerprint: parsed.data.maxSameFingerprint ?? DEFAULT_MAX_SAME_FINGERPRINT,
|
|
52
|
+
...(parsed.data.dailyDollarCap !== undefined ? { dailyDollarCap: parsed.data.dailyDollarCap } : {}),
|
|
49
53
|
adapter: parsed.data.adapter ?? "claude-code",
|
|
50
54
|
sandbox: parsed.data.sandbox ?? {},
|
|
51
55
|
workflowSandbox: workflowSandboxFrom(parsed.data.workflows),
|
|
@@ -112,6 +116,12 @@ export function addRepo(options) {
|
|
|
112
116
|
const branchTemplate = previousString(previous, "branch");
|
|
113
117
|
if (branchTemplate)
|
|
114
118
|
entry.branch = branchTemplate;
|
|
119
|
+
const maxAttempts = previousNumber(previous, "maxAttempts");
|
|
120
|
+
if (maxAttempts !== undefined)
|
|
121
|
+
entry.maxAttempts = maxAttempts;
|
|
122
|
+
const maxFixRounds = previousNumber(previous, "maxFixRounds");
|
|
123
|
+
if (maxFixRounds !== undefined)
|
|
124
|
+
entry.maxFixRounds = maxFixRounds;
|
|
115
125
|
repos[options.name] = entry;
|
|
116
126
|
raw.repos = repos;
|
|
117
127
|
const parsed = KilnConfigFileSchema.safeParse(raw);
|
|
@@ -138,6 +148,14 @@ export function addRepo(options) {
|
|
|
138
148
|
lines.push(`wrote ${configPath}`);
|
|
139
149
|
return `${lines.join("\n")}\n`;
|
|
140
150
|
}
|
|
151
|
+
/** Attempt cap for one repo: the repo override, otherwise the global cap. */
|
|
152
|
+
export function attemptsFor(config, repo) {
|
|
153
|
+
return config.repos[repo]?.maxAttempts ?? config.maxAttempts;
|
|
154
|
+
}
|
|
155
|
+
/** Fix-round cap for one repo: the repo override, otherwise the global cap. */
|
|
156
|
+
export function fixRoundsFor(config, repo) {
|
|
157
|
+
return config.repos[repo]?.maxFixRounds ?? config.maxFixRounds;
|
|
158
|
+
}
|
|
141
159
|
export function renderRepoList(config) {
|
|
142
160
|
const names = Object.keys(config.repos).sort();
|
|
143
161
|
if (names.length === 0)
|
|
@@ -226,6 +244,12 @@ function previousString(previous, key) {
|
|
|
226
244
|
const value = previous[key];
|
|
227
245
|
return typeof value === "string" && value.length > 0 ? value : undefined;
|
|
228
246
|
}
|
|
247
|
+
function previousNumber(previous, key) {
|
|
248
|
+
if (typeof previous !== "object" || previous === null || !(key in previous))
|
|
249
|
+
return undefined;
|
|
250
|
+
const value = previous[key];
|
|
251
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
252
|
+
}
|
|
229
253
|
function previousDeliver(previous) {
|
|
230
254
|
const value = previousString(previous, "deliver");
|
|
231
255
|
if (value === "local" || value === "pr")
|
package/dist/driver.js
CHANGED
|
@@ -4,16 +4,20 @@ import { closeSync, existsSync, linkSync, mkdirSync, openSync, readdirSync, read
|
|
|
4
4
|
import { basename, dirname, isAbsolute, join } from "node:path";
|
|
5
5
|
import { claudeCredentialPreflight } from "./auth.js";
|
|
6
6
|
import { buildAttention } from "./attention.js";
|
|
7
|
+
import { attemptsFor, fixRoundsFor } from "./config.js";
|
|
7
8
|
import { clearLaunchError, deliverJob, plannedRemoteBranch, readDelivery, readLaunchError, resolveLaunchBase, writeDelivery, writeLaunchError, isMerged, } from "./delivery.js";
|
|
8
9
|
import { feedbackSpec, openFeedbackRound, planFeedback, readFeedbackLedger, writeFeedbackLedger, } from "./feedback.js";
|
|
9
|
-
import { appendEvent } from "./events.js";
|
|
10
|
+
import { appendEvent, readEvents } from "./events.js";
|
|
11
|
+
import { costFromJournal } from "./failure.js";
|
|
12
|
+
import { ensureFixTrailers } from "./fix-commit.js";
|
|
13
|
+
import { holdDetail, pausedFrom, recordDailyCap, reviseNotesSinceRun, spendSnapshot, stopReasonFromEvents, withReviseNotes, } from "./rails.js";
|
|
10
14
|
import { appendInvestigationFindings, existingFindingsPath, findingsFilePath, INVESTIGATION_WORKFLOW, readFindingsMarkdown, writeFindings, } from "./findings.js";
|
|
11
15
|
import { defaultHost } from "./host.js";
|
|
12
16
|
import { readJobSpec, resolveJobSpec } from "./jobs.js";
|
|
13
17
|
import { jobBasename, packageRoot } from "./paths.js";
|
|
14
18
|
import { decide, isLanded, jobState, } from "./reconcile.js";
|
|
15
19
|
import { fileMtimeMs, isPidAlive, readAttemptLedger, recoverRuns, resetAttemptLedger, } from "./reliability.js";
|
|
16
|
-
import { projectRunJournal } from "./project.js";
|
|
20
|
+
import { projectRunJournal, promptsHash } from "./project.js";
|
|
17
21
|
import { gateAnswerCommand, renderStatus, renderTick, statusView, yakCliPath, sandboxFailureDetail, } from "./report.js";
|
|
18
22
|
import { AGENT_SANDBOX_IMAGE, injectAgentSandbox, jobIsSandboxed, probeSandboxEnvironment, PROXY_SANDBOX_IMAGE, requiredSandboxImages, resolveSandbox, sandboxIsEnabled, sandboxPreflight, sandboxPreflightFor, } from "./sandbox.js";
|
|
19
23
|
import { JobSchema } from "./schemas.js";
|
|
@@ -42,16 +46,19 @@ export function runTick(config, host = defaultHost(), options = {}) {
|
|
|
42
46
|
const startup = options.startup ?? true;
|
|
43
47
|
const now = options.now ?? new Date();
|
|
44
48
|
return withTickLock(config.kilnDir, () => {
|
|
45
|
-
const notes = recoverRuns(config, observe(config, host).observation.jobs, { startup, now });
|
|
49
|
+
const notes = recoverRuns(config, observe(config, host, now).observation.jobs, { startup, now });
|
|
46
50
|
const facts = sandboxIsEnabled(config)
|
|
47
51
|
? (options.sandboxFacts ?? probeSandboxEnvironment(process.env, requiredSandboxImages(config)))
|
|
48
52
|
: undefined;
|
|
49
|
-
const midway = observe(config, host);
|
|
53
|
+
const midway = observe(config, host, now);
|
|
50
54
|
bindOpenFixRuns(config, midway);
|
|
51
55
|
const published = publishReadyFixes(config, host, midway);
|
|
52
|
-
const world = observe(config, host);
|
|
56
|
+
const world = observe(config, host, now);
|
|
53
57
|
const feedbackNotes = watchPullRequests(config, world, host);
|
|
54
58
|
const actions = decide(world.observation, now);
|
|
59
|
+
if (world.observation.spendBlocked) {
|
|
60
|
+
recordDailyCap(config.kilnDir, spendSnapshot(readEvents(config.kilnDir, { warn: () => { } }), config.dailyDollarCap, now), now);
|
|
61
|
+
}
|
|
55
62
|
const body = apply(config, world, actions, host, notes, now, [...published, ...feedbackNotes], facts);
|
|
56
63
|
const heads = [
|
|
57
64
|
claudeCredentialPreflight(process.env, config.adapter),
|
|
@@ -63,7 +70,7 @@ export function runTick(config, host = defaultHost(), options = {}) {
|
|
|
63
70
|
/** Startup pass shared by `kiln tick` (via `runTick`) and `kiln serve`. Idempotent. */
|
|
64
71
|
export function startupRecovery(config, host = defaultHost(), now = new Date()) {
|
|
65
72
|
withTickLock(config.kilnDir, () => {
|
|
66
|
-
recoverRuns(config, observe(config, host).observation.jobs, { startup: true, now });
|
|
73
|
+
recoverRuns(config, observe(config, host, now).observation.jobs, { startup: true, now });
|
|
67
74
|
});
|
|
68
75
|
}
|
|
69
76
|
export function answerJob(config, jobId, choice, host = defaultHost()) {
|
|
@@ -126,8 +133,8 @@ export function retryJob(config, jobId, host = defaultHost()) {
|
|
|
126
133
|
if (observed.error)
|
|
127
134
|
throw new Error(`Job ${jobId} is invalid: ${observed.error}`);
|
|
128
135
|
const state = jobState(observed, world.observation);
|
|
129
|
-
if (state
|
|
130
|
-
throw new Error(`Job ${jobId} is ${state}, not failed or
|
|
136
|
+
if (!canManualRetry(state, observed)) {
|
|
137
|
+
throw new Error(`Job ${jobId} is ${state}, not failed, dead, or stopped for a person.`);
|
|
131
138
|
}
|
|
132
139
|
const job = world.jobs.get(jobId);
|
|
133
140
|
if (!job)
|
|
@@ -154,7 +161,7 @@ export function retryJob(config, jobId, host = defaultHost()) {
|
|
|
154
161
|
const detail = report.detail.endsWith("\n") || report.detail.length === 0 ? report.detail : `${report.detail}\n`;
|
|
155
162
|
return `retry ${jobId} ${job.repo}\n${clearedLine}${detail}`;
|
|
156
163
|
}
|
|
157
|
-
if (state === "dead" && observed.run) {
|
|
164
|
+
if ((state === "dead" || observed.stopReason) && observed.run) {
|
|
158
165
|
try {
|
|
159
166
|
spawnSync(process.execPath, [yakCliPath(), "cancel", observed.run.runId], {
|
|
160
167
|
cwd: observed.run.checkout,
|
|
@@ -174,6 +181,64 @@ export function retryJob(config, jobId, host = defaultHost()) {
|
|
|
174
181
|
return `retry ${report.jobId} ${report.repo}\n${clearedLine}${output}`;
|
|
175
182
|
});
|
|
176
183
|
}
|
|
184
|
+
function canManualRetry(state, job) {
|
|
185
|
+
if (state === "failed" || state === "dead")
|
|
186
|
+
return true;
|
|
187
|
+
return state === "needs-human" && Boolean(job.stopReason) && (job.run?.gates.length ?? 0) === 0;
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Send a gated job back for another round. The note is stored on a
|
|
191
|
+
* `gate.resolved` event (redacted, otherwise verbatim) and appended to the
|
|
192
|
+
* spec the next launch passes into the prompt.
|
|
193
|
+
*/
|
|
194
|
+
export function reviseJob(config, jobId, note, host = defaultHost()) {
|
|
195
|
+
const trimmed = note.trim();
|
|
196
|
+
if (!trimmed)
|
|
197
|
+
throw new Error("A revise note is required.");
|
|
198
|
+
if (trimmed.length > 8_000)
|
|
199
|
+
throw new Error("A revise note must be 8000 characters or fewer.");
|
|
200
|
+
return withTickLock(config.kilnDir, () => {
|
|
201
|
+
const world = observe(config, host);
|
|
202
|
+
const observed = world.observation.jobs.find((candidate) => candidate.jobId === jobId);
|
|
203
|
+
if (!observed)
|
|
204
|
+
throw new Error(`No job ${jobId}.`);
|
|
205
|
+
if (observed.error)
|
|
206
|
+
throw new Error(`Job ${jobId} is invalid: ${observed.error}`);
|
|
207
|
+
const state = jobState(observed, world.observation);
|
|
208
|
+
const gate = observed.run?.gates[0];
|
|
209
|
+
if (state !== "needs-human" || !observed.run || !gate) {
|
|
210
|
+
throw new Error(`Job ${jobId} is ${state}, not waiting on a gate.`);
|
|
211
|
+
}
|
|
212
|
+
const job = world.jobs.get(jobId);
|
|
213
|
+
if (!job)
|
|
214
|
+
throw new Error(`No job ${jobId}.`);
|
|
215
|
+
appendEvent(config.kilnDir, {
|
|
216
|
+
job_id: jobId,
|
|
217
|
+
run_id: observed.run.runId,
|
|
218
|
+
event: "gate.resolved",
|
|
219
|
+
step: gate.stepId,
|
|
220
|
+
properties: {
|
|
221
|
+
decision: "revise",
|
|
222
|
+
by: "human",
|
|
223
|
+
repo: job.repo,
|
|
224
|
+
note: trimmed,
|
|
225
|
+
},
|
|
226
|
+
});
|
|
227
|
+
resetAttemptLedger(config.kilnDir, jobId, observed.run.runId);
|
|
228
|
+
rmSync(breadcrumbPath(config.kilnDir, jobId), { force: true });
|
|
229
|
+
clearLaunchError(config.kilnDir, jobId);
|
|
230
|
+
const events = readEvents(config.kilnDir, { warn: () => { } });
|
|
231
|
+
const spend = spendSnapshot(events, config.dailyDollarCap);
|
|
232
|
+
const paused = pausedFrom(events);
|
|
233
|
+
const hold = holdDetail(paused, spend);
|
|
234
|
+
if (hold)
|
|
235
|
+
return `revised ${jobId}\n${hold}\n`;
|
|
236
|
+
const again = observe(config, host);
|
|
237
|
+
const report = launchJob(config, again, job, host);
|
|
238
|
+
const output = report.output.endsWith("\n") || report.output.length === 0 ? report.output : `${report.output}\n`;
|
|
239
|
+
return `revised ${jobId} ${report.repo}\n${output}`;
|
|
240
|
+
});
|
|
241
|
+
}
|
|
177
242
|
function shouldRedeliver(job) {
|
|
178
243
|
if (!isLanded(job))
|
|
179
244
|
return false;
|
|
@@ -185,9 +250,13 @@ function gateForChoice(gates, choice) {
|
|
|
185
250
|
return (gates.find((gate) => gate.choices?.includes(choice)) ??
|
|
186
251
|
gates.find((gate) => !gate.choices?.length));
|
|
187
252
|
}
|
|
188
|
-
function observe(config, host) {
|
|
253
|
+
function observe(config, host, now = new Date()) {
|
|
189
254
|
const loaded = loadJobs(config);
|
|
190
255
|
const runs = scanRuns(config);
|
|
256
|
+
const events = readEvents(config.kilnDir, { warn: () => { } });
|
|
257
|
+
const paused = pausedFrom(events);
|
|
258
|
+
const spend = spendSnapshot(events, config.dailyDollarCap, now);
|
|
259
|
+
const waiting = holdDetail(paused, spend);
|
|
191
260
|
const breadcrumbs = readBreadcrumbs(config.kilnDir);
|
|
192
261
|
const jobs = [];
|
|
193
262
|
const records = new Map();
|
|
@@ -234,10 +303,13 @@ function observe(config, host) {
|
|
|
234
303
|
const launchActivityMs = later(crumb ? fileMtimeMs(crumb.path) : undefined, logPath ? fileMtimeMs(logPath) : undefined);
|
|
235
304
|
const attemptCount = ledger?.attempts.length ?? 0;
|
|
236
305
|
const lastFailure = ledger?.attempts.at(-1);
|
|
237
|
-
const
|
|
306
|
+
const attemptCap = attemptsFor(config, entry.job?.repo ?? "");
|
|
307
|
+
const exhausted = attemptCount >= attemptCap &&
|
|
238
308
|
attemptCount > 0 &&
|
|
239
309
|
!launching &&
|
|
240
310
|
(picked.retired || !run || run.status === "dead" || run.status === "failed");
|
|
311
|
+
const eventStop = entry.job ? stopReasonFromEvents(events, entry.job.id) : undefined;
|
|
312
|
+
const stopReason = eventStop ?? (exhausted ? "attempt-cap" : undefined);
|
|
241
313
|
const specPath = entry.job ? resolveJobSpec(config.kilnDir, entry.job.spec) : undefined;
|
|
242
314
|
const findingsPath = entry.job?.workflow === INVESTIGATION_WORKFLOW
|
|
243
315
|
? existingFindingsPath(config.kilnDir, entry.job.id)
|
|
@@ -276,6 +348,8 @@ function observe(config, host) {
|
|
|
276
348
|
...(lastFailure ? { lastFailureReason: lastFailure.reason } : {}),
|
|
277
349
|
...(ledger?.nextRetryAt && !exhausted ? { nextRetryAt: ledger.nextRetryAt } : {}),
|
|
278
350
|
...(exhausted ? { retriesExhausted: true } : {}),
|
|
351
|
+
...(stopReason ? { stopReason } : {}),
|
|
352
|
+
attemptCap,
|
|
279
353
|
...(picked.retired ? { runRetired: true } : {}),
|
|
280
354
|
...(ledgerResult === "corrupt" ? { ledgerError: "attempt ledger is unreadable" } : {}),
|
|
281
355
|
...(entry.job ? { sandboxed: jobIsSandboxed(config, entry.job.id, entry.job.workflow) } : {}),
|
|
@@ -296,6 +370,9 @@ function observe(config, host) {
|
|
|
296
370
|
maxAttempts: config.maxAttempts,
|
|
297
371
|
jobs,
|
|
298
372
|
breadcrumbsToClear,
|
|
373
|
+
...(paused ? { paused: true } : {}),
|
|
374
|
+
...(spend.blocked ? { spendBlocked: true } : {}),
|
|
375
|
+
...(waiting ? { holdDetail: waiting } : {}),
|
|
299
376
|
},
|
|
300
377
|
jobs: records,
|
|
301
378
|
};
|
|
@@ -438,7 +515,8 @@ function specHandedToYak(config, world, job) {
|
|
|
438
515
|
}
|
|
439
516
|
sections.push({ title: dep.title, markdown: readFileSync(path, "utf8") });
|
|
440
517
|
}
|
|
441
|
-
|
|
518
|
+
const noted = withReviseNotes(appendInvestigationFindings(base, sections), reviseNotesSinceRun(readEvents(config.kilnDir, { warn: () => { } }), job.id));
|
|
519
|
+
return noted;
|
|
442
520
|
}
|
|
443
521
|
/** Remote name the stacked pull request targets. Falls back to the local yak ref. */
|
|
444
522
|
function upstreamBranch(world, job) {
|
|
@@ -550,6 +628,15 @@ function publishReadyFixes(config, host, world) {
|
|
|
550
628
|
const record = readDelivery(config.kilnDir, job.jobId);
|
|
551
629
|
const checkout = job.run.checkout;
|
|
552
630
|
const source = `yak/${job.run.runId}`;
|
|
631
|
+
const worktree = join(checkout, ".yak", "worktrees", job.run.runId);
|
|
632
|
+
if (existsSync(worktree)) {
|
|
633
|
+
ensureFixTrailers(worktree, {
|
|
634
|
+
jobId: job.jobId,
|
|
635
|
+
round: open.round,
|
|
636
|
+
promptsHash: hashForRun(config.kilnDir, job.jobId, job.run.runId),
|
|
637
|
+
title: job.title,
|
|
638
|
+
}, open.headSha);
|
|
639
|
+
}
|
|
553
640
|
if (!record?.branch) {
|
|
554
641
|
notes.push(`feedback ${job.jobId} failed: no pull request branch to publish`);
|
|
555
642
|
continue;
|
|
@@ -607,6 +694,14 @@ function publishReadyFixes(config, host, world) {
|
|
|
607
694
|
}
|
|
608
695
|
return notes;
|
|
609
696
|
}
|
|
697
|
+
function hashForRun(kilnDir, jobId, runId) {
|
|
698
|
+
for (const event of readEvents(kilnDir, { warn: () => { } })) {
|
|
699
|
+
if (event.job_id === jobId && event.run_id === runId && event.event === "run.started") {
|
|
700
|
+
return event.properties.prompts_hash;
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
return promptsHash("fix", kilnDir, jobId);
|
|
704
|
+
}
|
|
610
705
|
function isNonFastForward(message) {
|
|
611
706
|
return /non-fast-forward|\[rejected\]|failed to push some refs/i.test(message);
|
|
612
707
|
}
|
|
@@ -676,7 +771,7 @@ function watchPullRequests(config, world, host) {
|
|
|
676
771
|
continue;
|
|
677
772
|
}
|
|
678
773
|
job.feedbackPlan = planFeedback({
|
|
679
|
-
maxFixRounds: config.
|
|
774
|
+
maxFixRounds: fixRoundsFor(config, job.repo),
|
|
680
775
|
ledger,
|
|
681
776
|
snapshot,
|
|
682
777
|
});
|
|
@@ -769,7 +864,7 @@ function startFix(config, world, job, host, now, sandboxFacts) {
|
|
|
769
864
|
writeBreadcrumb(breadcrumbPath(config.kilnDir, job.id), { jobId: job.id, pid: 0, logPath });
|
|
770
865
|
return { jobId: job.id, repo: job.repo, kind: "fix", output: message };
|
|
771
866
|
}
|
|
772
|
-
const report = launchFixRun(config, job, host, repo.path, headSha, specText, sandboxFacts);
|
|
867
|
+
const report = launchFixRun(config, job, host, repo.path, headSha, specText, round, sandboxFacts);
|
|
773
868
|
return { ...report, output: `round ${round}\n${report.output}` };
|
|
774
869
|
}
|
|
775
870
|
/** Relaunch the open fix round. Does not append a round. Failures become a dead launch so retry can see them. */
|
|
@@ -790,7 +885,7 @@ function launchOpenFix(config, world, job, host, checkout, open, sandboxFacts) {
|
|
|
790
885
|
writeBreadcrumb(breadcrumbPath(config.kilnDir, job.id), { jobId: job.id, pid: 0, logPath });
|
|
791
886
|
return { jobId: job.id, repo: job.repo, kind: "fix", output: message };
|
|
792
887
|
}
|
|
793
|
-
return launchFixRun(config, job, host, checkout, headSha, specText, sandboxFacts);
|
|
888
|
+
return launchFixRun(config, job, host, checkout, headSha, specText, open.round, sandboxFacts);
|
|
794
889
|
}
|
|
795
890
|
function resolveCommit(host, checkout, sha) {
|
|
796
891
|
try {
|
|
@@ -802,7 +897,7 @@ function resolveCommit(host, checkout, sha) {
|
|
|
802
897
|
}
|
|
803
898
|
}
|
|
804
899
|
/** `workflows/fix.yaml` on `headSha`. Commits on `yak/<run-id>`. Does not push. */
|
|
805
|
-
function launchFixRun(config, job, host, checkout, headSha, specText, sandboxFacts) {
|
|
900
|
+
function launchFixRun(config, job, host, checkout, headSha, specText, round, sandboxFacts) {
|
|
806
901
|
const refused = claudeCredentialPreflight(process.env, config.adapter);
|
|
807
902
|
if (refused)
|
|
808
903
|
return { jobId: job.id, repo: job.repo, kind: "fix", output: refused };
|
|
@@ -814,6 +909,7 @@ function launchFixRun(config, job, host, checkout, headSha, specText, sandboxFac
|
|
|
814
909
|
const blocked = commitAgentSandbox(config, job.id, "fix", workflow, logPath, sandboxFacts);
|
|
815
910
|
if (blocked)
|
|
816
911
|
return { jobId: job.id, repo: job.repo, kind: "fix", output: blocked };
|
|
912
|
+
const hash = promptsHash("fix", config.kilnDir, job.id);
|
|
817
913
|
ensureCheckoutExcludes(checkout);
|
|
818
914
|
const input = workflowInputs(sourceWorkflow, {
|
|
819
915
|
title: job.title,
|
|
@@ -826,6 +922,9 @@ function launchFixRun(config, job, host, checkout, headSha, specText, sandboxFac
|
|
|
826
922
|
...process.env,
|
|
827
923
|
...(repo?.test ? { KILN_TEST: repo.test } : {}),
|
|
828
924
|
KILN_TASK_TITLE: job.title,
|
|
925
|
+
KILN_JOB: job.id,
|
|
926
|
+
KILN_ROUND: String(round),
|
|
927
|
+
KILN_PROMPTS_HASH: hash,
|
|
829
928
|
KILN_VERIFY_WRAPPER: join(packageRoot(), "dist", "truncate-output.js"),
|
|
830
929
|
};
|
|
831
930
|
return spawnLogged(config, host, {
|
|
@@ -1056,6 +1155,7 @@ function scanRuns(config) {
|
|
|
1056
1155
|
const stepPlan = readRunStepPlan(runDir);
|
|
1057
1156
|
const stepEvents = stepEventsFrom(events);
|
|
1058
1157
|
const sandboxError = sandboxFailureDetail(stepEvents);
|
|
1158
|
+
const cost = costFromJournal(events);
|
|
1059
1159
|
const pid = livePid(events);
|
|
1060
1160
|
const activityMs = eventActivityMs(events) ?? fileMtimeMs(join(runDir, "journal.jsonl"));
|
|
1061
1161
|
const finishedReason = finished && typeof finished.reason === "string" ? finished.reason : undefined;
|
|
@@ -1074,6 +1174,7 @@ function scanRuns(config) {
|
|
|
1074
1174
|
...(activityMs !== undefined ? { activityMs } : {}),
|
|
1075
1175
|
...(finishedReason ? { finishedReason } : {}),
|
|
1076
1176
|
...(sandboxError ? { sandboxError } : {}),
|
|
1177
|
+
cost,
|
|
1077
1178
|
};
|
|
1078
1179
|
const list = byTag.get(started.tag) ?? [];
|
|
1079
1180
|
list.push(run);
|