@lchase/kiln 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/assets/serve.css +13 -0
- package/assets/serve.js +49 -3
- package/dist/attention.js +13 -4
- package/dist/cli.js +63 -6
- package/dist/config.js +25 -1
- package/dist/driver.js +117 -16
- package/dist/events.js +40 -1
- package/dist/failure.js +140 -0
- package/dist/fix-commit.js +73 -0
- package/dist/ledger.js +11 -3
- package/dist/project.js +20 -4
- package/dist/rails.js +164 -0
- package/dist/reconcile.js +29 -3
- package/dist/reliability.js +53 -25
- package/dist/report.js +19 -5
- package/dist/schemas.js +35 -5
- package/dist/serve-page.js +21 -1
- package/dist/serve.js +70 -3
- package/dist/settings.js +9 -1
- package/package.json +2 -2
- package/workflows/fix.yaml +1 -1
package/dist/driver.js
CHANGED
|
@@ -4,16 +4,20 @@ import { closeSync, existsSync, linkSync, mkdirSync, openSync, readdirSync, read
|
|
|
4
4
|
import { basename, dirname, isAbsolute, join } from "node:path";
|
|
5
5
|
import { claudeCredentialPreflight } from "./auth.js";
|
|
6
6
|
import { buildAttention } from "./attention.js";
|
|
7
|
+
import { attemptsFor, fixRoundsFor } from "./config.js";
|
|
7
8
|
import { clearLaunchError, deliverJob, plannedRemoteBranch, readDelivery, readLaunchError, resolveLaunchBase, writeDelivery, writeLaunchError, isMerged, } from "./delivery.js";
|
|
8
9
|
import { feedbackSpec, openFeedbackRound, planFeedback, readFeedbackLedger, writeFeedbackLedger, } from "./feedback.js";
|
|
9
|
-
import { appendEvent } from "./events.js";
|
|
10
|
+
import { appendEvent, readEvents } from "./events.js";
|
|
11
|
+
import { costFromJournal } from "./failure.js";
|
|
12
|
+
import { ensureFixTrailers } from "./fix-commit.js";
|
|
13
|
+
import { holdDetail, pausedFrom, recordDailyCap, reviseNotesSinceRun, spendSnapshot, stopReasonFromEvents, withReviseNotes, } from "./rails.js";
|
|
10
14
|
import { appendInvestigationFindings, existingFindingsPath, findingsFilePath, INVESTIGATION_WORKFLOW, readFindingsMarkdown, writeFindings, } from "./findings.js";
|
|
11
15
|
import { defaultHost } from "./host.js";
|
|
12
16
|
import { readJobSpec, resolveJobSpec } from "./jobs.js";
|
|
13
17
|
import { jobBasename, packageRoot } from "./paths.js";
|
|
14
18
|
import { decide, isLanded, jobState, } from "./reconcile.js";
|
|
15
19
|
import { fileMtimeMs, isPidAlive, readAttemptLedger, recoverRuns, resetAttemptLedger, } from "./reliability.js";
|
|
16
|
-
import { projectRunJournal } from "./project.js";
|
|
20
|
+
import { projectRunJournal, promptsHash } from "./project.js";
|
|
17
21
|
import { gateAnswerCommand, renderStatus, renderTick, statusView, yakCliPath, sandboxFailureDetail, } from "./report.js";
|
|
18
22
|
import { AGENT_SANDBOX_IMAGE, injectAgentSandbox, jobIsSandboxed, probeSandboxEnvironment, PROXY_SANDBOX_IMAGE, requiredSandboxImages, resolveSandbox, sandboxIsEnabled, sandboxPreflight, sandboxPreflightFor, } from "./sandbox.js";
|
|
19
23
|
import { JobSchema } from "./schemas.js";
|
|
@@ -42,16 +46,19 @@ export function runTick(config, host = defaultHost(), options = {}) {
|
|
|
42
46
|
const startup = options.startup ?? true;
|
|
43
47
|
const now = options.now ?? new Date();
|
|
44
48
|
return withTickLock(config.kilnDir, () => {
|
|
45
|
-
const notes = recoverRuns(config, observe(config, host).observation.jobs, { startup, now });
|
|
49
|
+
const notes = recoverRuns(config, observe(config, host, now).observation.jobs, { startup, now });
|
|
46
50
|
const facts = sandboxIsEnabled(config)
|
|
47
51
|
? (options.sandboxFacts ?? probeSandboxEnvironment(process.env, requiredSandboxImages(config)))
|
|
48
52
|
: undefined;
|
|
49
|
-
const midway = observe(config, host);
|
|
53
|
+
const midway = observe(config, host, now);
|
|
50
54
|
bindOpenFixRuns(config, midway);
|
|
51
55
|
const published = publishReadyFixes(config, host, midway);
|
|
52
|
-
const world = observe(config, host);
|
|
56
|
+
const world = observe(config, host, now);
|
|
53
57
|
const feedbackNotes = watchPullRequests(config, world, host);
|
|
54
58
|
const actions = decide(world.observation, now);
|
|
59
|
+
if (world.observation.spendBlocked) {
|
|
60
|
+
recordDailyCap(config.kilnDir, spendSnapshot(readEvents(config.kilnDir, { warn: () => { } }), config.dailyDollarCap, now), now);
|
|
61
|
+
}
|
|
55
62
|
const body = apply(config, world, actions, host, notes, now, [...published, ...feedbackNotes], facts);
|
|
56
63
|
const heads = [
|
|
57
64
|
claudeCredentialPreflight(process.env, config.adapter),
|
|
@@ -63,7 +70,7 @@ export function runTick(config, host = defaultHost(), options = {}) {
|
|
|
63
70
|
/** Startup pass shared by `kiln tick` (via `runTick`) and `kiln serve`. Idempotent. */
|
|
64
71
|
export function startupRecovery(config, host = defaultHost(), now = new Date()) {
|
|
65
72
|
withTickLock(config.kilnDir, () => {
|
|
66
|
-
recoverRuns(config, observe(config, host).observation.jobs, { startup: true, now });
|
|
73
|
+
recoverRuns(config, observe(config, host, now).observation.jobs, { startup: true, now });
|
|
67
74
|
});
|
|
68
75
|
}
|
|
69
76
|
export function answerJob(config, jobId, choice, host = defaultHost()) {
|
|
@@ -126,8 +133,8 @@ export function retryJob(config, jobId, host = defaultHost()) {
|
|
|
126
133
|
if (observed.error)
|
|
127
134
|
throw new Error(`Job ${jobId} is invalid: ${observed.error}`);
|
|
128
135
|
const state = jobState(observed, world.observation);
|
|
129
|
-
if (state
|
|
130
|
-
throw new Error(`Job ${jobId} is ${state}, not failed or
|
|
136
|
+
if (!canManualRetry(state, observed)) {
|
|
137
|
+
throw new Error(`Job ${jobId} is ${state}, not failed, dead, or stopped for a person.`);
|
|
131
138
|
}
|
|
132
139
|
const job = world.jobs.get(jobId);
|
|
133
140
|
if (!job)
|
|
@@ -154,7 +161,7 @@ export function retryJob(config, jobId, host = defaultHost()) {
|
|
|
154
161
|
const detail = report.detail.endsWith("\n") || report.detail.length === 0 ? report.detail : `${report.detail}\n`;
|
|
155
162
|
return `retry ${jobId} ${job.repo}\n${clearedLine}${detail}`;
|
|
156
163
|
}
|
|
157
|
-
if (state === "dead" && observed.run) {
|
|
164
|
+
if ((state === "dead" || observed.stopReason) && observed.run) {
|
|
158
165
|
try {
|
|
159
166
|
spawnSync(process.execPath, [yakCliPath(), "cancel", observed.run.runId], {
|
|
160
167
|
cwd: observed.run.checkout,
|
|
@@ -174,6 +181,64 @@ export function retryJob(config, jobId, host = defaultHost()) {
|
|
|
174
181
|
return `retry ${report.jobId} ${report.repo}\n${clearedLine}${output}`;
|
|
175
182
|
});
|
|
176
183
|
}
|
|
184
|
+
function canManualRetry(state, job) {
|
|
185
|
+
if (state === "failed" || state === "dead")
|
|
186
|
+
return true;
|
|
187
|
+
return state === "needs-human" && Boolean(job.stopReason) && (job.run?.gates.length ?? 0) === 0;
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Send a gated job back for another round. The note is stored on a
|
|
191
|
+
* `gate.resolved` event (redacted, otherwise verbatim) and appended to the
|
|
192
|
+
* spec the next launch passes into the prompt.
|
|
193
|
+
*/
|
|
194
|
+
export function reviseJob(config, jobId, note, host = defaultHost()) {
|
|
195
|
+
const trimmed = note.trim();
|
|
196
|
+
if (!trimmed)
|
|
197
|
+
throw new Error("A revise note is required.");
|
|
198
|
+
if (trimmed.length > 8_000)
|
|
199
|
+
throw new Error("A revise note must be 8000 characters or fewer.");
|
|
200
|
+
return withTickLock(config.kilnDir, () => {
|
|
201
|
+
const world = observe(config, host);
|
|
202
|
+
const observed = world.observation.jobs.find((candidate) => candidate.jobId === jobId);
|
|
203
|
+
if (!observed)
|
|
204
|
+
throw new Error(`No job ${jobId}.`);
|
|
205
|
+
if (observed.error)
|
|
206
|
+
throw new Error(`Job ${jobId} is invalid: ${observed.error}`);
|
|
207
|
+
const state = jobState(observed, world.observation);
|
|
208
|
+
const gate = observed.run?.gates[0];
|
|
209
|
+
if (state !== "needs-human" || !observed.run || !gate) {
|
|
210
|
+
throw new Error(`Job ${jobId} is ${state}, not waiting on a gate.`);
|
|
211
|
+
}
|
|
212
|
+
const job = world.jobs.get(jobId);
|
|
213
|
+
if (!job)
|
|
214
|
+
throw new Error(`No job ${jobId}.`);
|
|
215
|
+
appendEvent(config.kilnDir, {
|
|
216
|
+
job_id: jobId,
|
|
217
|
+
run_id: observed.run.runId,
|
|
218
|
+
event: "gate.resolved",
|
|
219
|
+
step: gate.stepId,
|
|
220
|
+
properties: {
|
|
221
|
+
decision: "revise",
|
|
222
|
+
by: "human",
|
|
223
|
+
repo: job.repo,
|
|
224
|
+
note: trimmed,
|
|
225
|
+
},
|
|
226
|
+
});
|
|
227
|
+
resetAttemptLedger(config.kilnDir, jobId, observed.run.runId);
|
|
228
|
+
rmSync(breadcrumbPath(config.kilnDir, jobId), { force: true });
|
|
229
|
+
clearLaunchError(config.kilnDir, jobId);
|
|
230
|
+
const events = readEvents(config.kilnDir, { warn: () => { } });
|
|
231
|
+
const spend = spendSnapshot(events, config.dailyDollarCap);
|
|
232
|
+
const paused = pausedFrom(events);
|
|
233
|
+
const hold = holdDetail(paused, spend);
|
|
234
|
+
if (hold)
|
|
235
|
+
return `revised ${jobId}\n${hold}\n`;
|
|
236
|
+
const again = observe(config, host);
|
|
237
|
+
const report = launchJob(config, again, job, host);
|
|
238
|
+
const output = report.output.endsWith("\n") || report.output.length === 0 ? report.output : `${report.output}\n`;
|
|
239
|
+
return `revised ${jobId} ${report.repo}\n${output}`;
|
|
240
|
+
});
|
|
241
|
+
}
|
|
177
242
|
function shouldRedeliver(job) {
|
|
178
243
|
if (!isLanded(job))
|
|
179
244
|
return false;
|
|
@@ -185,9 +250,13 @@ function gateForChoice(gates, choice) {
|
|
|
185
250
|
return (gates.find((gate) => gate.choices?.includes(choice)) ??
|
|
186
251
|
gates.find((gate) => !gate.choices?.length));
|
|
187
252
|
}
|
|
188
|
-
function observe(config, host) {
|
|
253
|
+
function observe(config, host, now = new Date()) {
|
|
189
254
|
const loaded = loadJobs(config);
|
|
190
255
|
const runs = scanRuns(config);
|
|
256
|
+
const events = readEvents(config.kilnDir, { warn: () => { } });
|
|
257
|
+
const paused = pausedFrom(events);
|
|
258
|
+
const spend = spendSnapshot(events, config.dailyDollarCap, now);
|
|
259
|
+
const waiting = holdDetail(paused, spend);
|
|
191
260
|
const breadcrumbs = readBreadcrumbs(config.kilnDir);
|
|
192
261
|
const jobs = [];
|
|
193
262
|
const records = new Map();
|
|
@@ -234,10 +303,13 @@ function observe(config, host) {
|
|
|
234
303
|
const launchActivityMs = later(crumb ? fileMtimeMs(crumb.path) : undefined, logPath ? fileMtimeMs(logPath) : undefined);
|
|
235
304
|
const attemptCount = ledger?.attempts.length ?? 0;
|
|
236
305
|
const lastFailure = ledger?.attempts.at(-1);
|
|
237
|
-
const
|
|
306
|
+
const attemptCap = attemptsFor(config, entry.job?.repo ?? "");
|
|
307
|
+
const exhausted = attemptCount >= attemptCap &&
|
|
238
308
|
attemptCount > 0 &&
|
|
239
309
|
!launching &&
|
|
240
310
|
(picked.retired || !run || run.status === "dead" || run.status === "failed");
|
|
311
|
+
const eventStop = entry.job ? stopReasonFromEvents(events, entry.job.id) : undefined;
|
|
312
|
+
const stopReason = eventStop ?? (exhausted ? "attempt-cap" : undefined);
|
|
241
313
|
const specPath = entry.job ? resolveJobSpec(config.kilnDir, entry.job.spec) : undefined;
|
|
242
314
|
const findingsPath = entry.job?.workflow === INVESTIGATION_WORKFLOW
|
|
243
315
|
? existingFindingsPath(config.kilnDir, entry.job.id)
|
|
@@ -276,6 +348,8 @@ function observe(config, host) {
|
|
|
276
348
|
...(lastFailure ? { lastFailureReason: lastFailure.reason } : {}),
|
|
277
349
|
...(ledger?.nextRetryAt && !exhausted ? { nextRetryAt: ledger.nextRetryAt } : {}),
|
|
278
350
|
...(exhausted ? { retriesExhausted: true } : {}),
|
|
351
|
+
...(stopReason ? { stopReason } : {}),
|
|
352
|
+
attemptCap,
|
|
279
353
|
...(picked.retired ? { runRetired: true } : {}),
|
|
280
354
|
...(ledgerResult === "corrupt" ? { ledgerError: "attempt ledger is unreadable" } : {}),
|
|
281
355
|
...(entry.job ? { sandboxed: jobIsSandboxed(config, entry.job.id, entry.job.workflow) } : {}),
|
|
@@ -296,6 +370,9 @@ function observe(config, host) {
|
|
|
296
370
|
maxAttempts: config.maxAttempts,
|
|
297
371
|
jobs,
|
|
298
372
|
breadcrumbsToClear,
|
|
373
|
+
...(paused ? { paused: true } : {}),
|
|
374
|
+
...(spend.blocked ? { spendBlocked: true } : {}),
|
|
375
|
+
...(waiting ? { holdDetail: waiting } : {}),
|
|
299
376
|
},
|
|
300
377
|
jobs: records,
|
|
301
378
|
};
|
|
@@ -438,7 +515,8 @@ function specHandedToYak(config, world, job) {
|
|
|
438
515
|
}
|
|
439
516
|
sections.push({ title: dep.title, markdown: readFileSync(path, "utf8") });
|
|
440
517
|
}
|
|
441
|
-
|
|
518
|
+
const noted = withReviseNotes(appendInvestigationFindings(base, sections), reviseNotesSinceRun(readEvents(config.kilnDir, { warn: () => { } }), job.id));
|
|
519
|
+
return noted;
|
|
442
520
|
}
|
|
443
521
|
/** Remote name the stacked pull request targets. Falls back to the local yak ref. */
|
|
444
522
|
function upstreamBranch(world, job) {
|
|
@@ -550,6 +628,15 @@ function publishReadyFixes(config, host, world) {
|
|
|
550
628
|
const record = readDelivery(config.kilnDir, job.jobId);
|
|
551
629
|
const checkout = job.run.checkout;
|
|
552
630
|
const source = `yak/${job.run.runId}`;
|
|
631
|
+
const worktree = join(checkout, ".yak", "worktrees", job.run.runId);
|
|
632
|
+
if (existsSync(worktree)) {
|
|
633
|
+
ensureFixTrailers(worktree, {
|
|
634
|
+
jobId: job.jobId,
|
|
635
|
+
round: open.round,
|
|
636
|
+
promptsHash: hashForRun(config.kilnDir, job.jobId, job.run.runId),
|
|
637
|
+
title: job.title,
|
|
638
|
+
}, open.headSha);
|
|
639
|
+
}
|
|
553
640
|
if (!record?.branch) {
|
|
554
641
|
notes.push(`feedback ${job.jobId} failed: no pull request branch to publish`);
|
|
555
642
|
continue;
|
|
@@ -607,6 +694,14 @@ function publishReadyFixes(config, host, world) {
|
|
|
607
694
|
}
|
|
608
695
|
return notes;
|
|
609
696
|
}
|
|
697
|
+
function hashForRun(kilnDir, jobId, runId) {
|
|
698
|
+
for (const event of readEvents(kilnDir, { warn: () => { } })) {
|
|
699
|
+
if (event.job_id === jobId && event.run_id === runId && event.event === "run.started") {
|
|
700
|
+
return event.properties.prompts_hash;
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
return promptsHash("fix", kilnDir, jobId);
|
|
704
|
+
}
|
|
610
705
|
function isNonFastForward(message) {
|
|
611
706
|
return /non-fast-forward|\[rejected\]|failed to push some refs/i.test(message);
|
|
612
707
|
}
|
|
@@ -676,7 +771,7 @@ function watchPullRequests(config, world, host) {
|
|
|
676
771
|
continue;
|
|
677
772
|
}
|
|
678
773
|
job.feedbackPlan = planFeedback({
|
|
679
|
-
maxFixRounds: config.
|
|
774
|
+
maxFixRounds: fixRoundsFor(config, job.repo),
|
|
680
775
|
ledger,
|
|
681
776
|
snapshot,
|
|
682
777
|
});
|
|
@@ -769,7 +864,7 @@ function startFix(config, world, job, host, now, sandboxFacts) {
|
|
|
769
864
|
writeBreadcrumb(breadcrumbPath(config.kilnDir, job.id), { jobId: job.id, pid: 0, logPath });
|
|
770
865
|
return { jobId: job.id, repo: job.repo, kind: "fix", output: message };
|
|
771
866
|
}
|
|
772
|
-
const report = launchFixRun(config, job, host, repo.path, headSha, specText, sandboxFacts);
|
|
867
|
+
const report = launchFixRun(config, job, host, repo.path, headSha, specText, round, sandboxFacts);
|
|
773
868
|
return { ...report, output: `round ${round}\n${report.output}` };
|
|
774
869
|
}
|
|
775
870
|
/** Relaunch the open fix round. Does not append a round. Failures become a dead launch so retry can see them. */
|
|
@@ -790,7 +885,7 @@ function launchOpenFix(config, world, job, host, checkout, open, sandboxFacts) {
|
|
|
790
885
|
writeBreadcrumb(breadcrumbPath(config.kilnDir, job.id), { jobId: job.id, pid: 0, logPath });
|
|
791
886
|
return { jobId: job.id, repo: job.repo, kind: "fix", output: message };
|
|
792
887
|
}
|
|
793
|
-
return launchFixRun(config, job, host, checkout, headSha, specText, sandboxFacts);
|
|
888
|
+
return launchFixRun(config, job, host, checkout, headSha, specText, open.round, sandboxFacts);
|
|
794
889
|
}
|
|
795
890
|
function resolveCommit(host, checkout, sha) {
|
|
796
891
|
try {
|
|
@@ -802,7 +897,7 @@ function resolveCommit(host, checkout, sha) {
|
|
|
802
897
|
}
|
|
803
898
|
}
|
|
804
899
|
/** `workflows/fix.yaml` on `headSha`. Commits on `yak/<run-id>`. Does not push. */
|
|
805
|
-
function launchFixRun(config, job, host, checkout, headSha, specText, sandboxFacts) {
|
|
900
|
+
function launchFixRun(config, job, host, checkout, headSha, specText, round, sandboxFacts) {
|
|
806
901
|
const refused = claudeCredentialPreflight(process.env, config.adapter);
|
|
807
902
|
if (refused)
|
|
808
903
|
return { jobId: job.id, repo: job.repo, kind: "fix", output: refused };
|
|
@@ -814,6 +909,7 @@ function launchFixRun(config, job, host, checkout, headSha, specText, sandboxFac
|
|
|
814
909
|
const blocked = commitAgentSandbox(config, job.id, "fix", workflow, logPath, sandboxFacts);
|
|
815
910
|
if (blocked)
|
|
816
911
|
return { jobId: job.id, repo: job.repo, kind: "fix", output: blocked };
|
|
912
|
+
const hash = promptsHash("fix", config.kilnDir, job.id);
|
|
817
913
|
ensureCheckoutExcludes(checkout);
|
|
818
914
|
const input = workflowInputs(sourceWorkflow, {
|
|
819
915
|
title: job.title,
|
|
@@ -826,6 +922,9 @@ function launchFixRun(config, job, host, checkout, headSha, specText, sandboxFac
|
|
|
826
922
|
...process.env,
|
|
827
923
|
...(repo?.test ? { KILN_TEST: repo.test } : {}),
|
|
828
924
|
KILN_TASK_TITLE: job.title,
|
|
925
|
+
KILN_JOB: job.id,
|
|
926
|
+
KILN_ROUND: String(round),
|
|
927
|
+
KILN_PROMPTS_HASH: hash,
|
|
829
928
|
KILN_VERIFY_WRAPPER: join(packageRoot(), "dist", "truncate-output.js"),
|
|
830
929
|
};
|
|
831
930
|
return spawnLogged(config, host, {
|
|
@@ -1056,6 +1155,7 @@ function scanRuns(config) {
|
|
|
1056
1155
|
const stepPlan = readRunStepPlan(runDir);
|
|
1057
1156
|
const stepEvents = stepEventsFrom(events);
|
|
1058
1157
|
const sandboxError = sandboxFailureDetail(stepEvents);
|
|
1158
|
+
const cost = costFromJournal(events);
|
|
1059
1159
|
const pid = livePid(events);
|
|
1060
1160
|
const activityMs = eventActivityMs(events) ?? fileMtimeMs(join(runDir, "journal.jsonl"));
|
|
1061
1161
|
const finishedReason = finished && typeof finished.reason === "string" ? finished.reason : undefined;
|
|
@@ -1074,6 +1174,7 @@ function scanRuns(config) {
|
|
|
1074
1174
|
...(activityMs !== undefined ? { activityMs } : {}),
|
|
1075
1175
|
...(finishedReason ? { finishedReason } : {}),
|
|
1076
1176
|
...(sandboxError ? { sandboxError } : {}),
|
|
1177
|
+
cost,
|
|
1077
1178
|
};
|
|
1078
1179
|
const list = byTag.get(started.tag) ?? [];
|
|
1079
1180
|
list.push(run);
|
package/dist/events.js
CHANGED
|
@@ -9,7 +9,8 @@ import { redactDeep } from "./redact.js";
|
|
|
9
9
|
* parses it. A write failure is reported and does not escape to the caller.
|
|
10
10
|
*
|
|
11
11
|
* `run_id` and `step` are null when the moment has none. `job_id` is null
|
|
12
|
-
*
|
|
12
|
+
* for `config.reloaded`, `kiln.paused`, `kiln.resumed`, and `budget.blocked`,
|
|
13
|
+
* which are not a job.
|
|
13
14
|
*/
|
|
14
15
|
export const EVENT_NAMES = [
|
|
15
16
|
"job.added",
|
|
@@ -27,6 +28,9 @@ export const EVENT_NAMES = [
|
|
|
27
28
|
"worktree.kept",
|
|
28
29
|
"config.reloaded",
|
|
29
30
|
"needs-human",
|
|
31
|
+
"kiln.paused",
|
|
32
|
+
"kiln.resumed",
|
|
33
|
+
"budget.blocked",
|
|
30
34
|
];
|
|
31
35
|
const ISO_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/;
|
|
32
36
|
const once = z.string().min(1).optional();
|
|
@@ -80,14 +84,24 @@ const GateResolved = z.object({
|
|
|
80
84
|
edit: z.string().min(1).optional(),
|
|
81
85
|
once,
|
|
82
86
|
});
|
|
87
|
+
const costFields = {
|
|
88
|
+
/** Sum of yak `budget.consumed` tokens. Null when the journal did not report any. */
|
|
89
|
+
tokens: z.number().nonnegative().nullable().optional(),
|
|
90
|
+
/** Sum of reported dollars. Null when the journal did not report any. Never invented. */
|
|
91
|
+
usd: z.number().nonnegative().nullable().optional(),
|
|
92
|
+
};
|
|
83
93
|
const RunFailed = z.object({
|
|
84
94
|
reason: z.string().min(1).optional(),
|
|
85
95
|
detail: z.string().min(1).optional(),
|
|
86
96
|
status: z.string().min(1).optional(),
|
|
97
|
+
class: z.enum(["transient", "deterministic", "budget"]).optional(),
|
|
98
|
+
fingerprint: z.string().min(1).optional(),
|
|
99
|
+
...costFields,
|
|
87
100
|
once,
|
|
88
101
|
});
|
|
89
102
|
const RunFinished = z.object({
|
|
90
103
|
status: z.literal("ok"),
|
|
104
|
+
...costFields,
|
|
91
105
|
once,
|
|
92
106
|
});
|
|
93
107
|
const PrOpened = z.object({
|
|
@@ -110,6 +124,13 @@ const NeedsHuman = z.object({
|
|
|
110
124
|
reason: z.string().min(1),
|
|
111
125
|
once,
|
|
112
126
|
});
|
|
127
|
+
const FactoryToggle = z.object({ once });
|
|
128
|
+
const BudgetBlocked = z.object({
|
|
129
|
+
day: z.string().min(1),
|
|
130
|
+
usd: z.number().nonnegative(),
|
|
131
|
+
cap: z.number().nonnegative(),
|
|
132
|
+
once,
|
|
133
|
+
});
|
|
113
134
|
const envelope = {
|
|
114
135
|
id: z.string().min(1),
|
|
115
136
|
ts: z.string().regex(ISO_UTC, "ts must be ISO-8601 UTC"),
|
|
@@ -137,6 +158,24 @@ export const EventSchema = z.discriminatedUnion("event", [
|
|
|
137
158
|
properties: ConfigReloaded,
|
|
138
159
|
}),
|
|
139
160
|
z.object({ ...envelope, event: z.literal("needs-human"), job_id: z.string().min(1), properties: NeedsHuman }),
|
|
161
|
+
z.object({
|
|
162
|
+
...envelope,
|
|
163
|
+
event: z.literal("kiln.paused"),
|
|
164
|
+
job_id: z.string().min(1).nullable(),
|
|
165
|
+
properties: FactoryToggle,
|
|
166
|
+
}),
|
|
167
|
+
z.object({
|
|
168
|
+
...envelope,
|
|
169
|
+
event: z.literal("kiln.resumed"),
|
|
170
|
+
job_id: z.string().min(1).nullable(),
|
|
171
|
+
properties: FactoryToggle,
|
|
172
|
+
}),
|
|
173
|
+
z.object({
|
|
174
|
+
...envelope,
|
|
175
|
+
event: z.literal("budget.blocked"),
|
|
176
|
+
job_id: z.string().min(1).nullable(),
|
|
177
|
+
properties: BudgetBlocked,
|
|
178
|
+
}),
|
|
140
179
|
]);
|
|
141
180
|
const LOCK_WAIT_MS = 5_000;
|
|
142
181
|
const pauseBuf = new Int32Array(new SharedArrayBuffer(4));
|
package/dist/failure.js
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { redact } from "./redact.js";
|
|
3
|
+
/**
|
|
4
|
+
* One failed run is transient, deterministic, or budget.
|
|
5
|
+
* Transient may retry with backoff inside the attempt cap.
|
|
6
|
+
* Deterministic does not. A cap hit is budget and stops in Needs human.
|
|
7
|
+
*/
|
|
8
|
+
export const FAILURE_CLASSES = ["transient", "deterministic", "budget"];
|
|
9
|
+
const TRANSIENT = [
|
|
10
|
+
/\b429\b/,
|
|
11
|
+
/rate limit/i,
|
|
12
|
+
/too many requests/i,
|
|
13
|
+
/\b5\d\d\b/,
|
|
14
|
+
/internal server error/i,
|
|
15
|
+
/bad gateway/i,
|
|
16
|
+
/service unavailable/i,
|
|
17
|
+
/gateway timeout/i,
|
|
18
|
+
/ECONNRESET|ECONNREFUSED|ENOTFOUND|ETIMEDOUT|EAI_AGAIN|EPIPE/i,
|
|
19
|
+
/socket hang up/i,
|
|
20
|
+
/network (?:error|unreachable|is down)/i,
|
|
21
|
+
/fetch failed/i,
|
|
22
|
+
/connection (?:reset|refused|timed out|closed)/i,
|
|
23
|
+
/container never started/i,
|
|
24
|
+
/failed to start (?:the )?(?:container|sandbox)/i,
|
|
25
|
+
/docker daemon/i,
|
|
26
|
+
/cannot connect to the docker/i,
|
|
27
|
+
/OCI runtime/i,
|
|
28
|
+
/error creating container/i,
|
|
29
|
+
/sandbox (?:start|flake)/i,
|
|
30
|
+
];
|
|
31
|
+
const DETERMINISTIC = [
|
|
32
|
+
/\berror TS\d+\b/,
|
|
33
|
+
/type error/i,
|
|
34
|
+
/cannot find name/i,
|
|
35
|
+
/is not assignable/i,
|
|
36
|
+
/syntax error/i,
|
|
37
|
+
/\bAssertionError\b/,
|
|
38
|
+
/\b(?:tests?|assertion) failed\b/i,
|
|
39
|
+
/\bfailing tests?\b/i,
|
|
40
|
+
/\bbuild failed\b/i,
|
|
41
|
+
/\bcompilation failed\b/i,
|
|
42
|
+
/\bcompile error\b/i,
|
|
43
|
+
/npm ERR!/i,
|
|
44
|
+
];
|
|
45
|
+
/** A container or sandbox that did not start. Those are transient. */
|
|
46
|
+
export function isSandboxStartFlake(text) {
|
|
47
|
+
return /container never started|failed to start (?:the )?(?:container|sandbox)|docker daemon|cannot connect to the docker|OCI runtime|error creating container|sandbox (?:start|flake)/i.test(text);
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Strip timestamps, temp and worktree paths, hex ids, and line numbers so
|
|
51
|
+
* the same failure hashes the same way across runs.
|
|
52
|
+
*/
|
|
53
|
+
export function normalizeFailureText(text) {
|
|
54
|
+
let out = redact(text).replace(/\r\n/g, "\n").trim();
|
|
55
|
+
out = out.replace(/\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?/g, "<ts>");
|
|
56
|
+
out = out.replace(/\b\d{2}:\d{2}:\d{2}(?:\.\d+)?\b/g, "<ts>");
|
|
57
|
+
out = out.replace(/(?:\/private)?\/(?:tmp|var\/folders)\/[^\s'")\]:]+/g, "<tmp>");
|
|
58
|
+
out = out.replace(/[^\s'")\]]*[/\\]\.yak[/\\]worktrees[/\\][^\s'")\]:]+/g, "<worktree>");
|
|
59
|
+
out = out.replace(/[^\s'")\]]+[/\\]worktrees[/\\][^\s'")\]:]+/g, "<worktree>");
|
|
60
|
+
out = out.replace(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi, "<id>");
|
|
61
|
+
out = out.replace(/\b[0-9a-f]{8,}\b/gi, "<id>");
|
|
62
|
+
out = out.replace(/:(\d+)(?::\d+)?\b/g, ":<line>");
|
|
63
|
+
out = out.replace(/\bline\s+\d+\b/gi, "line <line>");
|
|
64
|
+
out = out.replace(/[ \t]+/g, " ");
|
|
65
|
+
out = out.replace(/\n{3,}/g, "\n\n");
|
|
66
|
+
return out.toLowerCase();
|
|
67
|
+
}
|
|
68
|
+
/** Stable id for one normalized failure. */
|
|
69
|
+
export function failureFingerprint(text) {
|
|
70
|
+
return createHash("sha256").update(normalizeFailureText(text)).digest("hex");
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Transient when the text is a network, rate-limit, 5xx, or start flake.
|
|
74
|
+
* A test, build, or type error is deterministic and wins over a transient
|
|
75
|
+
* pattern in the same text. Anything else is deterministic.
|
|
76
|
+
*/
|
|
77
|
+
export function classifyFailureText(text) {
|
|
78
|
+
if (DETERMINISTIC.some((pattern) => pattern.test(text)))
|
|
79
|
+
return "deterministic";
|
|
80
|
+
if (TRANSIENT.some((pattern) => pattern.test(text)))
|
|
81
|
+
return "transient";
|
|
82
|
+
return "deterministic";
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* The only decision for what a recorded failure does next.
|
|
86
|
+
* Attempt cap wins over the fingerprint breaker when both trip on the same run.
|
|
87
|
+
*/
|
|
88
|
+
export function decideFailure(input) {
|
|
89
|
+
const fingerprint = failureFingerprint(input.text);
|
|
90
|
+
const deterministicHit = DETERMINISTIC.some((pattern) => pattern.test(input.text));
|
|
91
|
+
const transientHit = TRANSIENT.some((pattern) => pattern.test(input.text));
|
|
92
|
+
// Stall, crash, orphan, and a bare non-zero exit stay transient unless the
|
|
93
|
+
// text is a test, build, or type error, or the failure is blocked.
|
|
94
|
+
let failureClass = deterministicHit
|
|
95
|
+
? "deterministic"
|
|
96
|
+
: transientHit || (input.operational && !input.blocked)
|
|
97
|
+
? "transient"
|
|
98
|
+
: "deterministic";
|
|
99
|
+
let retry = failureClass === "transient" && !input.blocked;
|
|
100
|
+
let needsHuman;
|
|
101
|
+
if (input.attempt >= input.maxAttempts) {
|
|
102
|
+
failureClass = "budget";
|
|
103
|
+
retry = false;
|
|
104
|
+
needsHuman = "attempt-cap";
|
|
105
|
+
}
|
|
106
|
+
else if (input.sameFingerprint >= input.maxSameFingerprint) {
|
|
107
|
+
retry = false;
|
|
108
|
+
needsHuman = "fingerprint";
|
|
109
|
+
}
|
|
110
|
+
else if (input.blocked || failureClass === "deterministic") {
|
|
111
|
+
retry = false;
|
|
112
|
+
}
|
|
113
|
+
return { class: failureClass, fingerprint, retry, ...(needsHuman ? { needsHuman } : {}) };
|
|
114
|
+
}
|
|
115
|
+
/** Sum yak `budget.consumed` lines. Missing numbers stay null. Never invents a figure. */
|
|
116
|
+
export function costFromJournal(events) {
|
|
117
|
+
let tokens = 0;
|
|
118
|
+
let usd = 0;
|
|
119
|
+
let sawTokens = false;
|
|
120
|
+
let sawUsd = false;
|
|
121
|
+
for (const event of events) {
|
|
122
|
+
if (!event || typeof event !== "object")
|
|
123
|
+
continue;
|
|
124
|
+
const record = event;
|
|
125
|
+
if (record.t !== "budget.consumed")
|
|
126
|
+
continue;
|
|
127
|
+
if (typeof record.tokens === "number" && Number.isFinite(record.tokens) && record.tokens >= 0) {
|
|
128
|
+
tokens += record.tokens;
|
|
129
|
+
sawTokens = true;
|
|
130
|
+
}
|
|
131
|
+
if (typeof record.usd === "number" && Number.isFinite(record.usd) && record.usd >= 0) {
|
|
132
|
+
usd += record.usd;
|
|
133
|
+
sawUsd = true;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return {
|
|
137
|
+
tokens: sawTokens ? tokens : null,
|
|
138
|
+
usd: sawUsd ? usd : null,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
export function fixTrailerLines(input) {
|
|
3
|
+
return [`Kiln-Job: ${input.jobId}`, `Kiln-Round: ${input.round}`, `Kiln-Prompts-Hash: ${input.promptsHash}`];
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* The fix workflow `land` step. `workflows/fix.yaml` keeps the same command.
|
|
7
|
+
* Launch sets `KILN_JOB`, `KILN_ROUND`, and `KILN_PROMPTS_HASH`.
|
|
8
|
+
*/
|
|
9
|
+
export const FIX_LAND_COMMAND = 'git add -A && git diff --cached --quiet || git commit -m "kiln: $KILN_TASK_TITLE" --trailer "Kiln-Job: $KILN_JOB" --trailer "Kiln-Round: $KILN_ROUND" --trailer "Kiln-Prompts-Hash: $KILN_PROMPTS_HASH"';
|
|
10
|
+
/**
|
|
11
|
+
* Make the tip of `dir` carry the fix-round trailers.
|
|
12
|
+
* A dirty tree is committed. A tip that already moved and lacks the trailers
|
|
13
|
+
* is amended. A clean tree still on `baseSha` is left alone.
|
|
14
|
+
*/
|
|
15
|
+
export function ensureFixTrailers(dir, input, baseSha) {
|
|
16
|
+
if (!hasTrailers(dir, input)) {
|
|
17
|
+
if (isDirty(dir)) {
|
|
18
|
+
commit(dir, input, false);
|
|
19
|
+
return "committed";
|
|
20
|
+
}
|
|
21
|
+
const head = revParse(dir, "HEAD");
|
|
22
|
+
if (!head || (baseSha && head === baseSha))
|
|
23
|
+
return "clean";
|
|
24
|
+
commit(dir, input, true);
|
|
25
|
+
return "amended";
|
|
26
|
+
}
|
|
27
|
+
return "present";
|
|
28
|
+
}
|
|
29
|
+
export function commitMessageHasTrailers(message, input) {
|
|
30
|
+
return fixTrailerLines(input).every((line) => message.split("\n").some((entry) => entry.trim() === line));
|
|
31
|
+
}
|
|
32
|
+
function hasTrailers(dir, input) {
|
|
33
|
+
const message = git(dir, ["log", "-1", "--format=%B"]);
|
|
34
|
+
if (message.status !== 0)
|
|
35
|
+
return false;
|
|
36
|
+
return commitMessageHasTrailers(message.stdout, input);
|
|
37
|
+
}
|
|
38
|
+
function isDirty(dir) {
|
|
39
|
+
const status = git(dir, ["status", "--porcelain"]);
|
|
40
|
+
if (status.status !== 0)
|
|
41
|
+
return false;
|
|
42
|
+
return status.stdout.trim().length > 0;
|
|
43
|
+
}
|
|
44
|
+
function revParse(dir, ref) {
|
|
45
|
+
const result = git(dir, ["rev-parse", ref]);
|
|
46
|
+
if (result.status !== 0)
|
|
47
|
+
return undefined;
|
|
48
|
+
const sha = result.stdout.trim();
|
|
49
|
+
return sha || undefined;
|
|
50
|
+
}
|
|
51
|
+
function commit(dir, input, amend) {
|
|
52
|
+
if (!amend) {
|
|
53
|
+
const added = git(dir, ["add", "-A"]);
|
|
54
|
+
if (added.status !== 0)
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
const args = ["commit"];
|
|
58
|
+
if (amend)
|
|
59
|
+
args.push("--amend", "--no-edit");
|
|
60
|
+
else
|
|
61
|
+
args.push("-m", `kiln: ${input.title}`);
|
|
62
|
+
for (const line of fixTrailerLines(input))
|
|
63
|
+
args.push("--trailer", line);
|
|
64
|
+
git(dir, args);
|
|
65
|
+
}
|
|
66
|
+
function git(cwd, args) {
|
|
67
|
+
const result = spawnSync("git", args, {
|
|
68
|
+
cwd,
|
|
69
|
+
encoding: "utf8",
|
|
70
|
+
env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
|
|
71
|
+
});
|
|
72
|
+
return { status: result.status ?? 1, stdout: result.stdout ?? "" };
|
|
73
|
+
}
|
package/dist/ledger.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { readEvents } from "./events.js";
|
|
2
2
|
import { redact } from "./redact.js";
|
|
3
|
+
import { formatSpend, spendSnapshot } from "./rails.js";
|
|
3
4
|
/**
|
|
4
5
|
* Interventions are derived from the event log. Nothing is stored beside it.
|
|
5
6
|
* A human had to act when a gate was rejected, a job was retried by hand,
|
|
@@ -64,9 +65,14 @@ export function ledgerRows(events, options = {}) {
|
|
|
64
65
|
return rows;
|
|
65
66
|
}
|
|
66
67
|
export function renderLedger(events, options = {}) {
|
|
68
|
+
const spend = options.spend ??
|
|
69
|
+
(options.dailyDollarCap !== undefined || options.now
|
|
70
|
+
? spendSnapshot(events, options.dailyDollarCap, options.now ?? new Date())
|
|
71
|
+
: undefined);
|
|
72
|
+
const head = spend ? `${formatSpend(spend)}\n` : "";
|
|
67
73
|
const rows = ledgerRows(events, options);
|
|
68
74
|
if (rows.length === 0)
|
|
69
|
-
return redact(
|
|
75
|
+
return redact(`${head}0 interventions\n`);
|
|
70
76
|
const lines = [];
|
|
71
77
|
let grand = 0;
|
|
72
78
|
let index = 0;
|
|
@@ -90,10 +96,12 @@ export function renderLedger(events, options = {}) {
|
|
|
90
96
|
}
|
|
91
97
|
lines.push("");
|
|
92
98
|
lines.push(`${grand} ${grand === 1 ? "intervention" : "interventions"}`);
|
|
93
|
-
return redact(`${lines.join("\n")}\n`);
|
|
99
|
+
return redact(`${head}${lines.join("\n")}\n`);
|
|
94
100
|
}
|
|
95
101
|
export function renderLedgerFile(kilnDir, options = {}) {
|
|
96
|
-
|
|
102
|
+
const events = readEvents(kilnDir);
|
|
103
|
+
const spend = spendSnapshot(events, options.dailyDollarCap, options.now ?? new Date());
|
|
104
|
+
return renderLedger(events, { ...options, spend });
|
|
97
105
|
}
|
|
98
106
|
function reasonsFor(event) {
|
|
99
107
|
const reasons = [];
|