@lchase/kiln 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/assets/serve.css +13 -0
- package/assets/serve.js +49 -3
- package/dist/attention.js +13 -4
- package/dist/cli.js +63 -6
- package/dist/config.js +25 -1
- package/dist/driver.js +117 -16
- package/dist/events.js +40 -1
- package/dist/failure.js +140 -0
- package/dist/fix-commit.js +73 -0
- package/dist/ledger.js +11 -3
- package/dist/project.js +20 -4
- package/dist/rails.js +164 -0
- package/dist/reconcile.js +29 -3
- package/dist/reliability.js +53 -25
- package/dist/report.js +19 -5
- package/dist/schemas.js +35 -5
- package/dist/serve-page.js +21 -1
- package/dist/serve.js +70 -3
- package/dist/settings.js +9 -1
- package/package.json +2 -2
- package/workflows/fix.yaml +1 -1
package/dist/project.js
CHANGED
|
@@ -3,6 +3,7 @@ import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
|
3
3
|
import { createRequire } from "node:module";
|
|
4
4
|
import { basename, join, relative, sep } from "node:path";
|
|
5
5
|
import { appendEvent } from "./events.js";
|
|
6
|
+
import { costFromJournal, decideFailure } from "./failure.js";
|
|
6
7
|
import { jobBasename, packageRoot } from "./paths.js";
|
|
7
8
|
/**
|
|
8
9
|
* Project a yak journal into the kiln event log. Called when a run is
|
|
@@ -186,25 +187,40 @@ function recordGateResolved(input, event) {
|
|
|
186
187
|
});
|
|
187
188
|
}
|
|
188
189
|
function recordRunFinished(input, event) {
|
|
190
|
+
const cost = costFromJournal(input.events);
|
|
189
191
|
if (event.status === "ok") {
|
|
190
192
|
appendEvent(input.kilnDir, {
|
|
191
193
|
job_id: input.jobId,
|
|
192
194
|
run_id: input.runId,
|
|
193
195
|
event: "run.finished",
|
|
194
|
-
properties: { status: "ok", once: "run.finished" },
|
|
196
|
+
properties: { status: "ok", tokens: cost.tokens, usd: cost.usd, once: "run.finished" },
|
|
195
197
|
});
|
|
196
198
|
return;
|
|
197
199
|
}
|
|
198
|
-
|
|
200
|
+
// A normal failure is classified in recoverRuns, which owns the retry decision.
|
|
201
|
+
// A cancelled run is not recovered, so the class is recorded here.
|
|
202
|
+
if (event.status !== "failed" || event.reason !== "cancelled")
|
|
199
203
|
return;
|
|
200
|
-
const
|
|
204
|
+
const decision = decideFailure({
|
|
205
|
+
text: "cancelled",
|
|
206
|
+
operational: false,
|
|
207
|
+
blocked: true,
|
|
208
|
+
attempt: 1,
|
|
209
|
+
maxAttempts: Number.MAX_SAFE_INTEGER,
|
|
210
|
+
sameFingerprint: 1,
|
|
211
|
+
maxSameFingerprint: Number.MAX_SAFE_INTEGER,
|
|
212
|
+
});
|
|
201
213
|
appendEvent(input.kilnDir, {
|
|
202
214
|
job_id: input.jobId,
|
|
203
215
|
run_id: input.runId,
|
|
204
216
|
event: "run.failed",
|
|
205
217
|
properties: {
|
|
206
218
|
status: "failed",
|
|
207
|
-
|
|
219
|
+
reason: "cancelled",
|
|
220
|
+
class: decision.class,
|
|
221
|
+
fingerprint: decision.fingerprint,
|
|
222
|
+
tokens: cost.tokens,
|
|
223
|
+
usd: cost.usd,
|
|
208
224
|
once: `run.failed:${input.runId}`,
|
|
209
225
|
},
|
|
210
226
|
});
|
package/dist/rails.js
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { appendEvent, readEvents } from "./events.js";
|
|
2
|
+
export function localDay(now) {
|
|
3
|
+
const year = now.getFullYear();
|
|
4
|
+
const month = String(now.getMonth() + 1).padStart(2, "0");
|
|
5
|
+
const day = String(now.getDate()).padStart(2, "0");
|
|
6
|
+
return `${year}-${month}-${day}`;
|
|
7
|
+
}
|
|
8
|
+
export function isPaused(kilnDir) {
|
|
9
|
+
return pausedFrom(readEvents(kilnDir, { warn: () => { } }));
|
|
10
|
+
}
|
|
11
|
+
export function pausedFrom(events) {
|
|
12
|
+
let paused = false;
|
|
13
|
+
for (const event of events) {
|
|
14
|
+
if (event.event === "kiln.paused")
|
|
15
|
+
paused = true;
|
|
16
|
+
else if (event.event === "kiln.resumed")
|
|
17
|
+
paused = false;
|
|
18
|
+
}
|
|
19
|
+
return paused;
|
|
20
|
+
}
|
|
21
|
+
/** Append a pause event when the factory is running. Already paused is a no-op. */
|
|
22
|
+
export function pauseFactory(kilnDir, now = new Date()) {
|
|
23
|
+
if (pausedFrom(readEvents(kilnDir, { warn: () => { } })))
|
|
24
|
+
return false;
|
|
25
|
+
appendEvent(kilnDir, {
|
|
26
|
+
job_id: null,
|
|
27
|
+
event: "kiln.paused",
|
|
28
|
+
properties: {},
|
|
29
|
+
ts: now.toISOString(),
|
|
30
|
+
});
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
/** Append a resume event when the factory is paused. Already running is a no-op. */
|
|
34
|
+
export function resumeFactory(kilnDir, now = new Date()) {
|
|
35
|
+
if (!pausedFrom(readEvents(kilnDir, { warn: () => { } })))
|
|
36
|
+
return false;
|
|
37
|
+
appendEvent(kilnDir, {
|
|
38
|
+
job_id: null,
|
|
39
|
+
event: "kiln.resumed",
|
|
40
|
+
properties: {},
|
|
41
|
+
ts: now.toISOString(),
|
|
42
|
+
});
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
export function spendSnapshot(events, cap, now = new Date()) {
|
|
46
|
+
const day = localDay(now);
|
|
47
|
+
let usd = 0;
|
|
48
|
+
for (const event of events) {
|
|
49
|
+
if (localDay(new Date(event.ts)) !== day)
|
|
50
|
+
continue;
|
|
51
|
+
const amount = dollarCost(event);
|
|
52
|
+
if (amount === undefined)
|
|
53
|
+
continue;
|
|
54
|
+
usd += amount;
|
|
55
|
+
}
|
|
56
|
+
const limit = cap === undefined ? null : cap;
|
|
57
|
+
return {
|
|
58
|
+
day,
|
|
59
|
+
usd,
|
|
60
|
+
cap: limit,
|
|
61
|
+
blocked: limit !== null && usd >= limit,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
export function spendFor(kilnDir, cap, now = new Date()) {
|
|
65
|
+
return spendSnapshot(readEvents(kilnDir, { warn: () => { } }), cap, now);
|
|
66
|
+
}
|
|
67
|
+
/** One event per local day when the dollar cap is blocking new runs. */
|
|
68
|
+
export function recordDailyCap(kilnDir, spend, now = new Date()) {
|
|
69
|
+
if (!spend.blocked || spend.cap === null)
|
|
70
|
+
return;
|
|
71
|
+
appendEvent(kilnDir, {
|
|
72
|
+
job_id: null,
|
|
73
|
+
event: "budget.blocked",
|
|
74
|
+
properties: {
|
|
75
|
+
day: spend.day,
|
|
76
|
+
usd: spend.usd,
|
|
77
|
+
cap: spend.cap,
|
|
78
|
+
once: `daily-cap:${spend.day}`,
|
|
79
|
+
},
|
|
80
|
+
ts: now.toISOString(),
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
export function formatSpend(spend) {
|
|
84
|
+
const cap = spend.cap === null ? "no cap" : `$${spend.cap.toFixed(2)}`;
|
|
85
|
+
return `today ${spend.day} $${spend.usd.toFixed(2)} / ${cap}`;
|
|
86
|
+
}
|
|
87
|
+
export function holdDetail(paused, spend) {
|
|
88
|
+
if (paused)
|
|
89
|
+
return "Paused. New runs wait until kiln resume.";
|
|
90
|
+
if (spend.blocked && spend.cap !== null) {
|
|
91
|
+
return `Daily cap reached ($${spend.usd.toFixed(2)} of $${spend.cap.toFixed(2)}). New runs wait until the local day rolls over.`;
|
|
92
|
+
}
|
|
93
|
+
return undefined;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Revise notes written since the latest `run.started` for this job.
|
|
97
|
+
* The next launch appends them to the spec, which is the next prompt's input.
|
|
98
|
+
*/
|
|
99
|
+
export function reviseNotesSinceRun(events, jobId) {
|
|
100
|
+
const notes = [];
|
|
101
|
+
for (const event of events) {
|
|
102
|
+
if (event.job_id !== jobId)
|
|
103
|
+
continue;
|
|
104
|
+
if (event.event === "run.started") {
|
|
105
|
+
notes.length = 0;
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
if (event.event === "gate.resolved" && event.properties.decision === "revise" && event.properties.note) {
|
|
109
|
+
notes.push(event.properties.note);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return notes;
|
|
113
|
+
}
|
|
114
|
+
export function withReviseNotes(spec, notes) {
|
|
115
|
+
if (notes.length === 0)
|
|
116
|
+
return spec;
|
|
117
|
+
return [
|
|
118
|
+
spec.trimEnd(),
|
|
119
|
+
"",
|
|
120
|
+
"## Human note",
|
|
121
|
+
"",
|
|
122
|
+
"Apply every note below on this round. A person typed it at the gate.",
|
|
123
|
+
"",
|
|
124
|
+
notes.join("\n\n"),
|
|
125
|
+
"",
|
|
126
|
+
].join("\n");
|
|
127
|
+
}
|
|
128
|
+
/** Fingerprint or attempt-cap stop that a later manual retry has not cleared. */
|
|
129
|
+
export function stopReasonFromEvents(events, jobId) {
|
|
130
|
+
let reason;
|
|
131
|
+
for (const event of events) {
|
|
132
|
+
if (event.job_id !== jobId)
|
|
133
|
+
continue;
|
|
134
|
+
if (event.event === "job.retried") {
|
|
135
|
+
reason = undefined;
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
if (event.event !== "needs-human")
|
|
139
|
+
continue;
|
|
140
|
+
if (event.properties.reason === "fingerprint" || event.properties.reason === "attempt-cap") {
|
|
141
|
+
reason = event.properties.reason;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return reason;
|
|
145
|
+
}
|
|
146
|
+
/** How many `run.failed` lines for this job already carry `fingerprint`. */
|
|
147
|
+
export function fingerprintCount(events, jobId, fingerprint) {
|
|
148
|
+
let count = 0;
|
|
149
|
+
for (const event of events) {
|
|
150
|
+
if (event.job_id !== jobId || event.event !== "run.failed")
|
|
151
|
+
continue;
|
|
152
|
+
if (event.properties.fingerprint === fingerprint)
|
|
153
|
+
count += 1;
|
|
154
|
+
}
|
|
155
|
+
return count;
|
|
156
|
+
}
|
|
157
|
+
function dollarCost(event) {
|
|
158
|
+
if (event.event !== "run.finished" && event.event !== "run.failed")
|
|
159
|
+
return undefined;
|
|
160
|
+
const usd = event.properties.usd;
|
|
161
|
+
if (typeof usd !== "number" || !Number.isFinite(usd) || usd < 0)
|
|
162
|
+
return undefined;
|
|
163
|
+
return usd;
|
|
164
|
+
}
|
package/dist/reconcile.js
CHANGED
|
@@ -10,10 +10,32 @@ export function inFlightCount(observation) {
|
|
|
10
10
|
function inFlight(job, observation) {
|
|
11
11
|
if (job.error || job.feedbackExhausted || job.feedbackAmbiguous)
|
|
12
12
|
return false;
|
|
13
|
+
// A cap or fingerprint stop is waiting on a person. It does not hold a slot.
|
|
14
|
+
if (job.retriesExhausted || job.stopReason)
|
|
15
|
+
return false;
|
|
13
16
|
const state = jobState(job, observation);
|
|
14
17
|
return state === "running" || state === "needs-human" || state === "approved";
|
|
15
18
|
}
|
|
16
19
|
/**
|
|
20
|
+
* The only job-state transition. First match wins.
|
|
21
|
+
*
|
|
22
|
+
* | when | state |
|
|
23
|
+
* | fix awaiting publish | in-review |
|
|
24
|
+
* | launching, and the selected run is absent, retired, or a fix | running |
|
|
25
|
+
* | feedback exhausted or ambiguous, landed, pull request merged | done |
|
|
26
|
+
* | feedback exhausted or ambiguous, landed, pull request closed | failed |
|
|
27
|
+
* | feedback exhausted or ambiguous, landed, pull request still open | needs-human |
|
|
28
|
+
* | selected run not retired | that run: landed, rejected, failed, dead, needs-human, approved, running |
|
|
29
|
+
* | retired run that landed | done, in-review, or failed from the pull request |
|
|
30
|
+
* | retired run whose gate rejected | rejected |
|
|
31
|
+
* | retired suspended run | needs-human |
|
|
32
|
+
* | attempt cap or fingerprint breaker | needs-human |
|
|
33
|
+
* | retired, failed, or dead run | failed |
|
|
34
|
+
* | launching breadcrumb | running |
|
|
35
|
+
* | dependencies unmet | blocked |
|
|
36
|
+
* | launch error | failed |
|
|
37
|
+
* | otherwise | queued |
|
|
38
|
+
*
|
|
17
39
|
* `deliver=local`: `done` is a finished run whose gate says approve.
|
|
18
40
|
* `deliver=pr`: that landed run is `in-review` while the pull request is open
|
|
19
41
|
* (and in the gap before tick opens it) and `done` only once it has merged.
|
|
@@ -21,6 +43,7 @@ function inFlight(job, observation) {
|
|
|
21
43
|
* a slot and does not unblock dependents. An ok run with no readable
|
|
22
44
|
* decision is `failed`, so a missing artifact cannot unblock the next job.
|
|
23
45
|
* An investigation is `done` on approve and never `in-review`.
|
|
46
|
+
* A cap or fingerprint stop is `needs-human` and does not hold a slot.
|
|
24
47
|
*/
|
|
25
48
|
export function jobState(job, observation) {
|
|
26
49
|
if (job.fixAwaitingPublish)
|
|
@@ -42,8 +65,8 @@ export function jobState(job, observation) {
|
|
|
42
65
|
return "rejected";
|
|
43
66
|
if (job.run?.status === "suspended")
|
|
44
67
|
return "needs-human";
|
|
45
|
-
if (job.retriesExhausted)
|
|
46
|
-
return "
|
|
68
|
+
if ((job.retriesExhausted || job.stopReason) && !job.launching)
|
|
69
|
+
return "needs-human";
|
|
47
70
|
if (job.runRetired || job.run?.status === "failed" || job.run?.status === "dead")
|
|
48
71
|
return "failed";
|
|
49
72
|
if (job.launching)
|
|
@@ -150,7 +173,7 @@ function canStartFix(job) {
|
|
|
150
173
|
}
|
|
151
174
|
/** A recorded transient failure whose backoff has elapsed. One retry per tick, ahead of a new job. */
|
|
152
175
|
function canRetry(job, observation, now) {
|
|
153
|
-
if (job.error || job.launching || job.retriesExhausted || job.ledgerError)
|
|
176
|
+
if (job.error || job.launching || job.retriesExhausted || job.stopReason || job.ledgerError)
|
|
154
177
|
return false;
|
|
155
178
|
if (job.run && !job.runRetired)
|
|
156
179
|
return false;
|
|
@@ -192,6 +215,9 @@ export function decide(observation, now = new Date()) {
|
|
|
192
215
|
}
|
|
193
216
|
if (inFlightCount(observation) >= observation.maxConcurrent)
|
|
194
217
|
return actions;
|
|
218
|
+
// Pause and the daily dollar cap stop new runs. In-flight work is not touched.
|
|
219
|
+
if (observation.paused || observation.spendBlocked)
|
|
220
|
+
return actions;
|
|
195
221
|
const retry = observation.jobs.find((job) => canRetry(job, observation, now));
|
|
196
222
|
if (retry) {
|
|
197
223
|
actions.push({ type: "retry", jobId: retry.jobId, repo: retry.repo });
|
package/dist/reliability.js
CHANGED
|
@@ -4,7 +4,10 @@ import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readdirSync
|
|
|
4
4
|
import { dirname, join, resolve, sep } from "node:path";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
import { authenticationFailureLine } from "./auth.js";
|
|
7
|
-
import {
|
|
7
|
+
import { attemptsFor } from "./config.js";
|
|
8
|
+
import { appendEvent, readEvents } from "./events.js";
|
|
9
|
+
import { decideFailure, failureFingerprint, isSandboxStartFlake } from "./failure.js";
|
|
10
|
+
import { fingerprintCount } from "./rails.js";
|
|
8
11
|
import { jobBasename } from "./paths.js";
|
|
9
12
|
import { FAILURE_REASONS } from "./reconcile.js";
|
|
10
13
|
/**
|
|
@@ -88,25 +91,23 @@ export function resetAttemptLedger(kilnDir, jobId, runId) {
|
|
|
88
91
|
*/
|
|
89
92
|
export function recoverRuns(config, jobs, options) {
|
|
90
93
|
const notes = [];
|
|
94
|
+
const events = readEvents(config.kilnDir, { warn: () => { } });
|
|
91
95
|
for (const job of jobs) {
|
|
92
96
|
if (job.error)
|
|
93
97
|
continue;
|
|
94
98
|
const existing = readAttemptLedger(config.kilnDir, job.jobId);
|
|
95
99
|
if (existing === "corrupt")
|
|
96
100
|
continue;
|
|
97
|
-
if (job.run?.sandboxError && job.run.status !== "alive") {
|
|
98
|
-
suppressAutomaticRetry(config.kilnDir, job.jobId);
|
|
99
|
-
continue;
|
|
100
|
-
}
|
|
101
101
|
const pending = pendingFailure(job, config, options);
|
|
102
102
|
if (!pending)
|
|
103
103
|
continue;
|
|
104
104
|
if (pending.killPid !== undefined && !killTree(pending.killPid))
|
|
105
105
|
continue;
|
|
106
|
-
const recorded = recordFailure(config, job
|
|
106
|
+
const recorded = recordFailure(config, job, existing, pending, options.now, events);
|
|
107
107
|
if (recorded.created) {
|
|
108
108
|
appendLog(job.logPath, recorded.ledger.attempts.at(-1));
|
|
109
109
|
const runId = pending.runId ?? job.run?.runId;
|
|
110
|
+
const cost = job.run?.cost ?? { tokens: null, usd: null };
|
|
110
111
|
appendEvent(config.kilnDir, {
|
|
111
112
|
job_id: job.jobId,
|
|
112
113
|
run_id: runId ?? null,
|
|
@@ -114,15 +115,31 @@ export function recoverRuns(config, jobs, options) {
|
|
|
114
115
|
properties: {
|
|
115
116
|
reason: pending.reason,
|
|
116
117
|
...(pending.detail ? { detail: pending.detail } : {}),
|
|
118
|
+
class: recorded.decision.class,
|
|
119
|
+
fingerprint: recorded.decision.fingerprint,
|
|
120
|
+
tokens: cost.tokens,
|
|
121
|
+
usd: cost.usd,
|
|
117
122
|
once: runId ? `run.failed:${runId}` : `run.failed:launch:${recorded.ledger.attempts.length}`,
|
|
118
123
|
},
|
|
119
124
|
});
|
|
125
|
+
if (recorded.decision.needsHuman) {
|
|
126
|
+
appendEvent(config.kilnDir, {
|
|
127
|
+
job_id: job.jobId,
|
|
128
|
+
run_id: runId ?? null,
|
|
129
|
+
event: "needs-human",
|
|
130
|
+
properties: {
|
|
131
|
+
reason: recorded.decision.needsHuman,
|
|
132
|
+
once: `needs-human:${recorded.decision.needsHuman}:${runId ?? recorded.ledger.attempts.length}`,
|
|
133
|
+
},
|
|
134
|
+
});
|
|
135
|
+
}
|
|
120
136
|
notes.push({
|
|
121
137
|
jobId: job.jobId,
|
|
122
138
|
kind: kindFor(pending.reason),
|
|
123
139
|
reason: pending.reason,
|
|
124
140
|
attempt: recorded.ledger.attempts.length,
|
|
125
141
|
...(recorded.ledger.nextRetryAt ? { nextRetryAt: recorded.ledger.nextRetryAt } : {}),
|
|
142
|
+
...(recorded.decision.needsHuman ? { needsHuman: true } : {}),
|
|
126
143
|
});
|
|
127
144
|
}
|
|
128
145
|
const checkout = job.run?.checkout ?? config.repos[job.repo]?.path;
|
|
@@ -141,18 +158,6 @@ export function recoverRuns(config, jobs, options) {
|
|
|
141
158
|
}
|
|
142
159
|
return notes;
|
|
143
160
|
}
|
|
144
|
-
/** Drop a scheduled retry so a sandbox failure stays in Needs human. */
|
|
145
|
-
function suppressAutomaticRetry(kilnDir, jobId) {
|
|
146
|
-
const existing = readAttemptLedger(kilnDir, jobId);
|
|
147
|
-
if (!existing || existing === "corrupt" || !existing.nextRetryAt)
|
|
148
|
-
return;
|
|
149
|
-
const written = {
|
|
150
|
-
jobId: existing.jobId,
|
|
151
|
-
attempts: existing.attempts,
|
|
152
|
-
...(existing.ignoredRunIds && existing.ignoredRunIds.length > 0 ? { ignoredRunIds: existing.ignoredRunIds } : {}),
|
|
153
|
-
};
|
|
154
|
-
writeLedger(kilnDir, written);
|
|
155
|
-
}
|
|
156
161
|
function pendingFailure(job, config, options) {
|
|
157
162
|
if (job.run?.gateDecision === "reject")
|
|
158
163
|
return undefined;
|
|
@@ -261,10 +266,24 @@ function readAuthLog(logPath) {
|
|
|
261
266
|
closeSync(fd);
|
|
262
267
|
}
|
|
263
268
|
}
|
|
264
|
-
function recordFailure(config,
|
|
265
|
-
const ledger = existing ?? { jobId, attempts: [] };
|
|
269
|
+
function recordFailure(config, job, existing, pending, now, events) {
|
|
270
|
+
const ledger = existing ?? { jobId: job.jobId, attempts: [] };
|
|
271
|
+
const text = failureText(job, pending);
|
|
272
|
+
const decision = decideFailure({
|
|
273
|
+
text,
|
|
274
|
+
operational: pending.reason === "stalled" ||
|
|
275
|
+
pending.reason === "orphaned" ||
|
|
276
|
+
pending.reason === "crashed" ||
|
|
277
|
+
pending.reason === "failed",
|
|
278
|
+
blocked: pending.reason === "auth-failed" ||
|
|
279
|
+
(Boolean(job.run?.sandboxError) && !isSandboxStartFlake(text)),
|
|
280
|
+
attempt: ledger.attempts.length + 1,
|
|
281
|
+
maxAttempts: attemptsFor(config, job.repo),
|
|
282
|
+
sameFingerprint: fingerprintCount(events, job.jobId, failureFingerprint(text)) + 1,
|
|
283
|
+
maxSameFingerprint: config.maxSameFingerprint,
|
|
284
|
+
});
|
|
266
285
|
if (alreadyRecorded(ledger, pending))
|
|
267
|
-
return { ledger, created: false };
|
|
286
|
+
return { ledger, created: false, decision };
|
|
268
287
|
const attempt = {
|
|
269
288
|
n: ledger.attempts.length + 1,
|
|
270
289
|
at: now.toISOString(),
|
|
@@ -274,16 +293,25 @@ function recordFailure(config, jobId, existing, pending, now) {
|
|
|
274
293
|
detail: pending.detail,
|
|
275
294
|
};
|
|
276
295
|
const attempts = [...ledger.attempts, attempt];
|
|
277
|
-
const
|
|
278
|
-
const next = exhausted ? undefined : new Date(now.getTime() + retryDelayMs(attempts.length)).toISOString();
|
|
296
|
+
const next = decision.retry ? new Date(now.getTime() + retryDelayMs(attempts.length)).toISOString() : undefined;
|
|
279
297
|
const written = {
|
|
280
|
-
jobId,
|
|
298
|
+
jobId: job.jobId,
|
|
281
299
|
attempts,
|
|
282
300
|
...(next ? { nextRetryAt: next } : {}),
|
|
283
301
|
...(ledger.ignoredRunIds && ledger.ignoredRunIds.length > 0 ? { ignoredRunIds: ledger.ignoredRunIds } : {}),
|
|
284
302
|
};
|
|
285
303
|
writeLedger(config.kilnDir, written);
|
|
286
|
-
return { ledger: written, created: true };
|
|
304
|
+
return { ledger: written, created: true, decision };
|
|
305
|
+
}
|
|
306
|
+
function failureText(job, pending) {
|
|
307
|
+
const parts = [pending.detail];
|
|
308
|
+
if (job.run?.sandboxError)
|
|
309
|
+
parts.push(job.run.sandboxError);
|
|
310
|
+
for (const event of job.run?.stepEvents ?? []) {
|
|
311
|
+
if (event.detail)
|
|
312
|
+
parts.push(event.detail);
|
|
313
|
+
}
|
|
314
|
+
return parts.filter((part) => part.length > 0).join("\n");
|
|
287
315
|
}
|
|
288
316
|
function alreadyRecorded(ledger, pending) {
|
|
289
317
|
if (pending.runId)
|
package/dist/report.js
CHANGED
|
@@ -159,6 +159,10 @@ export function statusDetails(job) {
|
|
|
159
159
|
lines.push(`feedback ${job.feedbackRounds?.length}`);
|
|
160
160
|
if (job.state === "needs-human" && job.feedbackReason)
|
|
161
161
|
lines.push(`reason ${job.feedbackReason}`);
|
|
162
|
+
if (job.stopReason)
|
|
163
|
+
lines.push(`reason ${job.stopReason}`);
|
|
164
|
+
if (job.waitReason)
|
|
165
|
+
lines.push(job.waitReason);
|
|
162
166
|
if (job.attemptCount > 0)
|
|
163
167
|
lines.push(`attempts ${job.attemptCount}`);
|
|
164
168
|
if (job.lastFailureReason)
|
|
@@ -167,8 +171,9 @@ export function statusDetails(job) {
|
|
|
167
171
|
lines.push(AUTH_FAILED_LINE);
|
|
168
172
|
if (job.nextRetryAt)
|
|
169
173
|
lines.push(`next retry ${job.nextRetryAt}`);
|
|
170
|
-
if (job.state === "dead" && job.attemptCount > 0)
|
|
174
|
+
if ((job.state === "dead" || job.stopReason) && job.attemptCount > 0 && job.gates.length === 0) {
|
|
171
175
|
lines.push(`retry: kiln retry ${job.jobId}`);
|
|
176
|
+
}
|
|
172
177
|
if (job.noReadableApproval)
|
|
173
178
|
lines.push("no readable approval decision");
|
|
174
179
|
if (job.cancelCommand)
|
|
@@ -186,6 +191,9 @@ export function statusDetails(job) {
|
|
|
186
191
|
lines.push(`approve: kiln approve ${job.jobId}`);
|
|
187
192
|
if (gate.choices.includes("reject"))
|
|
188
193
|
lines.push(`reject: kiln reject ${job.jobId}`);
|
|
194
|
+
if (gate.choices.includes("approve") || gate.choices.includes("revise")) {
|
|
195
|
+
lines.push(`revise: kiln revise ${job.jobId} --note <text>`);
|
|
196
|
+
}
|
|
189
197
|
}
|
|
190
198
|
}
|
|
191
199
|
return lines;
|
|
@@ -216,7 +224,7 @@ function toStatusJob(job, observation) {
|
|
|
216
224
|
...(job.findingsPath ? { findingsPath: job.findingsPath } : {}),
|
|
217
225
|
launchFailed: job.launchFailed,
|
|
218
226
|
attemptCount: job.attemptCount ?? 0,
|
|
219
|
-
maxAttempts: observation.maxAttempts,
|
|
227
|
+
maxAttempts: job.attemptCap ?? observation.maxAttempts,
|
|
220
228
|
...(job.lastFailureReason ? { lastFailureReason: job.lastFailureReason } : {}),
|
|
221
229
|
...(job.nextRetryAt ? { nextRetryAt: job.nextRetryAt } : {}),
|
|
222
230
|
invalidReason: job.error,
|
|
@@ -226,6 +234,7 @@ function toStatusJob(job, observation) {
|
|
|
226
234
|
gates: [],
|
|
227
235
|
canApprove: false,
|
|
228
236
|
canReject: false,
|
|
237
|
+
canRevise: false,
|
|
229
238
|
steps: progressFor(job),
|
|
230
239
|
...(job.deliver ? { deliver: job.deliver } : {}),
|
|
231
240
|
...(job.createdAt ? { createdAt: job.createdAt } : {}),
|
|
@@ -239,7 +248,7 @@ function toStatusJob(job, observation) {
|
|
|
239
248
|
: job.feedbackAmbiguous
|
|
240
249
|
? "feedback-ambiguous"
|
|
241
250
|
: undefined;
|
|
242
|
-
const gates = state === "needs-human" && job.run && !feedbackReason ? job.run.gates.map(toGateView) : [];
|
|
251
|
+
const gates = state === "needs-human" && job.run && !feedbackReason && !job.stopReason ? job.run.gates.map(toGateView) : [];
|
|
243
252
|
const prNumber = job.delivery?.prNumber;
|
|
244
253
|
const runId = job.run?.runId;
|
|
245
254
|
const showBranch = runId !== undefined && (state === "done" || state === "in-review" || feedbackReason !== undefined);
|
|
@@ -263,8 +272,12 @@ function toStatusJob(job, observation) {
|
|
|
263
272
|
...(job.findingsPath ? { findingsPath: job.findingsPath } : {}),
|
|
264
273
|
launchFailed: job.launchFailed,
|
|
265
274
|
attemptCount: job.attemptCount ?? 0,
|
|
266
|
-
maxAttempts: observation.maxAttempts,
|
|
275
|
+
maxAttempts: job.attemptCap ?? observation.maxAttempts,
|
|
267
276
|
...(job.lastFailureReason ? { lastFailureReason: job.lastFailureReason } : {}),
|
|
277
|
+
...(job.stopReason ? { stopReason: job.stopReason } : {}),
|
|
278
|
+
...(observation.holdDetail && (state === "queued" || state === "blocked" || state === "failed")
|
|
279
|
+
? { waitReason: observation.holdDetail }
|
|
280
|
+
: {}),
|
|
268
281
|
...(job.nextRetryAt && !job.retriesExhausted ? { nextRetryAt: job.nextRetryAt } : {}),
|
|
269
282
|
...(job.ledgerError ? { ledgerError: job.ledgerError } : {}),
|
|
270
283
|
...(job.feedbackLedgerError ? { feedbackLedgerError: job.feedbackLedgerError } : {}),
|
|
@@ -281,6 +294,7 @@ function toStatusJob(job, observation) {
|
|
|
281
294
|
gates,
|
|
282
295
|
canApprove: gates.some((gate) => gate.choices.includes("approve")),
|
|
283
296
|
canReject: gates.some((gate) => gate.choices.includes("reject")),
|
|
297
|
+
canRevise: gates.some((gate) => gate.choices.includes("approve") || gate.choices.includes("revise")),
|
|
284
298
|
steps: progressFor(job),
|
|
285
299
|
...(job.createdAt ? { createdAt: job.createdAt } : {}),
|
|
286
300
|
...(job.run?.startedAt ? { startedAt: job.run.startedAt } : {}),
|
|
@@ -402,7 +416,7 @@ export function renderTick(observation, actions, launches, delivers = [], findin
|
|
|
402
416
|
: `findings ${recorded.jobId} missing`);
|
|
403
417
|
}
|
|
404
418
|
for (const note of notes) {
|
|
405
|
-
const next = note.nextRetryAt ? ` next ${note.nextRetryAt}` : note.
|
|
419
|
+
const next = note.nextRetryAt ? ` next ${note.nextRetryAt}` : note.needsHuman ? " needs human" : "";
|
|
406
420
|
lines.push(`${note.kind} ${note.jobId} ${note.reason} attempt ${note.attempt}${next}`);
|
|
407
421
|
}
|
|
408
422
|
for (const note of feedbackNotes) {
|
package/dist/schemas.js
CHANGED
|
@@ -36,6 +36,18 @@ export const RepoEntrySchema = z.object({
|
|
|
36
36
|
* The local branch stays `yak/<run-id>`.
|
|
37
37
|
*/
|
|
38
38
|
branch: z.string().min(1, "Branch template is empty.").optional(),
|
|
39
|
+
/** Overrides the global attempt cap for jobs in this repo. */
|
|
40
|
+
maxAttempts: z
|
|
41
|
+
.number()
|
|
42
|
+
.int("Max attempts must be a whole number.")
|
|
43
|
+
.min(1, "Max attempts must be at least 1.")
|
|
44
|
+
.optional(),
|
|
45
|
+
/** Overrides the global fix-round cap for jobs in this repo. */
|
|
46
|
+
maxFixRounds: z
|
|
47
|
+
.number()
|
|
48
|
+
.int("Max fix rounds must be a whole number.")
|
|
49
|
+
.min(1, "Max fix rounds must be at least 1.")
|
|
50
|
+
.optional(),
|
|
39
51
|
});
|
|
40
52
|
/** Repo map shared by the config file and the settings API. */
|
|
41
53
|
export const KilnReposSchema = z.record(z.string().regex(REPO_NAME, "must start with a letter or digit and contain only letters, digits, dots, underscores, and hyphens"), RepoEntrySchema);
|
|
@@ -75,21 +87,34 @@ export const EditableSandboxSchema = SandboxSettingsSchema.pick({
|
|
|
75
87
|
const WORKFLOW_KEY = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
76
88
|
/** No yak event and no log output for this long stalls a running job. `<= 0` disables it. */
|
|
77
89
|
export const DEFAULT_STALL_TIMEOUT_MS = 15 * 60 * 1000;
|
|
78
|
-
/** Transient failures retry until this many attempts, then the job
|
|
90
|
+
/** Transient failures retry until this many attempts, then the job needs a person. */
|
|
79
91
|
export const DEFAULT_MAX_ATTEMPTS = 3;
|
|
80
92
|
/** Pull request fix rounds before the job is needs-human (`feedback-exhausted`). */
|
|
81
93
|
export const DEFAULT_MAX_FIX_ROUNDS = 2;
|
|
94
|
+
/** Identical failure fingerprints before the job needs a person. */
|
|
95
|
+
export const DEFAULT_MAX_SAME_FINGERPRINT = 3;
|
|
82
96
|
/**
|
|
83
97
|
* Milliseconds without a yak event or log output before a running job is
|
|
84
98
|
* killed and retried. Default 15 minutes. `0` or negative disables stall detection.
|
|
85
99
|
*/
|
|
86
100
|
const stallTimeoutMs = z.number().int("Stall timeout must be a whole number of milliseconds.").optional();
|
|
87
|
-
/** Transient failures before the job
|
|
101
|
+
/** Transient failures before the job needs a person. Default 3. Hitting the cap is a budget failure. */
|
|
88
102
|
const maxAttempts = z
|
|
89
103
|
.number()
|
|
90
104
|
.int("Max attempts must be a whole number.")
|
|
91
105
|
.min(1, "Max attempts must be at least 1.")
|
|
92
106
|
.optional();
|
|
107
|
+
/** Same failure fingerprint before Needs human. Default 3. */
|
|
108
|
+
const maxSameFingerprint = z
|
|
109
|
+
.number()
|
|
110
|
+
.int("Fingerprint limit must be a whole number.")
|
|
111
|
+
.min(1, "Fingerprint limit must be at least 1.")
|
|
112
|
+
.optional();
|
|
113
|
+
/**
|
|
114
|
+
* Dollars of recorded run cost allowed on the local calendar day.
|
|
115
|
+
* Omitted means no cap. `0` blocks new runs immediately.
|
|
116
|
+
*/
|
|
117
|
+
const dailyDollarCap = z.number().nonnegative("Daily dollar cap cannot be negative.").optional();
|
|
93
118
|
/**
|
|
94
119
|
* Fix rounds for an open pull request before the job is needs-human with
|
|
95
120
|
* reason `feedback-exhausted`. Default 2. Kiln does not merge.
|
|
@@ -109,6 +134,8 @@ export const EditableKilnSettingsSchema = z.object({
|
|
|
109
134
|
stallTimeoutMs,
|
|
110
135
|
maxAttempts,
|
|
111
136
|
maxFixRounds,
|
|
137
|
+
maxSameFingerprint,
|
|
138
|
+
dailyDollarCap,
|
|
112
139
|
sandbox: EditableSandboxSchema.optional(),
|
|
113
140
|
});
|
|
114
141
|
/** PUT /api/settings body. `revision` is the sha256 of the file the client loaded. */
|
|
@@ -121,6 +148,8 @@ export const KilnConfigFileSchema = z.object({
|
|
|
121
148
|
stallTimeoutMs,
|
|
122
149
|
maxAttempts,
|
|
123
150
|
maxFixRounds,
|
|
151
|
+
maxSameFingerprint,
|
|
152
|
+
dailyDollarCap,
|
|
124
153
|
adapter: z.enum(["mock", "claude-code"]).optional(),
|
|
125
154
|
sandbox: SandboxSettingsSchema.optional(),
|
|
126
155
|
/** Per-workflow sandbox override. A set field replaces the global one; an omitted field inherits. */
|
|
@@ -176,9 +205,10 @@ export const JobSchema = z.object({
|
|
|
176
205
|
* `deliver=pr`. It does not enter `in-review`. A reject, a failure, a dead
|
|
177
206
|
* process, and an ok run with no readable approval do not unblock an
|
|
178
207
|
* after-merged dependent. A stacked job unblocks when its one upstream has
|
|
179
|
-
* landed, which includes `in-review`.
|
|
180
|
-
*
|
|
181
|
-
*
|
|
208
|
+
* landed, which includes `in-review`. A job that has used its attempt cap,
|
|
209
|
+
* fix-round cap, or fingerprint breaker is `needs-human`. `dead` is a run
|
|
210
|
+
* whose process died and has not been recorded yet. Attempts live under
|
|
211
|
+
* the kiln directory, not in this file.
|
|
182
212
|
*/
|
|
183
213
|
export const JOB_STATES = [
|
|
184
214
|
"queued",
|