@lchase/kiln 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/assets/serve.css +13 -0
- package/assets/serve.js +49 -3
- package/dist/attention.js +13 -4
- package/dist/cli.js +112 -8
- package/dist/config.js +25 -1
- package/dist/delivery.js +18 -0
- package/dist/driver.js +213 -14
- package/dist/events.js +393 -0
- package/dist/evidence.js +140 -0
- package/dist/failure.js +140 -0
- package/dist/fix-commit.js +73 -0
- package/dist/jobs.js +12 -0
- package/dist/ledger.js +152 -0
- package/dist/project.js +394 -0
- package/dist/rails.js +164 -0
- package/dist/reconcile.js +29 -3
- package/dist/redact.js +63 -0
- package/dist/reliability.js +75 -26
- package/dist/report.js +19 -5
- package/dist/schemas.js +35 -5
- package/dist/serve-page.js +21 -1
- package/dist/serve.js +86 -4
- package/dist/settings.js +9 -1
- package/package.json +2 -2
- package/workflows/fix.yaml +1 -1
package/dist/rails.js
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { appendEvent, readEvents } from "./events.js";
|
|
2
|
+
export function localDay(now) {
|
|
3
|
+
const year = now.getFullYear();
|
|
4
|
+
const month = String(now.getMonth() + 1).padStart(2, "0");
|
|
5
|
+
const day = String(now.getDate()).padStart(2, "0");
|
|
6
|
+
return `${year}-${month}-${day}`;
|
|
7
|
+
}
|
|
8
|
+
export function isPaused(kilnDir) {
|
|
9
|
+
return pausedFrom(readEvents(kilnDir, { warn: () => { } }));
|
|
10
|
+
}
|
|
11
|
+
export function pausedFrom(events) {
|
|
12
|
+
let paused = false;
|
|
13
|
+
for (const event of events) {
|
|
14
|
+
if (event.event === "kiln.paused")
|
|
15
|
+
paused = true;
|
|
16
|
+
else if (event.event === "kiln.resumed")
|
|
17
|
+
paused = false;
|
|
18
|
+
}
|
|
19
|
+
return paused;
|
|
20
|
+
}
|
|
21
|
+
/** Append a pause event when the factory is running. Already paused is a no-op. */
|
|
22
|
+
export function pauseFactory(kilnDir, now = new Date()) {
|
|
23
|
+
if (pausedFrom(readEvents(kilnDir, { warn: () => { } })))
|
|
24
|
+
return false;
|
|
25
|
+
appendEvent(kilnDir, {
|
|
26
|
+
job_id: null,
|
|
27
|
+
event: "kiln.paused",
|
|
28
|
+
properties: {},
|
|
29
|
+
ts: now.toISOString(),
|
|
30
|
+
});
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
/** Append a resume event when the factory is paused. Already running is a no-op. */
|
|
34
|
+
export function resumeFactory(kilnDir, now = new Date()) {
|
|
35
|
+
if (!pausedFrom(readEvents(kilnDir, { warn: () => { } })))
|
|
36
|
+
return false;
|
|
37
|
+
appendEvent(kilnDir, {
|
|
38
|
+
job_id: null,
|
|
39
|
+
event: "kiln.resumed",
|
|
40
|
+
properties: {},
|
|
41
|
+
ts: now.toISOString(),
|
|
42
|
+
});
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
export function spendSnapshot(events, cap, now = new Date()) {
|
|
46
|
+
const day = localDay(now);
|
|
47
|
+
let usd = 0;
|
|
48
|
+
for (const event of events) {
|
|
49
|
+
if (localDay(new Date(event.ts)) !== day)
|
|
50
|
+
continue;
|
|
51
|
+
const amount = dollarCost(event);
|
|
52
|
+
if (amount === undefined)
|
|
53
|
+
continue;
|
|
54
|
+
usd += amount;
|
|
55
|
+
}
|
|
56
|
+
const limit = cap === undefined ? null : cap;
|
|
57
|
+
return {
|
|
58
|
+
day,
|
|
59
|
+
usd,
|
|
60
|
+
cap: limit,
|
|
61
|
+
blocked: limit !== null && usd >= limit,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
export function spendFor(kilnDir, cap, now = new Date()) {
|
|
65
|
+
return spendSnapshot(readEvents(kilnDir, { warn: () => { } }), cap, now);
|
|
66
|
+
}
|
|
67
|
+
/** One event per local day when the dollar cap is blocking new runs. */
|
|
68
|
+
export function recordDailyCap(kilnDir, spend, now = new Date()) {
|
|
69
|
+
if (!spend.blocked || spend.cap === null)
|
|
70
|
+
return;
|
|
71
|
+
appendEvent(kilnDir, {
|
|
72
|
+
job_id: null,
|
|
73
|
+
event: "budget.blocked",
|
|
74
|
+
properties: {
|
|
75
|
+
day: spend.day,
|
|
76
|
+
usd: spend.usd,
|
|
77
|
+
cap: spend.cap,
|
|
78
|
+
once: `daily-cap:${spend.day}`,
|
|
79
|
+
},
|
|
80
|
+
ts: now.toISOString(),
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
export function formatSpend(spend) {
|
|
84
|
+
const cap = spend.cap === null ? "no cap" : `$${spend.cap.toFixed(2)}`;
|
|
85
|
+
return `today ${spend.day} $${spend.usd.toFixed(2)} / ${cap}`;
|
|
86
|
+
}
|
|
87
|
+
export function holdDetail(paused, spend) {
|
|
88
|
+
if (paused)
|
|
89
|
+
return "Paused. New runs wait until kiln resume.";
|
|
90
|
+
if (spend.blocked && spend.cap !== null) {
|
|
91
|
+
return `Daily cap reached ($${spend.usd.toFixed(2)} of $${spend.cap.toFixed(2)}). New runs wait until the local day rolls over.`;
|
|
92
|
+
}
|
|
93
|
+
return undefined;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Revise notes written since the latest `run.started` for this job.
|
|
97
|
+
* The next launch appends them to the spec, which is the next prompt's input.
|
|
98
|
+
*/
|
|
99
|
+
export function reviseNotesSinceRun(events, jobId) {
|
|
100
|
+
const notes = [];
|
|
101
|
+
for (const event of events) {
|
|
102
|
+
if (event.job_id !== jobId)
|
|
103
|
+
continue;
|
|
104
|
+
if (event.event === "run.started") {
|
|
105
|
+
notes.length = 0;
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
if (event.event === "gate.resolved" && event.properties.decision === "revise" && event.properties.note) {
|
|
109
|
+
notes.push(event.properties.note);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return notes;
|
|
113
|
+
}
|
|
114
|
+
export function withReviseNotes(spec, notes) {
|
|
115
|
+
if (notes.length === 0)
|
|
116
|
+
return spec;
|
|
117
|
+
return [
|
|
118
|
+
spec.trimEnd(),
|
|
119
|
+
"",
|
|
120
|
+
"## Human note",
|
|
121
|
+
"",
|
|
122
|
+
"Apply every note below on this round. A person typed it at the gate.",
|
|
123
|
+
"",
|
|
124
|
+
notes.join("\n\n"),
|
|
125
|
+
"",
|
|
126
|
+
].join("\n");
|
|
127
|
+
}
|
|
128
|
+
/** Fingerprint or attempt-cap stop that a later manual retry has not cleared. */
|
|
129
|
+
export function stopReasonFromEvents(events, jobId) {
|
|
130
|
+
let reason;
|
|
131
|
+
for (const event of events) {
|
|
132
|
+
if (event.job_id !== jobId)
|
|
133
|
+
continue;
|
|
134
|
+
if (event.event === "job.retried") {
|
|
135
|
+
reason = undefined;
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
if (event.event !== "needs-human")
|
|
139
|
+
continue;
|
|
140
|
+
if (event.properties.reason === "fingerprint" || event.properties.reason === "attempt-cap") {
|
|
141
|
+
reason = event.properties.reason;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return reason;
|
|
145
|
+
}
|
|
146
|
+
/** How many `run.failed` lines for this job already carry `fingerprint`. */
|
|
147
|
+
export function fingerprintCount(events, jobId, fingerprint) {
|
|
148
|
+
let count = 0;
|
|
149
|
+
for (const event of events) {
|
|
150
|
+
if (event.job_id !== jobId || event.event !== "run.failed")
|
|
151
|
+
continue;
|
|
152
|
+
if (event.properties.fingerprint === fingerprint)
|
|
153
|
+
count += 1;
|
|
154
|
+
}
|
|
155
|
+
return count;
|
|
156
|
+
}
|
|
157
|
+
function dollarCost(event) {
|
|
158
|
+
if (event.event !== "run.finished" && event.event !== "run.failed")
|
|
159
|
+
return undefined;
|
|
160
|
+
const usd = event.properties.usd;
|
|
161
|
+
if (typeof usd !== "number" || !Number.isFinite(usd) || usd < 0)
|
|
162
|
+
return undefined;
|
|
163
|
+
return usd;
|
|
164
|
+
}
|
package/dist/reconcile.js
CHANGED
|
@@ -10,10 +10,32 @@ export function inFlightCount(observation) {
|
|
|
10
10
|
function inFlight(job, observation) {
|
|
11
11
|
if (job.error || job.feedbackExhausted || job.feedbackAmbiguous)
|
|
12
12
|
return false;
|
|
13
|
+
// A cap or fingerprint stop is waiting on a person. It does not hold a slot.
|
|
14
|
+
if (job.retriesExhausted || job.stopReason)
|
|
15
|
+
return false;
|
|
13
16
|
const state = jobState(job, observation);
|
|
14
17
|
return state === "running" || state === "needs-human" || state === "approved";
|
|
15
18
|
}
|
|
16
19
|
/**
|
|
20
|
+
* The only job-state transition. First match wins.
|
|
21
|
+
*
|
|
22
|
+
* | when | state |
|
|
23
|
+
* | fix awaiting publish | in-review |
|
|
24
|
+
* | launching, and the selected run is absent, retired, or a fix | running |
|
|
25
|
+
* | feedback exhausted or ambiguous, landed, pull request merged | done |
|
|
26
|
+
* | feedback exhausted or ambiguous, landed, pull request closed | failed |
|
|
27
|
+
* | feedback exhausted or ambiguous, landed, pull request still open | needs-human |
|
|
28
|
+
* | selected run not retired | that run: landed, rejected, failed, dead, needs-human, approved, running |
|
|
29
|
+
* | retired run that landed | done, in-review, or failed from the pull request |
|
|
30
|
+
* | retired run whose gate rejected | rejected |
|
|
31
|
+
* | retired suspended run | needs-human |
|
|
32
|
+
* | attempt cap or fingerprint breaker | needs-human |
|
|
33
|
+
* | retired, failed, or dead run | failed |
|
|
34
|
+
* | launching breadcrumb | running |
|
|
35
|
+
* | dependencies unmet | blocked |
|
|
36
|
+
* | launch error | failed |
|
|
37
|
+
* | otherwise | queued |
|
|
38
|
+
*
|
|
17
39
|
* `deliver=local`: `done` is a finished run whose gate says approve.
|
|
18
40
|
* `deliver=pr`: that landed run is `in-review` while the pull request is open
|
|
19
41
|
* (and in the gap before tick opens it) and `done` only once it has merged.
|
|
@@ -21,6 +43,7 @@ function inFlight(job, observation) {
|
|
|
21
43
|
* a slot and does not unblock dependents. An ok run with no readable
|
|
22
44
|
* decision is `failed`, so a missing artifact cannot unblock the next job.
|
|
23
45
|
* An investigation is `done` on approve and never `in-review`.
|
|
46
|
+
* A cap or fingerprint stop is `needs-human` and does not hold a slot.
|
|
24
47
|
*/
|
|
25
48
|
export function jobState(job, observation) {
|
|
26
49
|
if (job.fixAwaitingPublish)
|
|
@@ -42,8 +65,8 @@ export function jobState(job, observation) {
|
|
|
42
65
|
return "rejected";
|
|
43
66
|
if (job.run?.status === "suspended")
|
|
44
67
|
return "needs-human";
|
|
45
|
-
if (job.retriesExhausted)
|
|
46
|
-
return "
|
|
68
|
+
if ((job.retriesExhausted || job.stopReason) && !job.launching)
|
|
69
|
+
return "needs-human";
|
|
47
70
|
if (job.runRetired || job.run?.status === "failed" || job.run?.status === "dead")
|
|
48
71
|
return "failed";
|
|
49
72
|
if (job.launching)
|
|
@@ -150,7 +173,7 @@ function canStartFix(job) {
|
|
|
150
173
|
}
|
|
151
174
|
/** A recorded transient failure whose backoff has elapsed. One retry per tick, ahead of a new job. */
|
|
152
175
|
function canRetry(job, observation, now) {
|
|
153
|
-
if (job.error || job.launching || job.retriesExhausted || job.ledgerError)
|
|
176
|
+
if (job.error || job.launching || job.retriesExhausted || job.stopReason || job.ledgerError)
|
|
154
177
|
return false;
|
|
155
178
|
if (job.run && !job.runRetired)
|
|
156
179
|
return false;
|
|
@@ -192,6 +215,9 @@ export function decide(observation, now = new Date()) {
|
|
|
192
215
|
}
|
|
193
216
|
if (inFlightCount(observation) >= observation.maxConcurrent)
|
|
194
217
|
return actions;
|
|
218
|
+
// Pause and the daily dollar cap stop new runs. In-flight work is not touched.
|
|
219
|
+
if (observation.paused || observation.spendBlocked)
|
|
220
|
+
return actions;
|
|
195
221
|
const retry = observation.jobs.find((job) => canRetry(job, observation, now));
|
|
196
222
|
if (retry) {
|
|
197
223
|
actions.push({ type: "retry", jobId: retry.jobId, repo: retry.repo });
|
package/dist/redact.js
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Masks secrets in text Kiln writes to the event log, the ledger, and a
|
|
3
|
+
* pull request body. One function owns that. A log or a PR is public-shaped
|
|
4
|
+
* even when a test command or a check excerpt is not.
|
|
5
|
+
*
|
|
6
|
+
* Token shapes cover the common issued-credential prefixes. A value is also
|
|
7
|
+
* masked when it belongs to an env assignment, a JSON field, or an auth
|
|
8
|
+
* header whose name contains TOKEN, KEY, SECRET, or PASSWORD.
|
|
9
|
+
*/
|
|
10
|
+
const SECRET_NAME = /token|key|secret|password/i;
|
|
11
|
+
const TOKEN_SHAPES = [
|
|
12
|
+
/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{20,}\b/g,
|
|
13
|
+
/\bgithub_pat_[A-Za-z0-9_]{20,}\b/g,
|
|
14
|
+
/\bglpat-[A-Za-z0-9\-_]{16,}\b/g,
|
|
15
|
+
/\bxox[baprs]-[A-Za-z0-9-]{10,}/g,
|
|
16
|
+
/\bAKIA[0-9A-Z]{16}\b/g,
|
|
17
|
+
/\bsk-ant-[A-Za-z0-9\-_]{10,}\b/g,
|
|
18
|
+
/\bsk-[A-Za-z0-9]{20,}\b/g,
|
|
19
|
+
/\bnpm_[A-Za-z0-9]{20,}\b/g,
|
|
20
|
+
/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g,
|
|
21
|
+
];
|
|
22
|
+
const AUTH_HEADER = /\b(authorization|proxy-authorization|x-api-key|x-auth-token)\s*:[^\n]*/gi;
|
|
23
|
+
const BEARER = /\bBearer\s+[A-Za-z0-9\-._~+/]{8,}={0,2}/g;
|
|
24
|
+
const ENV_ASSIGN = /(^|[\n\r;\s])((?:export[ \t]+)?)([A-Za-z_][A-Za-z0-9_]*)([ \t]*=[ \t]*)("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|[^\s#]+)/g;
|
|
25
|
+
const JSON_FIELD = /"([A-Za-z_][A-Za-z0-9_]*)"(\s*:\s*)("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')/g;
|
|
26
|
+
/** Replace secret-shaped text with `[redacted]`. Idempotent on its own output. */
|
|
27
|
+
export function redact(text) {
|
|
28
|
+
let out = text.replace(ENV_ASSIGN, (full, lead, prefix, name, eq, value) => {
|
|
29
|
+
if (!SECRET_NAME.test(name))
|
|
30
|
+
return full;
|
|
31
|
+
return `${lead}${prefix}${name}${eq}[redacted]`;
|
|
32
|
+
});
|
|
33
|
+
out = out.replace(JSON_FIELD, (full, name, mid, value) => {
|
|
34
|
+
if (!SECRET_NAME.test(name))
|
|
35
|
+
return full;
|
|
36
|
+
const quote = value.startsWith("'") ? "'" : '"';
|
|
37
|
+
return `"${name}"${mid}${quote}[redacted]${quote}`;
|
|
38
|
+
});
|
|
39
|
+
out = out.replace(AUTH_HEADER, (_full, name) => `${name}: [redacted]`);
|
|
40
|
+
out = out.replace(BEARER, "Bearer [redacted]");
|
|
41
|
+
for (const pattern of TOKEN_SHAPES) {
|
|
42
|
+
out = out.replace(pattern, "[redacted]");
|
|
43
|
+
}
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
/** Walk a JSON value and redact every string. Object keys that name a secret drop the value. */
|
|
47
|
+
export function redactDeep(value) {
|
|
48
|
+
if (typeof value === "string")
|
|
49
|
+
return redact(value);
|
|
50
|
+
if (typeof value !== "object" || value === null)
|
|
51
|
+
return value;
|
|
52
|
+
if (Array.isArray(value))
|
|
53
|
+
return value.map((item) => redactDeep(item));
|
|
54
|
+
const out = {};
|
|
55
|
+
for (const [key, inner] of Object.entries(value)) {
|
|
56
|
+
if (typeof inner === "string" && SECRET_NAME.test(key)) {
|
|
57
|
+
out[key] = "[redacted]";
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
out[key] = redactDeep(inner);
|
|
61
|
+
}
|
|
62
|
+
return out;
|
|
63
|
+
}
|
package/dist/reliability.js
CHANGED
|
@@ -4,6 +4,10 @@ import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readdirSync
|
|
|
4
4
|
import { dirname, join, resolve, sep } from "node:path";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
import { authenticationFailureLine } from "./auth.js";
|
|
7
|
+
import { attemptsFor } from "./config.js";
|
|
8
|
+
import { appendEvent, readEvents } from "./events.js";
|
|
9
|
+
import { decideFailure, failureFingerprint, isSandboxStartFlake } from "./failure.js";
|
|
10
|
+
import { fingerprintCount } from "./rails.js";
|
|
7
11
|
import { jobBasename } from "./paths.js";
|
|
8
12
|
import { FAILURE_REASONS } from "./reconcile.js";
|
|
9
13
|
/**
|
|
@@ -87,51 +91,73 @@ export function resetAttemptLedger(kilnDir, jobId, runId) {
|
|
|
87
91
|
*/
|
|
88
92
|
export function recoverRuns(config, jobs, options) {
|
|
89
93
|
const notes = [];
|
|
94
|
+
const events = readEvents(config.kilnDir, { warn: () => { } });
|
|
90
95
|
for (const job of jobs) {
|
|
91
96
|
if (job.error)
|
|
92
97
|
continue;
|
|
93
98
|
const existing = readAttemptLedger(config.kilnDir, job.jobId);
|
|
94
99
|
if (existing === "corrupt")
|
|
95
100
|
continue;
|
|
96
|
-
if (job.run?.sandboxError && job.run.status !== "alive") {
|
|
97
|
-
suppressAutomaticRetry(config.kilnDir, job.jobId);
|
|
98
|
-
continue;
|
|
99
|
-
}
|
|
100
101
|
const pending = pendingFailure(job, config, options);
|
|
101
102
|
if (!pending)
|
|
102
103
|
continue;
|
|
103
104
|
if (pending.killPid !== undefined && !killTree(pending.killPid))
|
|
104
105
|
continue;
|
|
105
|
-
const recorded = recordFailure(config, job
|
|
106
|
+
const recorded = recordFailure(config, job, existing, pending, options.now, events);
|
|
106
107
|
if (recorded.created) {
|
|
107
108
|
appendLog(job.logPath, recorded.ledger.attempts.at(-1));
|
|
109
|
+
const runId = pending.runId ?? job.run?.runId;
|
|
110
|
+
const cost = job.run?.cost ?? { tokens: null, usd: null };
|
|
111
|
+
appendEvent(config.kilnDir, {
|
|
112
|
+
job_id: job.jobId,
|
|
113
|
+
run_id: runId ?? null,
|
|
114
|
+
event: "run.failed",
|
|
115
|
+
properties: {
|
|
116
|
+
reason: pending.reason,
|
|
117
|
+
...(pending.detail ? { detail: pending.detail } : {}),
|
|
118
|
+
class: recorded.decision.class,
|
|
119
|
+
fingerprint: recorded.decision.fingerprint,
|
|
120
|
+
tokens: cost.tokens,
|
|
121
|
+
usd: cost.usd,
|
|
122
|
+
once: runId ? `run.failed:${runId}` : `run.failed:launch:${recorded.ledger.attempts.length}`,
|
|
123
|
+
},
|
|
124
|
+
});
|
|
125
|
+
if (recorded.decision.needsHuman) {
|
|
126
|
+
appendEvent(config.kilnDir, {
|
|
127
|
+
job_id: job.jobId,
|
|
128
|
+
run_id: runId ?? null,
|
|
129
|
+
event: "needs-human",
|
|
130
|
+
properties: {
|
|
131
|
+
reason: recorded.decision.needsHuman,
|
|
132
|
+
once: `needs-human:${recorded.decision.needsHuman}:${runId ?? recorded.ledger.attempts.length}`,
|
|
133
|
+
},
|
|
134
|
+
});
|
|
135
|
+
}
|
|
108
136
|
notes.push({
|
|
109
137
|
jobId: job.jobId,
|
|
110
138
|
kind: kindFor(pending.reason),
|
|
111
139
|
reason: pending.reason,
|
|
112
140
|
attempt: recorded.ledger.attempts.length,
|
|
113
141
|
...(recorded.ledger.nextRetryAt ? { nextRetryAt: recorded.ledger.nextRetryAt } : {}),
|
|
142
|
+
...(recorded.decision.needsHuman ? { needsHuman: true } : {}),
|
|
114
143
|
});
|
|
115
144
|
}
|
|
116
145
|
const checkout = job.run?.checkout ?? config.repos[job.repo]?.path;
|
|
117
146
|
const runId = pending.runId ?? job.run?.runId;
|
|
118
|
-
if (checkout && runId)
|
|
119
|
-
cleanupRunWorktree(checkout, runId);
|
|
147
|
+
if (checkout && runId) {
|
|
148
|
+
const outcome = cleanupRunWorktree(checkout, runId);
|
|
149
|
+
if (outcome === "removed" || outcome === "kept") {
|
|
150
|
+
appendEvent(config.kilnDir, {
|
|
151
|
+
job_id: job.jobId,
|
|
152
|
+
run_id: runId,
|
|
153
|
+
event: outcome === "removed" ? "worktree.cleaned" : "worktree.kept",
|
|
154
|
+
properties: { once: `worktree.${outcome}:${runId}` },
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
}
|
|
120
158
|
}
|
|
121
159
|
return notes;
|
|
122
160
|
}
|
|
123
|
-
/** Drop a scheduled retry so a sandbox failure stays in Needs human. */
|
|
124
|
-
function suppressAutomaticRetry(kilnDir, jobId) {
|
|
125
|
-
const existing = readAttemptLedger(kilnDir, jobId);
|
|
126
|
-
if (!existing || existing === "corrupt" || !existing.nextRetryAt)
|
|
127
|
-
return;
|
|
128
|
-
const written = {
|
|
129
|
-
jobId: existing.jobId,
|
|
130
|
-
attempts: existing.attempts,
|
|
131
|
-
...(existing.ignoredRunIds && existing.ignoredRunIds.length > 0 ? { ignoredRunIds: existing.ignoredRunIds } : {}),
|
|
132
|
-
};
|
|
133
|
-
writeLedger(kilnDir, written);
|
|
134
|
-
}
|
|
135
161
|
function pendingFailure(job, config, options) {
|
|
136
162
|
if (job.run?.gateDecision === "reject")
|
|
137
163
|
return undefined;
|
|
@@ -240,10 +266,24 @@ function readAuthLog(logPath) {
|
|
|
240
266
|
closeSync(fd);
|
|
241
267
|
}
|
|
242
268
|
}
|
|
243
|
-
function recordFailure(config,
|
|
244
|
-
const ledger = existing ?? { jobId, attempts: [] };
|
|
269
|
+
function recordFailure(config, job, existing, pending, now, events) {
|
|
270
|
+
const ledger = existing ?? { jobId: job.jobId, attempts: [] };
|
|
271
|
+
const text = failureText(job, pending);
|
|
272
|
+
const decision = decideFailure({
|
|
273
|
+
text,
|
|
274
|
+
operational: pending.reason === "stalled" ||
|
|
275
|
+
pending.reason === "orphaned" ||
|
|
276
|
+
pending.reason === "crashed" ||
|
|
277
|
+
pending.reason === "failed",
|
|
278
|
+
blocked: pending.reason === "auth-failed" ||
|
|
279
|
+
(Boolean(job.run?.sandboxError) && !isSandboxStartFlake(text)),
|
|
280
|
+
attempt: ledger.attempts.length + 1,
|
|
281
|
+
maxAttempts: attemptsFor(config, job.repo),
|
|
282
|
+
sameFingerprint: fingerprintCount(events, job.jobId, failureFingerprint(text)) + 1,
|
|
283
|
+
maxSameFingerprint: config.maxSameFingerprint,
|
|
284
|
+
});
|
|
245
285
|
if (alreadyRecorded(ledger, pending))
|
|
246
|
-
return { ledger, created: false };
|
|
286
|
+
return { ledger, created: false, decision };
|
|
247
287
|
const attempt = {
|
|
248
288
|
n: ledger.attempts.length + 1,
|
|
249
289
|
at: now.toISOString(),
|
|
@@ -253,16 +293,25 @@ function recordFailure(config, jobId, existing, pending, now) {
|
|
|
253
293
|
detail: pending.detail,
|
|
254
294
|
};
|
|
255
295
|
const attempts = [...ledger.attempts, attempt];
|
|
256
|
-
const
|
|
257
|
-
const next = exhausted ? undefined : new Date(now.getTime() + retryDelayMs(attempts.length)).toISOString();
|
|
296
|
+
const next = decision.retry ? new Date(now.getTime() + retryDelayMs(attempts.length)).toISOString() : undefined;
|
|
258
297
|
const written = {
|
|
259
|
-
jobId,
|
|
298
|
+
jobId: job.jobId,
|
|
260
299
|
attempts,
|
|
261
300
|
...(next ? { nextRetryAt: next } : {}),
|
|
262
301
|
...(ledger.ignoredRunIds && ledger.ignoredRunIds.length > 0 ? { ignoredRunIds: ledger.ignoredRunIds } : {}),
|
|
263
302
|
};
|
|
264
303
|
writeLedger(config.kilnDir, written);
|
|
265
|
-
return { ledger: written, created: true };
|
|
304
|
+
return { ledger: written, created: true, decision };
|
|
305
|
+
}
|
|
306
|
+
function failureText(job, pending) {
|
|
307
|
+
const parts = [pending.detail];
|
|
308
|
+
if (job.run?.sandboxError)
|
|
309
|
+
parts.push(job.run.sandboxError);
|
|
310
|
+
for (const event of job.run?.stepEvents ?? []) {
|
|
311
|
+
if (event.detail)
|
|
312
|
+
parts.push(event.detail);
|
|
313
|
+
}
|
|
314
|
+
return parts.filter((part) => part.length > 0).join("\n");
|
|
266
315
|
}
|
|
267
316
|
function alreadyRecorded(ledger, pending) {
|
|
268
317
|
if (pending.runId)
|
package/dist/report.js
CHANGED
|
@@ -159,6 +159,10 @@ export function statusDetails(job) {
|
|
|
159
159
|
lines.push(`feedback ${job.feedbackRounds?.length}`);
|
|
160
160
|
if (job.state === "needs-human" && job.feedbackReason)
|
|
161
161
|
lines.push(`reason ${job.feedbackReason}`);
|
|
162
|
+
if (job.stopReason)
|
|
163
|
+
lines.push(`reason ${job.stopReason}`);
|
|
164
|
+
if (job.waitReason)
|
|
165
|
+
lines.push(job.waitReason);
|
|
162
166
|
if (job.attemptCount > 0)
|
|
163
167
|
lines.push(`attempts ${job.attemptCount}`);
|
|
164
168
|
if (job.lastFailureReason)
|
|
@@ -167,8 +171,9 @@ export function statusDetails(job) {
|
|
|
167
171
|
lines.push(AUTH_FAILED_LINE);
|
|
168
172
|
if (job.nextRetryAt)
|
|
169
173
|
lines.push(`next retry ${job.nextRetryAt}`);
|
|
170
|
-
if (job.state === "dead" && job.attemptCount > 0)
|
|
174
|
+
if ((job.state === "dead" || job.stopReason) && job.attemptCount > 0 && job.gates.length === 0) {
|
|
171
175
|
lines.push(`retry: kiln retry ${job.jobId}`);
|
|
176
|
+
}
|
|
172
177
|
if (job.noReadableApproval)
|
|
173
178
|
lines.push("no readable approval decision");
|
|
174
179
|
if (job.cancelCommand)
|
|
@@ -186,6 +191,9 @@ export function statusDetails(job) {
|
|
|
186
191
|
lines.push(`approve: kiln approve ${job.jobId}`);
|
|
187
192
|
if (gate.choices.includes("reject"))
|
|
188
193
|
lines.push(`reject: kiln reject ${job.jobId}`);
|
|
194
|
+
if (gate.choices.includes("approve") || gate.choices.includes("revise")) {
|
|
195
|
+
lines.push(`revise: kiln revise ${job.jobId} --note <text>`);
|
|
196
|
+
}
|
|
189
197
|
}
|
|
190
198
|
}
|
|
191
199
|
return lines;
|
|
@@ -216,7 +224,7 @@ function toStatusJob(job, observation) {
|
|
|
216
224
|
...(job.findingsPath ? { findingsPath: job.findingsPath } : {}),
|
|
217
225
|
launchFailed: job.launchFailed,
|
|
218
226
|
attemptCount: job.attemptCount ?? 0,
|
|
219
|
-
maxAttempts: observation.maxAttempts,
|
|
227
|
+
maxAttempts: job.attemptCap ?? observation.maxAttempts,
|
|
220
228
|
...(job.lastFailureReason ? { lastFailureReason: job.lastFailureReason } : {}),
|
|
221
229
|
...(job.nextRetryAt ? { nextRetryAt: job.nextRetryAt } : {}),
|
|
222
230
|
invalidReason: job.error,
|
|
@@ -226,6 +234,7 @@ function toStatusJob(job, observation) {
|
|
|
226
234
|
gates: [],
|
|
227
235
|
canApprove: false,
|
|
228
236
|
canReject: false,
|
|
237
|
+
canRevise: false,
|
|
229
238
|
steps: progressFor(job),
|
|
230
239
|
...(job.deliver ? { deliver: job.deliver } : {}),
|
|
231
240
|
...(job.createdAt ? { createdAt: job.createdAt } : {}),
|
|
@@ -239,7 +248,7 @@ function toStatusJob(job, observation) {
|
|
|
239
248
|
: job.feedbackAmbiguous
|
|
240
249
|
? "feedback-ambiguous"
|
|
241
250
|
: undefined;
|
|
242
|
-
const gates = state === "needs-human" && job.run && !feedbackReason ? job.run.gates.map(toGateView) : [];
|
|
251
|
+
const gates = state === "needs-human" && job.run && !feedbackReason && !job.stopReason ? job.run.gates.map(toGateView) : [];
|
|
243
252
|
const prNumber = job.delivery?.prNumber;
|
|
244
253
|
const runId = job.run?.runId;
|
|
245
254
|
const showBranch = runId !== undefined && (state === "done" || state === "in-review" || feedbackReason !== undefined);
|
|
@@ -263,8 +272,12 @@ function toStatusJob(job, observation) {
|
|
|
263
272
|
...(job.findingsPath ? { findingsPath: job.findingsPath } : {}),
|
|
264
273
|
launchFailed: job.launchFailed,
|
|
265
274
|
attemptCount: job.attemptCount ?? 0,
|
|
266
|
-
maxAttempts: observation.maxAttempts,
|
|
275
|
+
maxAttempts: job.attemptCap ?? observation.maxAttempts,
|
|
267
276
|
...(job.lastFailureReason ? { lastFailureReason: job.lastFailureReason } : {}),
|
|
277
|
+
...(job.stopReason ? { stopReason: job.stopReason } : {}),
|
|
278
|
+
...(observation.holdDetail && (state === "queued" || state === "blocked" || state === "failed")
|
|
279
|
+
? { waitReason: observation.holdDetail }
|
|
280
|
+
: {}),
|
|
268
281
|
...(job.nextRetryAt && !job.retriesExhausted ? { nextRetryAt: job.nextRetryAt } : {}),
|
|
269
282
|
...(job.ledgerError ? { ledgerError: job.ledgerError } : {}),
|
|
270
283
|
...(job.feedbackLedgerError ? { feedbackLedgerError: job.feedbackLedgerError } : {}),
|
|
@@ -281,6 +294,7 @@ function toStatusJob(job, observation) {
|
|
|
281
294
|
gates,
|
|
282
295
|
canApprove: gates.some((gate) => gate.choices.includes("approve")),
|
|
283
296
|
canReject: gates.some((gate) => gate.choices.includes("reject")),
|
|
297
|
+
canRevise: gates.some((gate) => gate.choices.includes("approve") || gate.choices.includes("revise")),
|
|
284
298
|
steps: progressFor(job),
|
|
285
299
|
...(job.createdAt ? { createdAt: job.createdAt } : {}),
|
|
286
300
|
...(job.run?.startedAt ? { startedAt: job.run.startedAt } : {}),
|
|
@@ -402,7 +416,7 @@ export function renderTick(observation, actions, launches, delivers = [], findin
|
|
|
402
416
|
: `findings ${recorded.jobId} missing`);
|
|
403
417
|
}
|
|
404
418
|
for (const note of notes) {
|
|
405
|
-
const next = note.nextRetryAt ? ` next ${note.nextRetryAt}` : note.
|
|
419
|
+
const next = note.nextRetryAt ? ` next ${note.nextRetryAt}` : note.needsHuman ? " needs human" : "";
|
|
406
420
|
lines.push(`${note.kind} ${note.jobId} ${note.reason} attempt ${note.attempt}${next}`);
|
|
407
421
|
}
|
|
408
422
|
for (const note of feedbackNotes) {
|
package/dist/schemas.js
CHANGED
|
@@ -36,6 +36,18 @@ export const RepoEntrySchema = z.object({
|
|
|
36
36
|
* The local branch stays `yak/<run-id>`.
|
|
37
37
|
*/
|
|
38
38
|
branch: z.string().min(1, "Branch template is empty.").optional(),
|
|
39
|
+
/** Overrides the global attempt cap for jobs in this repo. */
|
|
40
|
+
maxAttempts: z
|
|
41
|
+
.number()
|
|
42
|
+
.int("Max attempts must be a whole number.")
|
|
43
|
+
.min(1, "Max attempts must be at least 1.")
|
|
44
|
+
.optional(),
|
|
45
|
+
/** Overrides the global fix-round cap for jobs in this repo. */
|
|
46
|
+
maxFixRounds: z
|
|
47
|
+
.number()
|
|
48
|
+
.int("Max fix rounds must be a whole number.")
|
|
49
|
+
.min(1, "Max fix rounds must be at least 1.")
|
|
50
|
+
.optional(),
|
|
39
51
|
});
|
|
40
52
|
/** Repo map shared by the config file and the settings API. */
|
|
41
53
|
export const KilnReposSchema = z.record(z.string().regex(REPO_NAME, "must start with a letter or digit and contain only letters, digits, dots, underscores, and hyphens"), RepoEntrySchema);
|
|
@@ -75,21 +87,34 @@ export const EditableSandboxSchema = SandboxSettingsSchema.pick({
|
|
|
75
87
|
const WORKFLOW_KEY = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
76
88
|
/** No yak event and no log output for this long stalls a running job. `<= 0` disables it. */
|
|
77
89
|
export const DEFAULT_STALL_TIMEOUT_MS = 15 * 60 * 1000;
|
|
78
|
-
/** Transient failures retry until this many attempts, then the job
|
|
90
|
+
/** Transient failures retry until this many attempts, then the job needs a person. */
|
|
79
91
|
export const DEFAULT_MAX_ATTEMPTS = 3;
|
|
80
92
|
/** Pull request fix rounds before the job is needs-human (`feedback-exhausted`). */
|
|
81
93
|
export const DEFAULT_MAX_FIX_ROUNDS = 2;
|
|
94
|
+
/** Identical failure fingerprints before the job needs a person. */
|
|
95
|
+
export const DEFAULT_MAX_SAME_FINGERPRINT = 3;
|
|
82
96
|
/**
|
|
83
97
|
* Milliseconds without a yak event or log output before a running job is
|
|
84
98
|
* killed and retried. Default 15 minutes. `0` or negative disables stall detection.
|
|
85
99
|
*/
|
|
86
100
|
const stallTimeoutMs = z.number().int("Stall timeout must be a whole number of milliseconds.").optional();
|
|
87
|
-
/** Transient failures before the job
|
|
101
|
+
/** Transient failures before the job needs a person. Default 3. Hitting the cap is a budget failure. */
|
|
88
102
|
const maxAttempts = z
|
|
89
103
|
.number()
|
|
90
104
|
.int("Max attempts must be a whole number.")
|
|
91
105
|
.min(1, "Max attempts must be at least 1.")
|
|
92
106
|
.optional();
|
|
107
|
+
/** Same failure fingerprint before Needs human. Default 3. */
|
|
108
|
+
const maxSameFingerprint = z
|
|
109
|
+
.number()
|
|
110
|
+
.int("Fingerprint limit must be a whole number.")
|
|
111
|
+
.min(1, "Fingerprint limit must be at least 1.")
|
|
112
|
+
.optional();
|
|
113
|
+
/**
|
|
114
|
+
* Dollars of recorded run cost allowed on the local calendar day.
|
|
115
|
+
* Omitted means no cap. `0` blocks new runs immediately.
|
|
116
|
+
*/
|
|
117
|
+
const dailyDollarCap = z.number().nonnegative("Daily dollar cap cannot be negative.").optional();
|
|
93
118
|
/**
|
|
94
119
|
* Fix rounds for an open pull request before the job is needs-human with
|
|
95
120
|
* reason `feedback-exhausted`. Default 2. Kiln does not merge.
|
|
@@ -109,6 +134,8 @@ export const EditableKilnSettingsSchema = z.object({
|
|
|
109
134
|
stallTimeoutMs,
|
|
110
135
|
maxAttempts,
|
|
111
136
|
maxFixRounds,
|
|
137
|
+
maxSameFingerprint,
|
|
138
|
+
dailyDollarCap,
|
|
112
139
|
sandbox: EditableSandboxSchema.optional(),
|
|
113
140
|
});
|
|
114
141
|
/** PUT /api/settings body. `revision` is the sha256 of the file the client loaded. */
|
|
@@ -121,6 +148,8 @@ export const KilnConfigFileSchema = z.object({
|
|
|
121
148
|
stallTimeoutMs,
|
|
122
149
|
maxAttempts,
|
|
123
150
|
maxFixRounds,
|
|
151
|
+
maxSameFingerprint,
|
|
152
|
+
dailyDollarCap,
|
|
124
153
|
adapter: z.enum(["mock", "claude-code"]).optional(),
|
|
125
154
|
sandbox: SandboxSettingsSchema.optional(),
|
|
126
155
|
/** Per-workflow sandbox override. A set field replaces the global one; an omitted field inherits. */
|
|
@@ -176,9 +205,10 @@ export const JobSchema = z.object({
|
|
|
176
205
|
* `deliver=pr`. It does not enter `in-review`. A reject, a failure, a dead
|
|
177
206
|
* process, and an ok run with no readable approval do not unblock an
|
|
178
207
|
* after-merged dependent. A stacked job unblocks when its one upstream has
|
|
179
|
-
* landed, which includes `in-review`.
|
|
180
|
-
*
|
|
181
|
-
*
|
|
208
|
+
* landed, which includes `in-review`. A job that has used its attempt cap,
|
|
209
|
+
* fix-round cap, or fingerprint breaker is `needs-human`. `dead` is a run
|
|
210
|
+
* whose process died and has not been recorded yet. Attempts live under
|
|
211
|
+
* the kiln directory, not in this file.
|
|
182
212
|
*/
|
|
183
213
|
export const JOB_STATES = [
|
|
184
214
|
"queued",
|