@hone-ai/cli 1.38.1 → 1.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/hone-cli.js +218 -157
- package/lib/anthropic-response.js +58 -0
- package/lib/batch-follow.js +641 -53
- package/lib/block-grammar.js +55 -0
- package/lib/branch-protection.js +141 -25
- package/lib/bundle-paths.js +77 -13
- package/lib/call-sites.js +489 -0
- package/lib/classify-spec.js +68 -0
- package/lib/context-reserve.js +270 -0
- package/lib/detect-test-framework.js +54 -0
- package/lib/hard-pause-remedy.js +1 -1
- package/lib/judge-provider.js +20 -11
- package/lib/learnings-sync.js +2 -1
- package/lib/release-review-config.js +3 -2
- package/lib/required-checks.js +354 -0
- package/lib/reviewer-optin.js +115 -0
- package/lib/run-context.js +21 -0
- package/lib/security-scanner.js +7 -4
- package/lib/story-line.js +48 -12
- package/lib/verify-patch.js +122 -12
- package/package.json +1 -1
package/lib/batch-follow.js
CHANGED
|
@@ -14,12 +14,39 @@
|
|
|
14
14
|
* - The loop never approves and never posts a verdict (`verify-patch --report` does). It never passes
|
|
15
15
|
* `--trust-source`, `--skip-verify`, `--ready` or `--acknowledge-*`: an untrusted story stays paused for a
|
|
16
16
|
* person (HC-GF-270), and so does one whose repository never consented to running its tests (HC-GF-064).
|
|
17
|
+
* - HC-GF-363: under `verification.gate: enforce` the server refuses to approve a step_4 attempt whose
|
|
18
|
+
* verify-patch verdict is `fail`, and the loop used to stop there (exit 2). It now sends that one case back
|
|
19
|
+
* to the builder: POST /orchestrate/:runId/reject at step_4, with the measured failing tests as the reason,
|
|
20
|
+
* at most AUTO_REJECT_LIMIT (2) times per run. It rejects ONLY when every one of these holds: the gate is
|
|
21
|
+
* enforce for verify-patch, the server refuses approval, the latest completed step_4 attempt's LATEST
|
|
22
|
+
* verify-patch record says `fail` (not timeout, unresolved, not_run, error, pass_narrowed), the run's error
|
|
23
|
+
* says the same, and fewer than two earlier attempts carry AUTO_REJECT_PREFIX in their reject reason. The
|
|
24
|
+
* count is read from the server's rows on every decision, never kept here, so a restart cannot reset it; a
|
|
25
|
+
* read that fails means "unknown", and unknown never rejects. The two rejects spend step_4's re-execution
|
|
26
|
+
* budget (`max_retries_per_step`, default 2): a later step_5 route-back can pause MAX_RETRIES_EXCEEDED.
|
|
27
|
+
* A run stopped at the limit is named for a person (step, attempts, count, last failure) and the loop exits 2.
|
|
28
|
+
* Upgrading changes one thing: a run already paused on a refused `fail` is rejected on the first tick.
|
|
17
29
|
* - It stops BEFORE acting when a breaker has fired. A terminal batch ends the loop after one last emit-pr pass.
|
|
18
30
|
* - It is idempotent across restarts: an attempt that already has a verify-patch verdict on the server is not
|
|
19
31
|
* verified again, and emit-pr's BRANCH_CONFLICT (5) means the branch was already emitted.
|
|
32
|
+
* - HC-GF-351: it is never silent. A run paused at anything the loop cannot act on is named once, with its step
|
|
33
|
+
* and the first line of why; when every unfinished run is waiting on a person the loop stops with NEEDS_YOU
|
|
34
|
+
* instead of waiting out its deadline. And a run hard-paused for files it was not given is re-served ONCE: the
|
|
35
|
+
* paused run is killed with a reason that names the new batch (the next start reads it back, which is what
|
|
36
|
+
* makes the bound survive a restart), and its story is queued again as a one-story batch whose line carries
|
|
37
|
+
* the files, which the loop then follows too. Only paths cli/lib/context-reserve.js lets through are served.
|
|
38
|
+
* Killing a run FAILS every queued or paused run in its batch that lists it in `config.dependsOnRunIds`
|
|
39
|
+
* (server/src/services/batch-deps.js cascadeOnFailure), so a run anything depends on is never re-served, and the
|
|
40
|
+
* decision reads that list, not the dependents' waiting message, which names only one of their blockers.
|
|
20
41
|
*/
|
|
21
42
|
|
|
22
43
|
const { classifyPollState, POLL_OUTCOMES } = require('./poll-decision');
|
|
44
|
+
const { routerSafe } = require('./verify-patch');
|
|
45
|
+
const { formatStoryLine } = require('./story-line');
|
|
46
|
+
const { isGitRev, isStoryId } = require('./run-target');
|
|
47
|
+
const {
|
|
48
|
+
reserveReason, reservedBatchOf, hardPauseRequest, judgeRequest, originalPathsOf, batchIdFromQueueOutput,
|
|
49
|
+
} = require('./context-reserve');
|
|
23
50
|
|
|
24
51
|
/** follow-batch's own exit codes. */
|
|
25
52
|
const EXIT = Object.freeze({ DONE: 0, FATAL: 1, NEEDS_YOU: 2, DEADLINE: 3 });
|
|
@@ -78,10 +105,26 @@ function isTerminal(batch) {
|
|
|
78
105
|
return !!(batch && batch.status && batch.status !== 'active');
|
|
79
106
|
}
|
|
80
107
|
|
|
81
|
-
/**
|
|
108
|
+
/**
|
|
109
|
+
* The verify-patch verdict the server holds for this run's step_4 attempt, or null: the LATEST record for it, as the
|
|
110
|
+
* server reads them (verdictForAttempt takes the last by reportedAt). A record names its attempt, or carries
|
|
111
|
+
* `forAttempt` (the latest completed attempt when it arrived). HC-GF-363: this used to return the FIRST match, so an
|
|
112
|
+
* attempt that failed and then passed read as failed.
|
|
113
|
+
*/
|
|
82
114
|
function priorVerdict(run, attempt) {
|
|
83
115
|
const list = Array.isArray(run && run.verifications) ? run.verifications : [];
|
|
84
|
-
|
|
116
|
+
const forThis = (v) => v && v.source === 'verify-patch' && v.stepKey === 'step_4'
|
|
117
|
+
&& (v.attempt != null ? Number(v.attempt) === attempt : v.forAttempt != null && Number(v.forAttempt) === attempt);
|
|
118
|
+
let best = null;
|
|
119
|
+
let bestAt = -Infinity;
|
|
120
|
+
for (const v of list) {
|
|
121
|
+
if (!forThis(v)) continue;
|
|
122
|
+
const at = Date.parse(v.reportedAt);
|
|
123
|
+
const t = Number.isFinite(at) ? at : -Infinity;
|
|
124
|
+
// Later by clock wins; equal or unreadable clocks fall back to list order (the server appends).
|
|
125
|
+
if (best === null || t >= bestAt) { best = v; bestAt = t; }
|
|
126
|
+
}
|
|
127
|
+
return best;
|
|
85
128
|
}
|
|
86
129
|
|
|
87
130
|
/**
|
|
@@ -91,7 +134,7 @@ function priorVerdict(run, attempt) {
|
|
|
91
134
|
*/
|
|
92
135
|
function step4Action(run) {
|
|
93
136
|
const c = classifyPollState(run);
|
|
94
|
-
// Not a step_4 gate: not this
|
|
137
|
+
// Not a step_4 gate: not this function's business. planTick asks otherPause who such a pause waits on (HC-GF-351).
|
|
95
138
|
if (c.outcome !== POLL_OUTCOMES.HUMAN_GATE || c.stepKey !== 'step_4') return { actionable: false };
|
|
96
139
|
const why = String((run && run.errorMessage) || '');
|
|
97
140
|
if (NOT_A_GATE.test(why)) return { actionable: false, reason: `paused, not at a gate: ${firstLine(why)}` };
|
|
@@ -109,6 +152,88 @@ function step4Action(run) {
|
|
|
109
152
|
return { actionable: true, attempt };
|
|
110
153
|
}
|
|
111
154
|
|
|
155
|
+
/** Marks a step_4 reject reason as this loop's own. It starts the reason (position 0), and is counted back from rows. */
|
|
156
|
+
const AUTO_REJECT_PREFIX = 'auto-rejected by follow-batch: ';
|
|
157
|
+
|
|
158
|
+
/** Automatic rejects per run. Each one is a full step_4 rebuild plus a verify-patch run. */
|
|
159
|
+
const AUTO_REJECT_LIMIT = 2;
|
|
160
|
+
|
|
161
|
+
/** The reason's tail, from the server-held verdict detail: one line, no control characters, bounded. */
|
|
162
|
+
const DETAIL_IN_REASON = 1500;
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* HC-GF-363: is this paused run a step_4 gate whose latest attempt FAILED verification under enforce, so that the
|
|
166
|
+
* loop may send it back? Every condition, none inferred from another (`approvalRefusal` alone is also set for a
|
|
167
|
+
* timeout, an unresolved or narrowed run, which are not measured failures and are never sent back):
|
|
168
|
+
* 1. a human gate at step_4 (not a breaker, budget, retry-limit or hard pause: condition 6 excludes those);
|
|
169
|
+
* 2. verification.gate is enforce for verify-patch;
|
|
170
|
+
* 3. the server refuses approval (`approvalRefusal`);
|
|
171
|
+
* 4. the latest step_4 attempt is completed (attempt L);
|
|
172
|
+
* 5. the LATEST verify-patch record for L has verdict exactly `fail`;
|
|
173
|
+
* 6. the run's error message is the one the enforce red branch writes for a `fail`.
|
|
174
|
+
* The count of earlier rejects (7) is read from the server by the caller: see autoRejectDecision.
|
|
175
|
+
* @returns {{attempt: number, detail: string}|null}
|
|
176
|
+
*/
|
|
177
|
+
function autoRejectCandidate(run) {
|
|
178
|
+
const c = classifyPollState(run);
|
|
179
|
+
if (c.outcome !== POLL_OUTCOMES.HUMAN_GATE || c.stepKey !== 'step_4') return null;
|
|
180
|
+
// Not a breaker, budget, retry-limit or hard pause (NOT_A_GATE): each carries its own error prefix, so the
|
|
181
|
+
// enforce-fail message required below already excludes them, and tests/…/batch-follow-reject.test.js drives them.
|
|
182
|
+
const why = String((run && run.errorMessage) || '');
|
|
183
|
+
if (!gateEnforced(run.config, 'verify-patch')) return null;
|
|
184
|
+
if (!run.approvalRefusal) return null;
|
|
185
|
+
const attempt = currentStep4Attempt(run);
|
|
186
|
+
if (!attempt) return null;
|
|
187
|
+
const row = run.steps.find((st) => st && st.stepKey === 'step_4' && (Number(st.attempt) || 1) === attempt);
|
|
188
|
+
if (!row || row.status !== 'completed') return null;
|
|
189
|
+
// A hard pause (files the step asked for) that a fail verdict then overwrote the error message of: HC-GF-351's
|
|
190
|
+
// re-serve owns it, not this reject.
|
|
191
|
+
const fr = row.diagnostics && row.diagnostics.failure_record;
|
|
192
|
+
if (fr && fr.context_requests && typeof fr.context_requests === 'object') return null;
|
|
193
|
+
const v = priorVerdict(run, attempt);
|
|
194
|
+
if (!v || v.verdict !== 'fail') return null;
|
|
195
|
+
if (!why.startsWith("verify verdict 'fail' at step_4")) return null;
|
|
196
|
+
return { attempt, detail: String(v.detail == null ? '' : v.detail) };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** The attempts (1-based, of `contexts`) whose reject reason starts with the prefix. */
|
|
200
|
+
function autoRejectedAttempts(contexts) {
|
|
201
|
+
const out = [];
|
|
202
|
+
(Array.isArray(contexts) ? contexts : []).forEach((ctx, i) => {
|
|
203
|
+
if (typeof ctx === 'string' && ctx.startsWith(AUTO_REJECT_PREFIX)) out.push(i + 1);
|
|
204
|
+
});
|
|
205
|
+
return out;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* HC-GF-363: what to do about a paused run, given each earlier step_4 attempt's `errorContext`. Pure.
|
|
210
|
+
* @param {object} run a fresh `GET /orchestrate/:runId`
|
|
211
|
+
* @param {Array<string|null>|null} contexts `errorContext` of attempts 1..L-1, in order; null when any read failed
|
|
212
|
+
* @returns {{kind: 'none'} | {kind: 'unknown', attempt: number, why: string}
|
|
213
|
+
* | {kind: 'exhausted', attempt: number, count: number, rejected: number[], note: string}
|
|
214
|
+
* | {kind: 'reject', attempt: number, n: number, reason: string}}
|
|
215
|
+
*/
|
|
216
|
+
function autoRejectDecision(run, contexts) {
|
|
217
|
+
const cand = autoRejectCandidate(run);
|
|
218
|
+
if (!cand) return { kind: 'none' };
|
|
219
|
+
const L = cand.attempt;
|
|
220
|
+
if (!Array.isArray(contexts) || contexts.length !== L - 1) {
|
|
221
|
+
return { kind: 'unknown', attempt: L, why: 'the earlier attempts could not all be read, so the rejects already used are unknown' };
|
|
222
|
+
}
|
|
223
|
+
const rejected = autoRejectedAttempts(contexts);
|
|
224
|
+
const count = rejected.length;
|
|
225
|
+
const clean = routerSafe(cand.detail).replace(/[\u0000-\u001f\u007f-\u009f]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, DETAIL_IN_REASON);
|
|
226
|
+
if (count >= AUTO_REJECT_LIMIT) {
|
|
227
|
+
const budget = contexts.some((x) => /MAX_RETRIES_EXCEEDED/.test(String(x || ''))) || /MAX_RETRIES_EXCEEDED/.test(String(run.errorMessage || ''))
|
|
228
|
+
? ' MAX_RETRIES_EXCEEDED was seen: the rejects use step_4\'s retry budget.' : '';
|
|
229
|
+
return { kind: 'exhausted', attempt: L, count, rejected,
|
|
230
|
+
note: `step_4 attempt ${L} failed verify-patch; ${count} automatic reject(s) used (attempt${rejected.length === 1 ? '' : 's'} ${rejected.join(', ')}); `
|
|
231
|
+
+ `last failure: ${clean.slice(0, 200) || '(no detail)'}.${budget} Read it: hone run-report ${run.runId || '<runId>'}` };
|
|
232
|
+
}
|
|
233
|
+
return { kind: 'reject', attempt: L, n: count + 1,
|
|
234
|
+
reason: `${AUTO_REJECT_PREFIX}attempt ${L} failed verify-patch under verification.gate: enforce (automatic reject ${count + 1} of ${AUTO_REJECT_LIMIT}). ${clean || '(no detail)'}` };
|
|
235
|
+
}
|
|
236
|
+
|
|
112
237
|
/**
|
|
113
238
|
* The argv for one child verb. Exit codes are the contract; `--format json` only carries detail.
|
|
114
239
|
* @param {'check-patch'|'verify-patch'|'emit-pr'} verb
|
|
@@ -124,6 +249,20 @@ function buildArgs(verb, runId, { attempt, openPr = false, branch } = {}) {
|
|
|
124
249
|
return args;
|
|
125
250
|
}
|
|
126
251
|
|
|
252
|
+
/**
|
|
253
|
+
* The argv that queues one story again (HC-GF-351). The file, repository and branch are values from outside this
|
|
254
|
+
* module, so they ride in `--flag=value` form: a value that starts with a dash cannot become an option. No flag
|
|
255
|
+
* here overrides a safety gate, and none is `--overnight` (the loop does not know the original batch's mode).
|
|
256
|
+
* @param {string} file a stories file holding the one line
|
|
257
|
+
*/
|
|
258
|
+
function buildReserveArgs(file, { repoName, branch } = {}) {
|
|
259
|
+
const args = ['queue-stories', `--file=${file}`];
|
|
260
|
+
if (repoName) args.push(`--repo=${repoName}`);
|
|
261
|
+
if (branch && isGitRev(branch)) args.push(`--branch=${branch}`);
|
|
262
|
+
args.push('--format', 'json');
|
|
263
|
+
return args;
|
|
264
|
+
}
|
|
265
|
+
|
|
127
266
|
/** The step_4 attempt a run is on now: the highest step_4 attempt in its steps, or null. */
|
|
128
267
|
function currentStep4Attempt(run) {
|
|
129
268
|
const steps = Array.isArray(run && run.steps) ? run.steps : [];
|
|
@@ -147,25 +286,207 @@ function draftBranch(storyId, runId, attempt) {
|
|
|
147
286
|
return `hone/${storyId || 'story'}-${String(runId).slice(0, 8)}-a${attempt}`;
|
|
148
287
|
}
|
|
149
288
|
|
|
289
|
+
/** One line of agent- or server-written text, safe to print: control characters (a terminal escape) become spaces. */
|
|
290
|
+
const oneLine = (s, max = 240) => firstLine(String(s || '')).replace(/[\u0000-\u001f\u007f-\u009f]/g, ' ').trim().slice(0, max);
|
|
291
|
+
|
|
292
|
+
/** Polls an accepted reject may wait for the worker before a person is told. */
|
|
293
|
+
const REJECT_WAIT_TICKS = 20;
|
|
294
|
+
|
|
295
|
+
/** The statuses a batch child ends in. Anything else is still moving as far as the loop can tell. */
|
|
296
|
+
const TERMINAL_RUN = new Set(['completed', 'failed', 'killed', 'stalled']);
|
|
297
|
+
|
|
298
|
+
/** The shape every planTick return carries, so a caller never meets an absent field. `stop` is always the caller's. */
|
|
299
|
+
const tickPlan = (over) => ({ actions: [], notes: [], waiting: [], moving: 0, follow: [], rejecting: [], ...over });
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* Batches the server's own rows say were re-served: a killed run whose reason names the batch its story went on in.
|
|
303
|
+
* @param {object} batch a `GET /orchestrate/batch/:id` payload
|
|
304
|
+
* @returns {Array<{storyId: string, batchId: string}>}
|
|
305
|
+
*/
|
|
306
|
+
function reservedFrom(batch) {
|
|
307
|
+
const out = [];
|
|
308
|
+
for (const r of Array.isArray(batch && batch.runs) ? batch.runs : []) {
|
|
309
|
+
const batchId = reservedBatchOf(r);
|
|
310
|
+
if (batchId) out.push({ storyId: r.storyId, batchId });
|
|
311
|
+
}
|
|
312
|
+
return out;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* A paused run that is not at a step_4 gate and has no request to serve: who is it waiting on?
|
|
317
|
+
* `person`: a gate or a refusal only a human clears. `dependency`: it will wake when the run it waits on ends.
|
|
318
|
+
* `moving`: paused without a gate or a reason the loop knows (the server decides what is next); a poll more costs
|
|
319
|
+
* an interval, and calling it stuck would end a live run.
|
|
320
|
+
* @returns {{kind: 'person', stepKey: string|null, reason: string}|{kind: 'dependency'}|{kind: 'moving'}}
|
|
321
|
+
*/
|
|
322
|
+
function otherPause(run, row) {
|
|
323
|
+
const msg = String((run && run.errorMessage) || '');
|
|
324
|
+
const stepKey = (run && run.currentStep) || (row && row.currentStep) || null;
|
|
325
|
+
if (/^BATCH_DEPENDENCY_WAITING:/.test(msg)) return { kind: 'dependency' };
|
|
326
|
+
const c = classifyPollState(run);
|
|
327
|
+
if (c.outcome === POLL_OUTCOMES.EDITOR_PARKED) {
|
|
328
|
+
return { kind: 'person', stepKey, reason: 'parked for your editor (execution.mode: editor): submit its output, then approve' };
|
|
329
|
+
}
|
|
330
|
+
if (c.outcome === POLL_OUTCOMES.EXHAUSTED || NOT_A_GATE.test(msg)) return { kind: 'person', stepKey, reason: oneLine(msg) };
|
|
331
|
+
if (c.outcome === POLL_OUTCOMES.HUMAN_GATE) {
|
|
332
|
+
return { kind: 'person', stepKey, reason: msg ? oneLine(msg) : `awaiting your approval: hone run-story ${run.runId} --approve ${stepKey}` };
|
|
333
|
+
}
|
|
334
|
+
return { kind: 'moving' };
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* True when a verdict from `source` drives the step_4 gate on this run. Mirrors isEnforced in
|
|
339
|
+
* server/src/services/verification-refusal.js (the CLI does not ship server code); tests/regression/HC-GF-351 pins
|
|
340
|
+
* the two equal.
|
|
341
|
+
*/
|
|
342
|
+
function gateEnforced(config, source = 'verify-patch') {
|
|
343
|
+
const mode = String((config && config.verification_gate) || 'advisory').toLowerCase();
|
|
344
|
+
const sources = config && Array.isArray(config.verification_gate_sources) && config.verification_gate_sources.length
|
|
345
|
+
? config.verification_gate_sources : ['verify-patch'];
|
|
346
|
+
return mode === 'enforce' && sources.includes(source);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* A paused step_4 gate whose attempt passed verify-patch under `verification_gate: enforce`. The pass only QUEUES the
|
|
351
|
+
* approval (POST /:runId/verdict → enqueueApproval); the run stays paused at step_4 until a worker takes the job. It
|
|
352
|
+
* is moving, not waiting on a person, and saying "approve it" in that window sends the operator to a gate that is
|
|
353
|
+
* already being approved.
|
|
354
|
+
*/
|
|
355
|
+
function approvalQueued(run) {
|
|
356
|
+
const why = String((run && run.errorMessage) || '');
|
|
357
|
+
return !!run && run.currentStep === 'step_4' && !NOT_A_GATE.test(why) && !run.approvalRefusal
|
|
358
|
+
&& gateEnforced(run.config) && verifiedStep4Attempt(run) !== null;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/** step_0 → 0, step_4 → 400, step_5b → 502: the order steps run in, for "after step_4". NaN for a key it cannot read. */
|
|
362
|
+
function stepRank(stepKey) {
|
|
363
|
+
const m = String(stepKey).match(/^step_(\d+)([a-z]?)$/);
|
|
364
|
+
return m ? Number(m[1]) * 100 + (m[2] ? m[2].charCodeAt(0) - 96 : 0) : NaN;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/**
|
|
368
|
+
* The queued or paused runs of this batch that killing `runId` would fail: those whose `config.dependsOnRunIds` lists
|
|
369
|
+
* it, or whose waiting message names it. `unread` are queued or paused runs whose config was not read, so nothing is
|
|
370
|
+
* known about them.
|
|
371
|
+
* @returns {{found: string[], unread: string[]}} story ids
|
|
372
|
+
*/
|
|
373
|
+
function dependentsOf(runId, runs, details) {
|
|
374
|
+
const found = [];
|
|
375
|
+
const unread = [];
|
|
376
|
+
for (const x of runs) {
|
|
377
|
+
if (!x || x.runId === runId || (x.status !== 'queued' && x.status !== 'paused')) continue;
|
|
378
|
+
const d = details[x.runId];
|
|
379
|
+
const listed = !!d && !!d.config && Array.isArray(d.config.dependsOnRunIds) && d.config.dependsOnRunIds.includes(runId);
|
|
380
|
+
const named = String(x.errorMessage || (d && d.errorMessage) || '').startsWith(`BATCH_DEPENDENCY_WAITING:${runId}`);
|
|
381
|
+
if (listed || named) found.push(x.storyId);
|
|
382
|
+
else if (!d) unread.push(x.storyId);
|
|
383
|
+
}
|
|
384
|
+
return { found, unread };
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/** Why a story that was served before is not served again, in the words of what actually happened. */
|
|
388
|
+
function priorWhy(prior, runId) {
|
|
389
|
+
const b = prior.batchId;
|
|
390
|
+
if (prior.state === 'kill-failed') {
|
|
391
|
+
return `it was queued again as batch ${b}, but this run could not be killed; kill it (hone run-story ${runId} --kill) once you have looked at that batch. Nothing more is tried`;
|
|
392
|
+
}
|
|
393
|
+
if (prior.state === 'adopted') {
|
|
394
|
+
return `a newer run of this story already exists${b ? ` (batch ${b})` : ''}, queued by an earlier re-serve or by you, so it is not re-served again`;
|
|
395
|
+
}
|
|
396
|
+
if (prior.state === 'queue-failed') return 're-serving it failed earlier in this run; nothing more is tried';
|
|
397
|
+
return `it names files again after being re-served as batch ${b}; a second re-serve is not tried`;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/**
|
|
401
|
+
* HC-GF-351: may this hard-paused run be re-served now? One of
|
|
402
|
+
* - `{ action }`: yes, here is the action;
|
|
403
|
+
* - `{ retry }`: not this tick, something transient (the text says what), and the run is still moving;
|
|
404
|
+
* - `{ why }`: no, and a person has to look.
|
|
405
|
+
*/
|
|
406
|
+
function decideReserve({ r, d, request, runs, details, ctx, prior, planned, batchId }) {
|
|
407
|
+
if (prior) return { why: priorWhy(prior, r.runId) };
|
|
408
|
+
if (planned.has(r.storyId)) return { why: 'the same story is hard-paused twice in this batch; only one is re-served' };
|
|
409
|
+
const dep = dependentsOf(r.runId, runs, details);
|
|
410
|
+
if (dep.found.length) {
|
|
411
|
+
return { why: `${dep.found.length} run(s) in this batch depend on it (${dep.found.slice(0, 5).join(', ')}), and killing it fails them` };
|
|
412
|
+
}
|
|
413
|
+
if (dep.unread.length) {
|
|
414
|
+
return { why: `${dep.unread.length} queued or paused run(s) in this batch could not be checked for a dependency on it, and killing it fails its dependents` };
|
|
415
|
+
}
|
|
416
|
+
if (!isStoryId(r.storyId)) return { why: 'its story id cannot be written into a stories file' };
|
|
417
|
+
const rank = stepRank(request.stepKey);
|
|
418
|
+
const verified = verifiedStep4Attempt(d);
|
|
419
|
+
if (rank > 400 && verified !== null) {
|
|
420
|
+
return { why: `step_4 attempt ${verified} already passed verify-patch, and re-serving from ${request.stepKey} would throw that verified work away` };
|
|
421
|
+
}
|
|
422
|
+
if (ctx.tracked == null && ctx.trackedRetry) {
|
|
423
|
+
return { retry: `could not list the files git tracks (${oneLine(ctx.trackedRetry, 120)}); trying again next poll` };
|
|
424
|
+
}
|
|
425
|
+
const original = originalPathsOf(d);
|
|
426
|
+
if (original.dropped) {
|
|
427
|
+
return { why: `${original.dropped} path(s) on the story's own line hold characters a shell could misread; nothing was served` };
|
|
428
|
+
}
|
|
429
|
+
const judged = judgeRequest(request, { tracked: ctx.tracked, realPath: ctx.realPath, original: original.paths });
|
|
430
|
+
if (!judged.ok) return { why: judged.reason };
|
|
431
|
+
return { action: { kind: 'reserve', runId: r.runId, storyId: r.storyId, stepKey: request.stepKey, batchId, paths: judged.tokens,
|
|
432
|
+
since: d.createdAt || null, repoName: r.repoName || ctx.repoName || null, branch: r.branch || null } };
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/** What the loop says when every unfinished run is waiting on a person. */
|
|
436
|
+
function stuckReason(waiting) {
|
|
437
|
+
const who = waiting.map((w) => `${w.storyId} ${w.runId} at ${w.stepKey || 'a step'} (${oneLine(w.reason, 400)})`);
|
|
438
|
+
return `every unfinished run is waiting on you: ${who.join('; ')}`;
|
|
439
|
+
}
|
|
440
|
+
|
|
150
441
|
/**
|
|
151
442
|
* One tick's decisions, from the batch and the details of its paused runs. Pure.
|
|
443
|
+
*
|
|
444
|
+
* `stop.kind` says why the batch ends the loop: `breaker` (stop before acting), `terminal` (the server settled
|
|
445
|
+
* it), or `stuck` (HC-GF-351: nothing is running, nothing is left to do, and the unfinished runs wait on a person).
|
|
446
|
+
* `waiting` lists those runs, `moving` counts the ones still in motion, and `follow` the batches this one's killed
|
|
447
|
+
* runs were re-served as.
|
|
448
|
+
*
|
|
152
449
|
* @param {object} batch `GET /orchestrate/batch/:id`
|
|
153
450
|
* @param {Record<string, object>} details runId → `GET /orchestrate/:runId`, for paused runs and runs to emit
|
|
154
|
-
* @param {{emitted: Set<string
|
|
155
|
-
*
|
|
156
|
-
*
|
|
451
|
+
* @param {{emitted: Set<string>, reserved?: Map<string, {batchId: string|null, state: string}>}} memory `emitted`: runs
|
|
452
|
+
* whose emit-pr already ran in this process; `reserved`: story → what became of its re-serve (`state`: served,
|
|
453
|
+
* kill-failed, queue-failed or adopted)
|
|
454
|
+
* @param {{repoName?: string|null, openPr?: boolean, tracked?: Set<string>|null, trackedRetry?: string|null,
|
|
455
|
+
* realPath?: ((file: string) => string|null)|null}} [ctx] this checkout, so a run for another repository is
|
|
456
|
+
* not touched; `tracked`: the regular files git tracks here, which a request is checked against (`trackedRetry`:
|
|
457
|
+
* why listing them failed this tick, which is worth another poll); `realPath`: where a path resolves on the disk
|
|
458
|
+
* @returns {{stop: {kind: string, exitCode: number, reason: string}|null, actions: object[], notes: string[],
|
|
459
|
+
* waiting: Array<{runId: string, storyId: string, stepKey: string|null, reason: string}>,
|
|
460
|
+
* moving: number, follow: Array<{storyId: string, batchId: string}>}}
|
|
157
461
|
*/
|
|
158
462
|
function planTick(batch, details, memory, ctx = {}) {
|
|
159
463
|
const b = (batch && typeof batch === 'object') ? batch : {};
|
|
160
464
|
const halt = breakerStop(b);
|
|
161
|
-
if (halt) return { stop: { exitCode: EXIT.NEEDS_YOU, reason: halt }
|
|
465
|
+
if (halt) return tickPlan({ stop: { kind: 'breaker', exitCode: EXIT.NEEDS_YOU, reason: halt } });
|
|
162
466
|
|
|
163
467
|
const actions = [];
|
|
164
468
|
const notes = [];
|
|
165
|
-
|
|
469
|
+
const waiting = [];
|
|
470
|
+
let moving = 0;
|
|
471
|
+
const rejecting = [];
|
|
472
|
+
const runs = Array.isArray(b.runs) ? b.runs : [];
|
|
473
|
+
// Stories already re-served: by this process (memory), or by an earlier one (a killed run says so).
|
|
474
|
+
const follow = reservedFrom(b);
|
|
475
|
+
const reserved = new Map(memory.reserved || []);
|
|
476
|
+
for (const f of follow) reserved.set(f.storyId, { batchId: f.batchId, state: 'served' });
|
|
477
|
+
const planned = new Set();
|
|
478
|
+
const waitOn = (r, stepKey, reason) => waiting.push({ runId: r.runId, storyId: r.storyId, stepKey, reason });
|
|
479
|
+
const needsYou = (r, stepKey, reason) => {
|
|
480
|
+
const why = oneLine(reason, 400);
|
|
481
|
+
notes.push(`${r.storyId} ${r.runId} needs you at ${stepKey || 'a step'}: ${why}`);
|
|
482
|
+
waitOn(r, stepKey, why);
|
|
483
|
+
};
|
|
484
|
+
|
|
485
|
+
for (const r of runs) {
|
|
166
486
|
if (!r || !r.runId) continue;
|
|
167
487
|
if (ctx.repoName && r.repoName && r.repoName !== ctx.repoName) {
|
|
168
488
|
notes.push(`${r.storyId} ${r.runId} belongs to ${r.repoName}, not this checkout (${ctx.repoName}): skipped`);
|
|
489
|
+
if (!TERMINAL_RUN.has(r.status) && r.status !== 'paused') moving += 1;
|
|
169
490
|
continue;
|
|
170
491
|
}
|
|
171
492
|
if (r.status === 'completed') {
|
|
@@ -178,14 +499,19 @@ function planTick(batch, details, memory, ctx = {}) {
|
|
|
178
499
|
}
|
|
179
500
|
continue;
|
|
180
501
|
}
|
|
181
|
-
if (r.status !== 'paused'
|
|
502
|
+
if (r.status !== 'paused') {
|
|
503
|
+
if (!TERMINAL_RUN.has(r.status)) moving += 1;
|
|
504
|
+
continue;
|
|
505
|
+
}
|
|
506
|
+
const d = details[r.runId];
|
|
507
|
+
if (!d) { moving += 1; continue; }
|
|
182
508
|
// HC-GF-301: opening PRs, a step_4 attempt that passed verify-patch gets its draft now, while step_5 and its
|
|
183
509
|
// gate are still ahead, instead of when the whole run completes.
|
|
184
|
-
const verified = ctx.openPr ? verifiedStep4Attempt(
|
|
510
|
+
const verified = ctx.openPr ? verifiedStep4Attempt(d) : null;
|
|
185
511
|
if (verified && !(memory.drafted && memory.drafted.has(`${r.runId}#${verified}`))) {
|
|
186
|
-
actions.push({ kind: 'draft', runId: r.runId, storyId: r.storyId, attempt: verified, run:
|
|
512
|
+
actions.push({ kind: 'draft', runId: r.runId, storyId: r.storyId, attempt: verified, run: d,
|
|
187
513
|
branch: draftBranch(r.storyId, r.runId, verified) });
|
|
188
|
-
const vs = Array.isArray(
|
|
514
|
+
const vs = Array.isArray(d.verifications) ? d.verifications : [];
|
|
189
515
|
const earlier = vs.filter((v) => v && v.source === 'verify-patch' && v.stepKey === 'step_4' && v.verdict === 'pass'
|
|
190
516
|
&& Number(v.attempt) < verified).map((v) => Number(v.attempt));
|
|
191
517
|
if (earlier.length) {
|
|
@@ -193,28 +519,190 @@ function planTick(batch, details, memory, ctx = {}) {
|
|
|
193
519
|
notes.push(`${r.storyId} ${r.runId}: the draft for attempt ${verified} supersedes the one on ${draftBranch(r.storyId, r.runId, m)}; close that one when you are ready`);
|
|
194
520
|
}
|
|
195
521
|
}
|
|
196
|
-
|
|
522
|
+
|
|
523
|
+
// HC-GF-351: a hard pause for files the step was not given. Served once per story, and only if every path passes.
|
|
524
|
+
const request = hardPauseRequest(d);
|
|
525
|
+
if (request) {
|
|
526
|
+
const decided = decideReserve({ r, d, request, runs, details, ctx, prior: reserved.get(r.storyId), planned, batchId: b.batchId });
|
|
527
|
+
if (decided.action) {
|
|
528
|
+
planned.add(r.storyId);
|
|
529
|
+
actions.push(decided.action);
|
|
530
|
+
} else if (decided.retry) {
|
|
531
|
+
notes.push(`${r.storyId} ${r.runId}: ${decided.retry}`);
|
|
532
|
+
moving += 1;
|
|
533
|
+
} else {
|
|
534
|
+
needsYou(r, request.stepKey, `it asked for files it was not given: ${decided.why}`);
|
|
535
|
+
}
|
|
536
|
+
continue;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
// HC-GF-363: a refused `fail` is sent back, once per attempt, by the executor, which re-reads the run and the
|
|
540
|
+
// earlier attempts' rows before it POSTs. `memory.rejected` and `rejectStuck` only stop this tick repeating
|
|
541
|
+
// itself; the bound is read from the server.
|
|
542
|
+
const cand = autoRejectCandidate(d);
|
|
543
|
+
if (cand) {
|
|
544
|
+
const key = `${r.runId}#${cand.attempt}`;
|
|
545
|
+
const stuck = memory.rejectStuck && memory.rejectStuck.get(key);
|
|
546
|
+
if (stuck) needsYou(r, 'step_4', stuck);
|
|
547
|
+
else if (memory.rejected && memory.rejected.has(key)) { moving += 1; rejecting.push(key); }
|
|
548
|
+
else actions.push({ kind: 'reject', runId: r.runId, storyId: r.storyId, attempt: cand.attempt, run: d });
|
|
549
|
+
continue;
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
const a = step4Action(d);
|
|
197
553
|
// verify-patch's verdict POST is best-effort: an attempt this process already ran is not re-run every tick.
|
|
198
554
|
if (a.actionable && memory.tried && memory.tried.has(`${r.runId}#${a.attempt}`)) {
|
|
199
|
-
|
|
555
|
+
needsYou(r, 'step_4', `attempt ${a.attempt} was handled here (see the lines above) and the server holds no verdict for it`);
|
|
200
556
|
continue;
|
|
201
557
|
}
|
|
202
558
|
if (a.actionable) {
|
|
203
|
-
actions.push({ kind: 'verify', runId: r.runId, storyId: r.storyId, attempt: a.attempt, run:
|
|
559
|
+
actions.push({ kind: 'verify', runId: r.runId, storyId: r.storyId, attempt: a.attempt, run: d });
|
|
560
|
+
} else if (a.reason && approvalQueued(d)) {
|
|
561
|
+
moving += 1;
|
|
204
562
|
} else if (a.reason) {
|
|
205
|
-
|
|
563
|
+
needsYou(r, 'step_4', a.reason);
|
|
564
|
+
} else {
|
|
565
|
+
const p = otherPause(d, r);
|
|
566
|
+
if (p.kind === 'person') needsYou(r, p.stepKey, p.reason);
|
|
567
|
+
else if (p.kind === 'moving') moving += 1;
|
|
206
568
|
}
|
|
207
569
|
}
|
|
208
570
|
|
|
209
|
-
if (!isTerminal(b))
|
|
210
|
-
|
|
211
|
-
|
|
571
|
+
if (!isTerminal(b)) {
|
|
572
|
+
const stuck = actions.length === 0 && moving === 0 && waiting.length > 0;
|
|
573
|
+
return tickPlan({ stop: stuck ? { kind: 'stuck', exitCode: EXIT.NEEDS_YOU, reason: stuckReason(waiting) } : null,
|
|
574
|
+
actions, notes, waiting, moving, follow, rejecting });
|
|
575
|
+
}
|
|
576
|
+
// A killed run that was re-served is not unfinished: its story went on in the batch it names.
|
|
577
|
+
const raw = (b.counts && ((b.counts.failed || 0) + (b.counts.killed || 0) + (b.counts.stalled || 0))) || 0;
|
|
578
|
+
const unfinished = Math.max(0, raw - follow.length);
|
|
579
|
+
return tickPlan({
|
|
212
580
|
stop: unfinished
|
|
213
|
-
? { exitCode: EXIT.NEEDS_YOU, reason: `batch ${b.status}: ${unfinished} run(s) did not complete` }
|
|
214
|
-
: { exitCode: EXIT.DONE, reason: `batch ${b.status}` },
|
|
215
|
-
actions,
|
|
216
|
-
|
|
217
|
-
|
|
581
|
+
? { kind: 'terminal', exitCode: EXIT.NEEDS_YOU, reason: `batch ${b.status}: ${unfinished} run(s) did not complete` }
|
|
582
|
+
: { kind: 'terminal', exitCode: EXIT.DONE, reason: `batch ${b.status}` },
|
|
583
|
+
actions, notes, waiting, moving, follow, rejecting,
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
/**
|
|
588
|
+
* HC-GF-351: queue the story again with the files its step asked for, then kill the paused run. In that order: the
|
|
589
|
+
* kill's reason names the new batch (the next start reads it back, which is what keeps the story from being served
|
|
590
|
+
* twice), and a story with no run in flight is worse than one with two. If the queue fails the paused run is left
|
|
591
|
+
* alone; if the kill fails the new batch still runs and the paused run is named for a person.
|
|
592
|
+
*
|
|
593
|
+
* Before queueing it asks the server whether the story was served already (io.priorServe): a kill that failed, or a
|
|
594
|
+
* crash between the queue and the kill, leaves no marker, and only a newer run of the same story shows it happened.
|
|
595
|
+
* @returns {Promise<{lines: string[], needsYou: boolean, batchId: string|null, state: string|null, retry?: boolean}>}
|
|
596
|
+
* `batchId`: the batch the story was queued as, set whenever it was, even when the kill then failed; `state`: what
|
|
597
|
+
* became of the re-serve (served, kill-failed, queue-failed, adopted); `retry`: nothing was decided, ask again
|
|
598
|
+
*/
|
|
599
|
+
async function reserve(action, io, tag) {
|
|
600
|
+
if (io.priorServe) {
|
|
601
|
+
let prior;
|
|
602
|
+
try {
|
|
603
|
+
prior = await io.priorServe(action);
|
|
604
|
+
} catch (e) {
|
|
605
|
+
return { lines: [`${tag} not re-served yet: could not ask whether the story was served already (${firstLine(e && e.message)}); asking again next poll`],
|
|
606
|
+
needsYou: false, batchId: null, state: null, retry: true };
|
|
607
|
+
}
|
|
608
|
+
if (prior && prior.kind === 'unknown') {
|
|
609
|
+
return { lines: [`${tag} not re-served: could not tell whether the story was served already; the paused run was left alone`],
|
|
610
|
+
needsYou: true, batchId: null, state: 'queue-failed' };
|
|
611
|
+
}
|
|
612
|
+
if (prior) {
|
|
613
|
+
const where = prior.batchId ? ` in batch ${prior.batchId}` : '';
|
|
614
|
+
return { lines: [`${tag} not re-served: run ${prior.runId}${where} is this story's ${prior.kind === 'later' ? 'newer' : 'earlier, re-served'} run; following it`],
|
|
615
|
+
needsYou: true, batchId: prior.batchId || null, state: 'adopted' };
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
let queued;
|
|
619
|
+
let file = null;
|
|
620
|
+
try {
|
|
621
|
+
file = io.writeStoriesFile(`${formatStoryLine({ storyId: action.storyId, paths: action.paths })}\n`);
|
|
622
|
+
queued = await io.runVerb(buildReserveArgs(file.file, { repoName: action.repoName, branch: action.branch }), { capture: true });
|
|
623
|
+
} catch (e) {
|
|
624
|
+
return { lines: [`${tag} not re-served: ${firstLine(e && e.message)}; the paused run was left alone`], needsYou: true, batchId: null, state: 'queue-failed' };
|
|
625
|
+
} finally {
|
|
626
|
+
if (file) file.remove();
|
|
627
|
+
}
|
|
628
|
+
const batchId = queued.code === 0 ? batchIdFromQueueOutput(queued.stdout) : null;
|
|
629
|
+
if (!batchId) {
|
|
630
|
+
const why = queued.code === 0 ? 'printed no batch id' : `exited ${queued.code}`;
|
|
631
|
+
return { lines: [`${tag} not re-served: queue-stories ${why}; the paused run was left alone`], needsYou: true, batchId: null, state: 'queue-failed' };
|
|
632
|
+
}
|
|
633
|
+
const lines = [`${tag} ${action.stepKey} named files it was not given: queued batch ${batchId} with paths:${action.paths.join(',')}`];
|
|
634
|
+
try {
|
|
635
|
+
await io.killRun(action.runId, reserveReason(batchId, action.stepKey));
|
|
636
|
+
} catch (e) {
|
|
637
|
+
const status = e && e.response && e.response.status;
|
|
638
|
+
lines.push(`${tag} could NOT kill the paused run (${status || firstLine(e && e.message)}); it is still paused beside batch ${batchId}: kill it with hone run-story ${action.runId} --kill`);
|
|
639
|
+
return { lines, needsYou: true, batchId, state: 'kill-failed' };
|
|
640
|
+
}
|
|
641
|
+
lines.push(`${tag} killed the paused run: ${reserveReason(batchId, action.stepKey)}`);
|
|
642
|
+
return { lines, needsYou: false, batchId, state: 'served' };
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
/** The HTTP status of a thrown axios error, or undefined. */
|
|
646
|
+
const statusOf = (e) => (e && e.response && e.response.status) || undefined;
|
|
647
|
+
|
|
648
|
+
/**
|
|
649
|
+
* HC-GF-363: send a refused step_4 `fail` back to the builder, if the bound allows. Re-reads the run and re-decides
|
|
650
|
+
* right before the POST (the plan is up to one poll old), then counts the earlier rejects from each earlier attempt's
|
|
651
|
+
* stored reject reason. Any read that fails means the count is unknown, and unknown never rejects.
|
|
652
|
+
* @returns {Promise<{lines: string[], needsYou: boolean, state: 'posted'|'moved'|'stuck'|'retry', note?: string,
|
|
653
|
+
* countsRead?: boolean, fatal?: string}>}
|
|
654
|
+
* `retry`: nothing was sent, ask again next poll (`countsRead: true` when it was a read that failed, which is
|
|
655
|
+
* given up on after three); `fatal`: the server answered 401, 403 or 404 where it never should
|
|
656
|
+
*/
|
|
657
|
+
async function autoReject(action, io, tag) {
|
|
658
|
+
const where = `step_4 attempt ${action.attempt}`;
|
|
659
|
+
const fatal = (e, what) => ([401, 403, 404].includes(statusOf(e)) ? `the server answered ${statusOf(e)} while ${what}` : null);
|
|
660
|
+
if (!io.rejectRun || !io.fetchStep) {
|
|
661
|
+
return { lines: [`${tag} ${where} failed verification; this client cannot reject, so it is left for you`], needsYou: true, state: 'stuck', note: `${where} failed verification and was not sent back` };
|
|
662
|
+
}
|
|
663
|
+
let fresh;
|
|
664
|
+
try {
|
|
665
|
+
fresh = await io.fetchRun(action.runId);
|
|
666
|
+
} catch (e) {
|
|
667
|
+
const f = fatal(e, `re-reading run ${action.runId}`);
|
|
668
|
+
if (f) return { lines: [], needsYou: false, state: 'retry', fatal: f };
|
|
669
|
+
return { lines: [`${tag} not rejected yet: could not re-read the run (${firstLine(e && e.message)}); asking again next poll`], needsYou: false, state: 'retry' };
|
|
670
|
+
}
|
|
671
|
+
const now = autoRejectCandidate(fresh);
|
|
672
|
+
if (!now || now.attempt !== action.attempt) {
|
|
673
|
+
return { lines: [`${tag} ${where} is no longer a refused failure; nothing sent`], needsYou: false, state: 'moved' };
|
|
674
|
+
}
|
|
675
|
+
const contexts = [];
|
|
676
|
+
for (let k = 1; k < action.attempt; k += 1) {
|
|
677
|
+
try {
|
|
678
|
+
const row = await io.fetchStep(action.runId, 'step_4', k);
|
|
679
|
+
contexts.push(row && typeof row.errorContext === 'string' ? row.errorContext : null);
|
|
680
|
+
} catch (e) {
|
|
681
|
+
return { lines: [`${tag} not rejected yet: could not read step_4 attempt ${k} (${statusOf(e) || firstLine(e && e.message)}), so the rejects already used are unknown; asking again next poll`],
|
|
682
|
+
needsYou: false, state: 'retry', countsRead: true };
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
const dec = autoRejectDecision(fresh, contexts);
|
|
686
|
+
if (dec.kind === 'exhausted') {
|
|
687
|
+
return { lines: [`${tag} ${dec.note}`], needsYou: true, state: 'stuck', note: dec.note };
|
|
688
|
+
}
|
|
689
|
+
if (dec.kind !== 'reject') {
|
|
690
|
+
return { lines: [`${tag} ${where}: no automatic reject (${dec.why || dec.kind})`], needsYou: false, state: 'retry', countsRead: true };
|
|
691
|
+
}
|
|
692
|
+
try {
|
|
693
|
+
await io.rejectRun(action.runId, { stepKey: 'step_4', reason: dec.reason });
|
|
694
|
+
} catch (e) {
|
|
695
|
+
const status = statusOf(e);
|
|
696
|
+
const f = fatal(e, `rejecting ${where}`);
|
|
697
|
+
if (f) return { lines: [], needsYou: false, state: 'retry', fatal: f };
|
|
698
|
+
if (status === 409) return { lines: [`${tag} ${where} was not rejected: the run moved on (409); this does not count`], needsYou: false, state: 'moved' };
|
|
699
|
+
if (status === 400) {
|
|
700
|
+
const note = `${where} failed verification and the server refused the reject (400: ${oneLine(e.response.data && e.response.data.error, 120)})`;
|
|
701
|
+
return { lines: [`${tag} ${note}`], needsYou: true, state: 'stuck', note };
|
|
702
|
+
}
|
|
703
|
+
return { lines: [`${tag} not rejected yet: ${status || firstLine(e && e.message)}; asking again next poll`], needsYou: false, state: 'retry' };
|
|
704
|
+
}
|
|
705
|
+
return { lines: [`${tag} ${where} failed verification: rejected it (automatic reject ${dec.n} of ${AUTO_REJECT_LIMIT}): ${oneLine(now.detail, 200)}`], needsYou: false, state: 'posted' };
|
|
218
706
|
}
|
|
219
707
|
|
|
220
708
|
/**
|
|
@@ -229,6 +717,8 @@ function planTick(batch, details, memory, ctx = {}) {
|
|
|
229
717
|
*/
|
|
230
718
|
async function runTask(action, io, opts = {}) {
|
|
231
719
|
const tag = `[follow-batch] ${action.storyId} ${action.runId}`;
|
|
720
|
+
if (action.kind === 'reserve') return reserve(action, io, tag);
|
|
721
|
+
if (action.kind === 'reject') return autoReject(action, io, tag);
|
|
232
722
|
if (action.kind === 'emit' || action.kind === 'draft') {
|
|
233
723
|
// emit-pr re-runs the tests before it commits, even as a local dry run, so it needs the same consent and
|
|
234
724
|
// provenance verify-patch does. Without this, a repository that never set ci.local_command gets `make ci`.
|
|
@@ -255,11 +745,34 @@ async function runTask(action, io, opts = {}) {
|
|
|
255
745
|
return { lines, needsYou: v.code !== 0 };
|
|
256
746
|
}
|
|
257
747
|
|
|
748
|
+
/**
|
|
749
|
+
* What the loop returns once every batch it follows has ended (settled by the server, or stuck on a person).
|
|
750
|
+
* DONE only when every one of them completed and no emit is owed; the reason of each batch that did not is kept.
|
|
751
|
+
* A batch stuck on a person is not over: step_5's gate is mandatory, so this is where most runs end, and the reason
|
|
752
|
+
* says how to continue (`again`: the batches to follow once the person has acted).
|
|
753
|
+
*/
|
|
754
|
+
function settle(stops, memory, again = []) {
|
|
755
|
+
const open = stops.filter((s) => s.exitCode !== EXIT.DONE);
|
|
756
|
+
const owed = memory.owed.size ? `${memory.owed.size} run(s) were not emitted: ${[...memory.owed.values()].join(', ')}` : '';
|
|
757
|
+
if (open.length === 0 && !owed) return { exitCode: EXIT.DONE, reason: stops.map((s) => s.reason).join('; ') };
|
|
758
|
+
const why = [...open.map((s) => s.reason), owed].filter(Boolean);
|
|
759
|
+
const next = again.length ? ` (after you act, run again to emit: hone follow-batch ${again.join('; hone follow-batch ')})` : '';
|
|
760
|
+
return { exitCode: EXIT.NEEDS_YOU, reason: (open.length ? why.join('; ') : `${stops.map((s) => s.reason).join('; ')}, but ${owed}`) + next };
|
|
761
|
+
}
|
|
762
|
+
|
|
258
763
|
/**
|
|
259
764
|
* The loop. Every effect is injected, so a test drives it with a stub server and a recording runVerb.
|
|
765
|
+
*
|
|
766
|
+
* It follows a SET of batches: the one it was given, and every batch a killed run says its story was re-served as
|
|
767
|
+
* (HC-GF-351), found in the same tick. It ends when every one of them has ended, or when one has fired a breaker.
|
|
260
768
|
* @param {string} batchId
|
|
261
769
|
* @param {{fetchBatch: Function, fetchRun: Function, runVerb: Function, verifyDecision: Function,
|
|
262
|
-
* sleep: (ms: number) => Promise<void>, now: () => number, log: (line: string) => void
|
|
770
|
+
* sleep: (ms: number) => Promise<void>, now: () => number, log: (line: string) => void,
|
|
771
|
+
* trackedPaths?: () => Promise<Set<string>>|Set<string>, realPath?: (file: string) => string|null,
|
|
772
|
+
* priorServe?: Function, writeStoriesFile?: Function, killRun?: Function, rejectRun?: Function,
|
|
773
|
+
* fetchStep?: Function}} io
|
|
774
|
+
* trackedPaths, realPath, priorServe, writeStoriesFile and killRun serve a re-serve; without trackedPaths or
|
|
775
|
+
* realPath every request is refused, and without priorServe nobody asks whether the story was served already.
|
|
263
776
|
* @param {{pollMs?: number, deadlineMs?: number|null, maxOfflineMs?: number, openPr?: boolean,
|
|
264
777
|
* repoName?: string|null}} [opts]
|
|
265
778
|
* @returns {Promise<{exitCode: number, reason: string}>}
|
|
@@ -268,8 +781,13 @@ async function followBatch(batchId, io, opts = {}) {
|
|
|
268
781
|
const pollMs = opts.pollMs || 30_000;
|
|
269
782
|
const maxOfflineMs = opts.maxOfflineMs || 15 * 60_000;
|
|
270
783
|
// owed: runs whose emit-pr needed a person. A completed batch with one of these is not DONE (HC-GF-207's shape:
|
|
271
|
-
// a failure path that exits 0).
|
|
272
|
-
|
|
784
|
+
// a failure path that exits 0). reserved: story → what became of its re-serve ({batchId, state}). trackedFails:
|
|
785
|
+
// consecutive ticks on which listing the tracked files failed.
|
|
786
|
+
const memory = { emitted: new Set(), tried: new Set(), drafted: new Set(), said: new Set(), owed: new Map(), reserved: new Map(), trackedFails: 0,
|
|
787
|
+
// HC-GF-363: only to stop a tick repeating itself. rejected: attempt key → ticks waited since the reject was accepted;
|
|
788
|
+
// rejectStuck: key → why a person has to act; rejectReads: key → consecutive failed reads of the earlier attempts.
|
|
789
|
+
rejected: new Map(), rejectStuck: new Map(), rejectReads: new Map() };
|
|
790
|
+
const followed = [batchId];
|
|
273
791
|
const start = io.now();
|
|
274
792
|
let offlineSince = null;
|
|
275
793
|
const say = (line) => { if (!memory.said.has(line)) { memory.said.add(line); io.log(line); } };
|
|
@@ -280,19 +798,36 @@ async function followBatch(batchId, io, opts = {}) {
|
|
|
280
798
|
}
|
|
281
799
|
// A tick reads everything or acts on nothing: a run that could not be read is the server being unreachable,
|
|
282
800
|
// not a run with no details (which would skip its gate, or emit it with no provenance to assess).
|
|
283
|
-
|
|
284
|
-
|
|
801
|
+
const reads = [];
|
|
802
|
+
let reading = batchId;
|
|
285
803
|
try {
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
const
|
|
289
|
-
|
|
804
|
+
for (let i = 0; i < followed.length; i += 1) {
|
|
805
|
+
reading = followed[i];
|
|
806
|
+
const batch = await io.fetchBatch(reading);
|
|
807
|
+
const details = {};
|
|
808
|
+
for (const r of Array.isArray(batch && batch.runs) ? batch.runs : []) {
|
|
809
|
+
const wanted = r && r.runId && (r.status === 'paused' || (r.status === 'completed' && !memory.emitted.has(r.runId)));
|
|
810
|
+
if (wanted) details[r.runId] = await io.fetchRun(r.runId);
|
|
811
|
+
}
|
|
812
|
+
// A run hard-paused on a request is only re-served if nothing depends on it, and a queued run has not said
|
|
813
|
+
// what it depends on: its config is the only place that is written. Read them when a request is waiting.
|
|
814
|
+
if (Object.values(details).some((d) => hardPauseRequest(d))) {
|
|
815
|
+
for (const r of Array.isArray(batch && batch.runs) ? batch.runs : []) {
|
|
816
|
+
if (r && r.runId && r.status === 'queued' && !details[r.runId]) details[r.runId] = await io.fetchRun(r.runId);
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
reads.push({ batch, details });
|
|
820
|
+
// A killed run that says where its story went: follow that batch, and do not serve the story again.
|
|
821
|
+
for (const f of reservedFrom(batch)) {
|
|
822
|
+
memory.reserved.set(f.storyId, { batchId: f.batchId, state: 'served' });
|
|
823
|
+
if (!followed.includes(f.batchId)) followed.push(f.batchId);
|
|
824
|
+
}
|
|
290
825
|
}
|
|
291
826
|
offlineSince = null;
|
|
292
827
|
} catch (e) {
|
|
293
828
|
const status = e && e.response && e.response.status;
|
|
294
829
|
if (status === 401 || status === 403 || status === 404) {
|
|
295
|
-
return { exitCode: EXIT.FATAL, reason: `the server answered ${status} while reading batch ${
|
|
830
|
+
return { exitCode: EXIT.FATAL, reason: `the server answered ${status} while reading batch ${reading}` };
|
|
296
831
|
}
|
|
297
832
|
if (offlineSince === null) offlineSince = io.now();
|
|
298
833
|
if (io.now() - offlineSince >= maxOfflineMs) {
|
|
@@ -302,32 +837,85 @@ async function followBatch(batchId, io, opts = {}) {
|
|
|
302
837
|
continue;
|
|
303
838
|
}
|
|
304
839
|
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
}
|
|
316
|
-
memory.
|
|
840
|
+
// The checkout's tracked files, asked for only when a run is hard-paused on a request. Without them every
|
|
841
|
+
// path is refused: a request cannot be served on a list that could not be read. A listing that fails is
|
|
842
|
+
// worth three more polls (git can be busy), not an exit.
|
|
843
|
+
let tracked = null;
|
|
844
|
+
let trackedRetry = null;
|
|
845
|
+
const asks = reads.some((x) => Object.values(x.details).some((d) => hardPauseRequest(d)));
|
|
846
|
+
if (asks && io.trackedPaths) {
|
|
847
|
+
try {
|
|
848
|
+
tracked = await io.trackedPaths();
|
|
849
|
+
memory.trackedFails = 0;
|
|
850
|
+
} catch (e) {
|
|
851
|
+
memory.trackedFails += 1;
|
|
852
|
+
if (memory.trackedFails <= 3) trackedRetry = firstLine(e && e.message) || 'git ls-files failed';
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
const ctx = { repoName: opts.repoName || null, openPr: !!opts.openPr, tracked, trackedRetry, realPath: io.realPath || null };
|
|
856
|
+
const plans = reads.map((x) => planTick(x.batch, x.details, memory, ctx));
|
|
857
|
+
for (const plan of plans) for (const n of plan.notes) say(`[follow-batch] ${n}`);
|
|
858
|
+
// A fired breaker stops the loop BEFORE any batch's action runs.
|
|
859
|
+
const halted = plans.find((plan) => plan.stop && plan.stop.kind === 'breaker');
|
|
860
|
+
if (halted) return { exitCode: halted.stop.exitCode, reason: halted.stop.reason };
|
|
861
|
+
|
|
862
|
+
// An accepted reject that the worker has not applied for REJECT_WAIT_TICKS polls is named for a person.
|
|
863
|
+
for (const plan of plans) {
|
|
864
|
+
for (const key of plan.rejecting) {
|
|
865
|
+
const waited = (memory.rejected.get(key) || 0) + 1;
|
|
866
|
+
memory.rejected.set(key, waited);
|
|
867
|
+
if (waited >= REJECT_WAIT_TICKS) {
|
|
868
|
+
memory.rejectStuck.set(key, `a reject was accepted ${waited} polls ago and the run is still paused at the same attempt; look at it with hone run-report`);
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
let grew = false;
|
|
874
|
+
for (const plan of plans) {
|
|
875
|
+
for (const action of plan.actions) {
|
|
876
|
+
const out = await runTask(action, io, { openPr: opts.openPr });
|
|
877
|
+
if (action.kind === 'reject') {
|
|
878
|
+
if (out.fatal) return { exitCode: EXIT.FATAL, reason: out.fatal };
|
|
879
|
+
const key = `${action.runId}#${action.attempt}`;
|
|
880
|
+
if (out.state === 'posted' || out.state === 'moved') memory.rejected.set(key, 0);
|
|
881
|
+
else if (out.state === 'stuck') memory.rejectStuck.set(key, out.note);
|
|
882
|
+
else if (out.countsRead) {
|
|
883
|
+
const n = (memory.rejectReads.get(key) || 0) + 1;
|
|
884
|
+
memory.rejectReads.set(key, n);
|
|
885
|
+
if (n >= 3) memory.rejectStuck.set(key, `step_4 attempt ${action.attempt} failed verification, but the rejects already used could not be read after ${n} polls; nothing was sent`);
|
|
886
|
+
}
|
|
887
|
+
for (const line of out.lines) say(line);
|
|
888
|
+
continue;
|
|
889
|
+
}
|
|
890
|
+
if (action.kind === 'reserve') {
|
|
891
|
+
if (!out.retry) {
|
|
892
|
+
memory.reserved.set(action.storyId, { batchId: out.batchId, state: out.state });
|
|
893
|
+
if (out.batchId && !followed.includes(out.batchId)) { followed.push(out.batchId); grew = true; }
|
|
894
|
+
}
|
|
895
|
+
} else if (action.kind === 'emit' || action.kind === 'draft') {
|
|
896
|
+
if (action.kind === 'emit') memory.emitted.add(action.runId);
|
|
897
|
+
else memory.drafted.add(`${action.runId}#${action.attempt}`);
|
|
898
|
+
// A run is owed until an emit for it succeeds: a draft that failed is retried by the completed-run emit.
|
|
899
|
+
if (out.needsYou) memory.owed.set(action.runId, `${action.storyId} ${action.runId}`);
|
|
900
|
+
else if (action.kind === 'emit') memory.owed.delete(action.runId);
|
|
901
|
+
} else {
|
|
902
|
+
memory.tried.add(`${action.runId}#${action.attempt}`);
|
|
903
|
+
}
|
|
904
|
+
for (const line of out.lines) io.log(line);
|
|
317
905
|
}
|
|
318
|
-
for (const line of out.lines) io.log(line);
|
|
319
906
|
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
907
|
+
// Ended means every batch read this tick has: settled by the server, or stuck on a person. A batch queued
|
|
908
|
+
// this tick has not been read yet.
|
|
909
|
+
if (!grew && plans.every((plan) => plan.stop)) {
|
|
910
|
+
return settle(plans.map((plan) => plan.stop), memory, followed.filter((id, i) => plans[i].stop.kind === 'stuck'));
|
|
323
911
|
}
|
|
324
|
-
if (plan.stop) return plan.stop;
|
|
325
912
|
await io.sleep(pollMs);
|
|
326
913
|
}
|
|
327
914
|
}
|
|
328
915
|
|
|
329
916
|
module.exports = {
|
|
330
917
|
EXIT, FORBIDDEN_FLAGS, NOT_A_GATE,
|
|
331
|
-
breakerStop, isTerminal, priorVerdict, step4Action, buildArgs, planTick, runTask, followBatch,
|
|
332
|
-
currentStep4Attempt, verifiedStep4Attempt, draftBranch,
|
|
918
|
+
breakerStop, isTerminal, priorVerdict, step4Action, buildArgs, buildReserveArgs, planTick, runTask, followBatch,
|
|
919
|
+
currentStep4Attempt, verifiedStep4Attempt, draftBranch, reservedFrom, otherPause, stuckReason, gateEnforced, stepRank,
|
|
920
|
+
AUTO_REJECT_PREFIX, AUTO_REJECT_LIMIT, autoRejectCandidate, autoRejectDecision, autoRejectedAttempts,
|
|
333
921
|
};
|