@hone-ai/cli 1.38.0 → 1.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/hone-cli.js +282 -309
- package/lib/batch-follow.js +418 -51
- package/lib/bundle-paths.js +168 -21
- package/lib/call-sites.js +489 -0
- package/lib/classify-spec.js +68 -0
- package/lib/context-reserve.js +270 -0
- package/lib/detect-test-framework.js +54 -0
- package/lib/doctor-admin-merge.js +11 -3
- package/lib/git-remote.js +103 -0
- package/lib/hard-pause-remedy.js +1 -1
- package/lib/judge-provider.js +39 -34
- package/lib/learnings-sync.js +2 -1
- package/lib/mcp-tools.js +42 -1
- package/lib/run-context.js +25 -3
- package/lib/run-target.js +34 -2
- package/lib/story-line.js +48 -12
- package/lib/story-provenance.js +19 -12
- package/package.json +1 -1
package/lib/batch-follow.js
CHANGED
|
@@ -17,9 +17,23 @@
|
|
|
17
17
|
* - It stops BEFORE acting when a breaker has fired. A terminal batch ends the loop after one last emit-pr pass.
|
|
18
18
|
* - It is idempotent across restarts: an attempt that already has a verify-patch verdict on the server is not
|
|
19
19
|
* verified again, and emit-pr's BRANCH_CONFLICT (5) means the branch was already emitted.
|
|
20
|
+
* - HC-GF-351: it is never silent. A run paused at anything the loop cannot act on is named once, with its step
|
|
21
|
+
* and the first line of why; when every unfinished run is waiting on a person the loop stops with NEEDS_YOU
|
|
22
|
+
* instead of waiting out its deadline. And a run hard-paused for files it was not given is re-served ONCE: the
|
|
23
|
+
* paused run is killed with a reason that names the new batch (the next start reads it back, which is what
|
|
24
|
+
* makes the bound survive a restart), and its story is queued again as a one-story batch whose line carries
|
|
25
|
+
* the files, which the loop then follows too. Only paths cli/lib/context-reserve.js lets through are served.
|
|
26
|
+
* Killing a run FAILS every queued or paused run in its batch that lists it in `config.dependsOnRunIds`
|
|
27
|
+
* (server/src/services/batch-deps.js cascadeOnFailure), so a run anything depends on is never re-served, and the
|
|
28
|
+
* decision reads that list, not the dependents' waiting message, which names only one of their blockers.
|
|
20
29
|
*/
|
|
21
30
|
|
|
22
31
|
const { classifyPollState, POLL_OUTCOMES } = require('./poll-decision');
|
|
32
|
+
const { formatStoryLine } = require('./story-line');
|
|
33
|
+
const { isGitRev, isStoryId } = require('./run-target');
|
|
34
|
+
const {
|
|
35
|
+
reserveReason, reservedBatchOf, hardPauseRequest, judgeRequest, originalPathsOf, batchIdFromQueueOutput,
|
|
36
|
+
} = require('./context-reserve');
|
|
23
37
|
|
|
24
38
|
/** follow-batch's own exit codes. */
|
|
25
39
|
const EXIT = Object.freeze({ DONE: 0, FATAL: 1, NEEDS_YOU: 2, DEADLINE: 3 });
|
|
@@ -91,7 +105,7 @@ function priorVerdict(run, attempt) {
|
|
|
91
105
|
*/
|
|
92
106
|
function step4Action(run) {
|
|
93
107
|
const c = classifyPollState(run);
|
|
94
|
-
// Not a step_4 gate: not this
|
|
108
|
+
// Not a step_4 gate: not this function's business. planTick asks otherPause who such a pause waits on (HC-GF-351).
|
|
95
109
|
if (c.outcome !== POLL_OUTCOMES.HUMAN_GATE || c.stepKey !== 'step_4') return { actionable: false };
|
|
96
110
|
const why = String((run && run.errorMessage) || '');
|
|
97
111
|
if (NOT_A_GATE.test(why)) return { actionable: false, reason: `paused, not at a gate: ${firstLine(why)}` };
|
|
@@ -124,6 +138,20 @@ function buildArgs(verb, runId, { attempt, openPr = false, branch } = {}) {
|
|
|
124
138
|
return args;
|
|
125
139
|
}
|
|
126
140
|
|
|
141
|
+
/**
|
|
142
|
+
* The argv that queues one story again (HC-GF-351). The file, repository and branch are values from outside this
|
|
143
|
+
* module, so they ride in `--flag=value` form: a value that starts with a dash cannot become an option. No flag
|
|
144
|
+
* here overrides a safety gate, and none is `--overnight` (the loop does not know the original batch's mode).
|
|
145
|
+
* @param {string} file a stories file holding the one line
|
|
146
|
+
*/
|
|
147
|
+
function buildReserveArgs(file, { repoName, branch } = {}) {
|
|
148
|
+
const args = ['queue-stories', `--file=${file}`];
|
|
149
|
+
if (repoName) args.push(`--repo=${repoName}`);
|
|
150
|
+
if (branch && isGitRev(branch)) args.push(`--branch=${branch}`);
|
|
151
|
+
args.push('--format', 'json');
|
|
152
|
+
return args;
|
|
153
|
+
}
|
|
154
|
+
|
|
127
155
|
/** The step_4 attempt a run is on now: the highest step_4 attempt in its steps, or null. */
|
|
128
156
|
function currentStep4Attempt(run) {
|
|
129
157
|
const steps = Array.isArray(run && run.steps) ? run.steps : [];
|
|
@@ -147,25 +175,203 @@ function draftBranch(storyId, runId, attempt) {
|
|
|
147
175
|
return `hone/${storyId || 'story'}-${String(runId).slice(0, 8)}-a${attempt}`;
|
|
148
176
|
}
|
|
149
177
|
|
|
178
|
+
/** One line of agent- or server-written text, safe to print: control characters (a terminal escape) become spaces. */
|
|
179
|
+
const oneLine = (s, max = 240) => firstLine(String(s || '')).replace(/[\u0000-\u001f\u007f-\u009f]/g, ' ').trim().slice(0, max);
|
|
180
|
+
|
|
181
|
+
/** The statuses a batch child ends in. Anything else is still moving as far as the loop can tell. */
|
|
182
|
+
const TERMINAL_RUN = new Set(['completed', 'failed', 'killed', 'stalled']);
|
|
183
|
+
|
|
184
|
+
/** The shape every planTick return carries, so a caller never meets an absent field. `stop` is always the caller's. */
|
|
185
|
+
const tickPlan = (over) => ({ actions: [], notes: [], waiting: [], moving: 0, follow: [], ...over });
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Batches the server's own rows say were re-served: a killed run whose reason names the batch its story went on in.
|
|
189
|
+
* @param {object} batch a `GET /orchestrate/batch/:id` payload
|
|
190
|
+
* @returns {Array<{storyId: string, batchId: string}>}
|
|
191
|
+
*/
|
|
192
|
+
function reservedFrom(batch) {
|
|
193
|
+
const out = [];
|
|
194
|
+
for (const r of Array.isArray(batch && batch.runs) ? batch.runs : []) {
|
|
195
|
+
const batchId = reservedBatchOf(r);
|
|
196
|
+
if (batchId) out.push({ storyId: r.storyId, batchId });
|
|
197
|
+
}
|
|
198
|
+
return out;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* A paused run that is not at a step_4 gate and has no request to serve: who is it waiting on?
|
|
203
|
+
* `person`: a gate or a refusal only a human clears. `dependency`: it will wake when the run it waits on ends.
|
|
204
|
+
* `moving`: paused without a gate or a reason the loop knows (the server decides what is next); a poll more costs
|
|
205
|
+
* an interval, and calling it stuck would end a live run.
|
|
206
|
+
* @returns {{kind: 'person', stepKey: string|null, reason: string}|{kind: 'dependency'}|{kind: 'moving'}}
|
|
207
|
+
*/
|
|
208
|
+
function otherPause(run, row) {
|
|
209
|
+
const msg = String((run && run.errorMessage) || '');
|
|
210
|
+
const stepKey = (run && run.currentStep) || (row && row.currentStep) || null;
|
|
211
|
+
if (/^BATCH_DEPENDENCY_WAITING:/.test(msg)) return { kind: 'dependency' };
|
|
212
|
+
const c = classifyPollState(run);
|
|
213
|
+
if (c.outcome === POLL_OUTCOMES.EDITOR_PARKED) {
|
|
214
|
+
return { kind: 'person', stepKey, reason: 'parked for your editor (execution.mode: editor): submit its output, then approve' };
|
|
215
|
+
}
|
|
216
|
+
if (c.outcome === POLL_OUTCOMES.EXHAUSTED || NOT_A_GATE.test(msg)) return { kind: 'person', stepKey, reason: oneLine(msg) };
|
|
217
|
+
if (c.outcome === POLL_OUTCOMES.HUMAN_GATE) {
|
|
218
|
+
return { kind: 'person', stepKey, reason: msg ? oneLine(msg) : `awaiting your approval: hone run-story ${run.runId} --approve ${stepKey}` };
|
|
219
|
+
}
|
|
220
|
+
return { kind: 'moving' };
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* True when a verdict from `source` drives the step_4 gate on this run. Mirrors isEnforced in
|
|
225
|
+
* server/src/services/verification-refusal.js (the CLI does not ship server code); tests/regression/HC-GF-351 pins
|
|
226
|
+
* the two equal.
|
|
227
|
+
*/
|
|
228
|
+
function gateEnforced(config, source = 'verify-patch') {
|
|
229
|
+
const mode = String((config && config.verification_gate) || 'advisory').toLowerCase();
|
|
230
|
+
const sources = config && Array.isArray(config.verification_gate_sources) && config.verification_gate_sources.length
|
|
231
|
+
? config.verification_gate_sources : ['verify-patch'];
|
|
232
|
+
return mode === 'enforce' && sources.includes(source);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* A paused step_4 gate whose attempt passed verify-patch under `verification_gate: enforce`. The pass only QUEUES the
|
|
237
|
+
* approval (POST /:runId/verdict → enqueueApproval); the run stays paused at step_4 until a worker takes the job. It
|
|
238
|
+
* is moving, not waiting on a person, and saying "approve it" in that window sends the operator to a gate that is
|
|
239
|
+
* already being approved.
|
|
240
|
+
*/
|
|
241
|
+
function approvalQueued(run) {
|
|
242
|
+
const why = String((run && run.errorMessage) || '');
|
|
243
|
+
return !!run && run.currentStep === 'step_4' && !NOT_A_GATE.test(why) && !run.approvalRefusal
|
|
244
|
+
&& gateEnforced(run.config) && verifiedStep4Attempt(run) !== null;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/** step_0 → 0, step_4 → 400, step_5b → 502: the order steps run in, for "after step_4". NaN for a key it cannot read. */
|
|
248
|
+
function stepRank(stepKey) {
|
|
249
|
+
const m = String(stepKey).match(/^step_(\d+)([a-z]?)$/);
|
|
250
|
+
return m ? Number(m[1]) * 100 + (m[2] ? m[2].charCodeAt(0) - 96 : 0) : NaN;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* The queued or paused runs of this batch that killing `runId` would fail: those whose `config.dependsOnRunIds` lists
|
|
255
|
+
* it, or whose waiting message names it. `unread` are queued or paused runs whose config was not read, so nothing is
|
|
256
|
+
* known about them.
|
|
257
|
+
* @returns {{found: string[], unread: string[]}} story ids
|
|
258
|
+
*/
|
|
259
|
+
function dependentsOf(runId, runs, details) {
|
|
260
|
+
const found = [];
|
|
261
|
+
const unread = [];
|
|
262
|
+
for (const x of runs) {
|
|
263
|
+
if (!x || x.runId === runId || (x.status !== 'queued' && x.status !== 'paused')) continue;
|
|
264
|
+
const d = details[x.runId];
|
|
265
|
+
const listed = !!d && !!d.config && Array.isArray(d.config.dependsOnRunIds) && d.config.dependsOnRunIds.includes(runId);
|
|
266
|
+
const named = String(x.errorMessage || (d && d.errorMessage) || '').startsWith(`BATCH_DEPENDENCY_WAITING:${runId}`);
|
|
267
|
+
if (listed || named) found.push(x.storyId);
|
|
268
|
+
else if (!d) unread.push(x.storyId);
|
|
269
|
+
}
|
|
270
|
+
return { found, unread };
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** Why a story that was served before is not served again, in the words of what actually happened. */
|
|
274
|
+
function priorWhy(prior, runId) {
|
|
275
|
+
const b = prior.batchId;
|
|
276
|
+
if (prior.state === 'kill-failed') {
|
|
277
|
+
return `it was queued again as batch ${b}, but this run could not be killed; kill it (hone run-story ${runId} --kill) once you have looked at that batch. Nothing more is tried`;
|
|
278
|
+
}
|
|
279
|
+
if (prior.state === 'adopted') {
|
|
280
|
+
return `a newer run of this story already exists${b ? ` (batch ${b})` : ''}, queued by an earlier re-serve or by you, so it is not re-served again`;
|
|
281
|
+
}
|
|
282
|
+
if (prior.state === 'queue-failed') return 're-serving it failed earlier in this run; nothing more is tried';
|
|
283
|
+
return `it names files again after being re-served as batch ${b}; a second re-serve is not tried`;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* HC-GF-351: may this hard-paused run be re-served now? One of
|
|
288
|
+
* - `{ action }`: yes, here is the action;
|
|
289
|
+
* - `{ retry }`: not this tick, something transient (the text says what), and the run is still moving;
|
|
290
|
+
* - `{ why }`: no, and a person has to look.
|
|
291
|
+
*/
|
|
292
|
+
function decideReserve({ r, d, request, runs, details, ctx, prior, planned, batchId }) {
|
|
293
|
+
if (prior) return { why: priorWhy(prior, r.runId) };
|
|
294
|
+
if (planned.has(r.storyId)) return { why: 'the same story is hard-paused twice in this batch; only one is re-served' };
|
|
295
|
+
const dep = dependentsOf(r.runId, runs, details);
|
|
296
|
+
if (dep.found.length) {
|
|
297
|
+
return { why: `${dep.found.length} run(s) in this batch depend on it (${dep.found.slice(0, 5).join(', ')}), and killing it fails them` };
|
|
298
|
+
}
|
|
299
|
+
if (dep.unread.length) {
|
|
300
|
+
return { why: `${dep.unread.length} queued or paused run(s) in this batch could not be checked for a dependency on it, and killing it fails its dependents` };
|
|
301
|
+
}
|
|
302
|
+
if (!isStoryId(r.storyId)) return { why: 'its story id cannot be written into a stories file' };
|
|
303
|
+
const rank = stepRank(request.stepKey);
|
|
304
|
+
const verified = verifiedStep4Attempt(d);
|
|
305
|
+
if (rank > 400 && verified !== null) {
|
|
306
|
+
return { why: `step_4 attempt ${verified} already passed verify-patch, and re-serving from ${request.stepKey} would throw that verified work away` };
|
|
307
|
+
}
|
|
308
|
+
if (ctx.tracked == null && ctx.trackedRetry) {
|
|
309
|
+
return { retry: `could not list the files git tracks (${oneLine(ctx.trackedRetry, 120)}); trying again next poll` };
|
|
310
|
+
}
|
|
311
|
+
const original = originalPathsOf(d);
|
|
312
|
+
if (original.dropped) {
|
|
313
|
+
return { why: `${original.dropped} path(s) on the story's own line hold characters a shell could misread; nothing was served` };
|
|
314
|
+
}
|
|
315
|
+
const judged = judgeRequest(request, { tracked: ctx.tracked, realPath: ctx.realPath, original: original.paths });
|
|
316
|
+
if (!judged.ok) return { why: judged.reason };
|
|
317
|
+
return { action: { kind: 'reserve', runId: r.runId, storyId: r.storyId, stepKey: request.stepKey, batchId, paths: judged.tokens,
|
|
318
|
+
since: d.createdAt || null, repoName: r.repoName || ctx.repoName || null, branch: r.branch || null } };
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** What the loop says when every unfinished run is waiting on a person. */
|
|
322
|
+
function stuckReason(waiting) {
|
|
323
|
+
const who = waiting.map((w) => `${w.storyId} ${w.runId} at ${w.stepKey || 'a step'} (${oneLine(w.reason, 120)})`);
|
|
324
|
+
return `every unfinished run is waiting on you: ${who.join('; ')}`;
|
|
325
|
+
}
|
|
326
|
+
|
|
150
327
|
/**
|
|
151
328
|
* One tick's decisions, from the batch and the details of its paused runs. Pure.
|
|
329
|
+
*
|
|
330
|
+
* `stop.kind` says why the batch ends the loop: `breaker` (stop before acting), `terminal` (the server settled
|
|
331
|
+
* it), or `stuck` (HC-GF-351: nothing is running, nothing is left to do, and the unfinished runs wait on a person).
|
|
332
|
+
* `waiting` lists those runs, `moving` counts the ones still in motion, and `follow` the batches this one's killed
|
|
333
|
+
* runs were re-served as.
|
|
334
|
+
*
|
|
152
335
|
* @param {object} batch `GET /orchestrate/batch/:id`
|
|
153
336
|
* @param {Record<string, object>} details runId → `GET /orchestrate/:runId`, for paused runs and runs to emit
|
|
154
|
-
* @param {{emitted: Set<string
|
|
155
|
-
*
|
|
156
|
-
*
|
|
337
|
+
* @param {{emitted: Set<string>, reserved?: Map<string, {batchId: string|null, state: string}>}} memory `emitted`: runs
|
|
338
|
+
* whose emit-pr already ran in this process; `reserved`: story → what became of its re-serve (`state`: served,
|
|
339
|
+
* kill-failed, queue-failed or adopted)
|
|
340
|
+
* @param {{repoName?: string|null, openPr?: boolean, tracked?: Set<string>|null, trackedRetry?: string|null,
|
|
341
|
+
* realPath?: ((file: string) => string|null)|null}} [ctx] this checkout, so a run for another repository is
|
|
342
|
+
* not touched; `tracked`: the regular files git tracks here, which a request is checked against (`trackedRetry`:
|
|
343
|
+
* why listing them failed this tick, which is worth another poll); `realPath`: where a path resolves on the disk
|
|
344
|
+
* @returns {{stop: {kind: string, exitCode: number, reason: string}|null, actions: object[], notes: string[],
|
|
345
|
+
* waiting: Array<{runId: string, storyId: string, stepKey: string|null, reason: string}>,
|
|
346
|
+
* moving: number, follow: Array<{storyId: string, batchId: string}>}}
|
|
157
347
|
*/
|
|
158
348
|
function planTick(batch, details, memory, ctx = {}) {
|
|
159
349
|
const b = (batch && typeof batch === 'object') ? batch : {};
|
|
160
350
|
const halt = breakerStop(b);
|
|
161
|
-
if (halt) return { stop: { exitCode: EXIT.NEEDS_YOU, reason: halt }
|
|
351
|
+
if (halt) return tickPlan({ stop: { kind: 'breaker', exitCode: EXIT.NEEDS_YOU, reason: halt } });
|
|
162
352
|
|
|
163
353
|
const actions = [];
|
|
164
354
|
const notes = [];
|
|
165
|
-
|
|
355
|
+
const waiting = [];
|
|
356
|
+
let moving = 0;
|
|
357
|
+
const runs = Array.isArray(b.runs) ? b.runs : [];
|
|
358
|
+
// Stories already re-served: by this process (memory), or by an earlier one (a killed run says so).
|
|
359
|
+
const follow = reservedFrom(b);
|
|
360
|
+
const reserved = new Map(memory.reserved || []);
|
|
361
|
+
for (const f of follow) reserved.set(f.storyId, { batchId: f.batchId, state: 'served' });
|
|
362
|
+
const planned = new Set();
|
|
363
|
+
const waitOn = (r, stepKey, reason) => waiting.push({ runId: r.runId, storyId: r.storyId, stepKey, reason });
|
|
364
|
+
const needsYou = (r, stepKey, reason) => {
|
|
365
|
+
const why = oneLine(reason, 300);
|
|
366
|
+
notes.push(`${r.storyId} ${r.runId} needs you at ${stepKey || 'a step'}: ${why}`);
|
|
367
|
+
waitOn(r, stepKey, why);
|
|
368
|
+
};
|
|
369
|
+
|
|
370
|
+
for (const r of runs) {
|
|
166
371
|
if (!r || !r.runId) continue;
|
|
167
372
|
if (ctx.repoName && r.repoName && r.repoName !== ctx.repoName) {
|
|
168
373
|
notes.push(`${r.storyId} ${r.runId} belongs to ${r.repoName}, not this checkout (${ctx.repoName}): skipped`);
|
|
374
|
+
if (!TERMINAL_RUN.has(r.status) && r.status !== 'paused') moving += 1;
|
|
169
375
|
continue;
|
|
170
376
|
}
|
|
171
377
|
if (r.status === 'completed') {
|
|
@@ -178,14 +384,19 @@ function planTick(batch, details, memory, ctx = {}) {
|
|
|
178
384
|
}
|
|
179
385
|
continue;
|
|
180
386
|
}
|
|
181
|
-
if (r.status !== 'paused'
|
|
387
|
+
if (r.status !== 'paused') {
|
|
388
|
+
if (!TERMINAL_RUN.has(r.status)) moving += 1;
|
|
389
|
+
continue;
|
|
390
|
+
}
|
|
391
|
+
const d = details[r.runId];
|
|
392
|
+
if (!d) { moving += 1; continue; }
|
|
182
393
|
// HC-GF-301: opening PRs, a step_4 attempt that passed verify-patch gets its draft now, while step_5 and its
|
|
183
394
|
// gate are still ahead, instead of when the whole run completes.
|
|
184
|
-
const verified = ctx.openPr ? verifiedStep4Attempt(
|
|
395
|
+
const verified = ctx.openPr ? verifiedStep4Attempt(d) : null;
|
|
185
396
|
if (verified && !(memory.drafted && memory.drafted.has(`${r.runId}#${verified}`))) {
|
|
186
|
-
actions.push({ kind: 'draft', runId: r.runId, storyId: r.storyId, attempt: verified, run:
|
|
397
|
+
actions.push({ kind: 'draft', runId: r.runId, storyId: r.storyId, attempt: verified, run: d,
|
|
187
398
|
branch: draftBranch(r.storyId, r.runId, verified) });
|
|
188
|
-
const vs = Array.isArray(
|
|
399
|
+
const vs = Array.isArray(d.verifications) ? d.verifications : [];
|
|
189
400
|
const earlier = vs.filter((v) => v && v.source === 'verify-patch' && v.stepKey === 'step_4' && v.verdict === 'pass'
|
|
190
401
|
&& Number(v.attempt) < verified).map((v) => Number(v.attempt));
|
|
191
402
|
if (earlier.length) {
|
|
@@ -193,28 +404,114 @@ function planTick(batch, details, memory, ctx = {}) {
|
|
|
193
404
|
notes.push(`${r.storyId} ${r.runId}: the draft for attempt ${verified} supersedes the one on ${draftBranch(r.storyId, r.runId, m)}; close that one when you are ready`);
|
|
194
405
|
}
|
|
195
406
|
}
|
|
196
|
-
|
|
407
|
+
|
|
408
|
+
// HC-GF-351: a hard pause for files the step was not given. Served once per story, and only if every path passes.
|
|
409
|
+
const request = hardPauseRequest(d);
|
|
410
|
+
if (request) {
|
|
411
|
+
const decided = decideReserve({ r, d, request, runs, details, ctx, prior: reserved.get(r.storyId), planned, batchId: b.batchId });
|
|
412
|
+
if (decided.action) {
|
|
413
|
+
planned.add(r.storyId);
|
|
414
|
+
actions.push(decided.action);
|
|
415
|
+
} else if (decided.retry) {
|
|
416
|
+
notes.push(`${r.storyId} ${r.runId}: ${decided.retry}`);
|
|
417
|
+
moving += 1;
|
|
418
|
+
} else {
|
|
419
|
+
needsYou(r, request.stepKey, `it asked for files it was not given: ${decided.why}`);
|
|
420
|
+
}
|
|
421
|
+
continue;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
const a = step4Action(d);
|
|
197
425
|
// verify-patch's verdict POST is best-effort: an attempt this process already ran is not re-run every tick.
|
|
198
426
|
if (a.actionable && memory.tried && memory.tried.has(`${r.runId}#${a.attempt}`)) {
|
|
199
|
-
|
|
427
|
+
needsYou(r, 'step_4', `attempt ${a.attempt} was handled here (see the lines above) and the server holds no verdict for it`);
|
|
200
428
|
continue;
|
|
201
429
|
}
|
|
202
430
|
if (a.actionable) {
|
|
203
|
-
actions.push({ kind: 'verify', runId: r.runId, storyId: r.storyId, attempt: a.attempt, run:
|
|
431
|
+
actions.push({ kind: 'verify', runId: r.runId, storyId: r.storyId, attempt: a.attempt, run: d });
|
|
432
|
+
} else if (a.reason && approvalQueued(d)) {
|
|
433
|
+
moving += 1;
|
|
204
434
|
} else if (a.reason) {
|
|
205
|
-
|
|
435
|
+
needsYou(r, 'step_4', a.reason);
|
|
436
|
+
} else {
|
|
437
|
+
const p = otherPause(d, r);
|
|
438
|
+
if (p.kind === 'person') needsYou(r, p.stepKey, p.reason);
|
|
439
|
+
else if (p.kind === 'moving') moving += 1;
|
|
206
440
|
}
|
|
207
441
|
}
|
|
208
442
|
|
|
209
|
-
if (!isTerminal(b))
|
|
210
|
-
|
|
211
|
-
|
|
443
|
+
if (!isTerminal(b)) {
|
|
444
|
+
const stuck = actions.length === 0 && moving === 0 && waiting.length > 0;
|
|
445
|
+
return tickPlan({ stop: stuck ? { kind: 'stuck', exitCode: EXIT.NEEDS_YOU, reason: stuckReason(waiting) } : null,
|
|
446
|
+
actions, notes, waiting, moving, follow });
|
|
447
|
+
}
|
|
448
|
+
// A killed run that was re-served is not unfinished: its story went on in the batch it names.
|
|
449
|
+
const raw = (b.counts && ((b.counts.failed || 0) + (b.counts.killed || 0) + (b.counts.stalled || 0))) || 0;
|
|
450
|
+
const unfinished = Math.max(0, raw - follow.length);
|
|
451
|
+
return tickPlan({
|
|
212
452
|
stop: unfinished
|
|
213
|
-
? { exitCode: EXIT.NEEDS_YOU, reason: `batch ${b.status}: ${unfinished} run(s) did not complete` }
|
|
214
|
-
: { exitCode: EXIT.DONE, reason: `batch ${b.status}` },
|
|
215
|
-
actions,
|
|
216
|
-
|
|
217
|
-
|
|
453
|
+
? { kind: 'terminal', exitCode: EXIT.NEEDS_YOU, reason: `batch ${b.status}: ${unfinished} run(s) did not complete` }
|
|
454
|
+
: { kind: 'terminal', exitCode: EXIT.DONE, reason: `batch ${b.status}` },
|
|
455
|
+
actions, notes, waiting, moving, follow,
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/**
|
|
460
|
+
* HC-GF-351: queue the story again with the files its step asked for, then kill the paused run. In that order: the
|
|
461
|
+
* kill's reason names the new batch (the next start reads it back, which is what keeps the story from being served
|
|
462
|
+
* twice), and a story with no run in flight is worse than one with two. If the queue fails the paused run is left
|
|
463
|
+
* alone; if the kill fails the new batch still runs and the paused run is named for a person.
|
|
464
|
+
*
|
|
465
|
+
* Before queueing it asks the server whether the story was served already (io.priorServe): a kill that failed, or a
|
|
466
|
+
* crash between the queue and the kill, leaves no marker, and only a newer run of the same story shows it happened.
|
|
467
|
+
* @returns {Promise<{lines: string[], needsYou: boolean, batchId: string|null, state: string|null, retry?: boolean}>}
|
|
468
|
+
* `batchId`: the batch the story was queued as, set whenever it was, even when the kill then failed; `state`: what
|
|
469
|
+
* became of the re-serve (served, kill-failed, queue-failed, adopted); `retry`: nothing was decided, ask again
|
|
470
|
+
*/
|
|
471
|
+
async function reserve(action, io, tag) {
|
|
472
|
+
if (io.priorServe) {
|
|
473
|
+
let prior;
|
|
474
|
+
try {
|
|
475
|
+
prior = await io.priorServe(action);
|
|
476
|
+
} catch (e) {
|
|
477
|
+
return { lines: [`${tag} not re-served yet: could not ask whether the story was served already (${firstLine(e && e.message)}); asking again next poll`],
|
|
478
|
+
needsYou: false, batchId: null, state: null, retry: true };
|
|
479
|
+
}
|
|
480
|
+
if (prior && prior.kind === 'unknown') {
|
|
481
|
+
return { lines: [`${tag} not re-served: could not tell whether the story was served already; the paused run was left alone`],
|
|
482
|
+
needsYou: true, batchId: null, state: 'queue-failed' };
|
|
483
|
+
}
|
|
484
|
+
if (prior) {
|
|
485
|
+
const where = prior.batchId ? ` in batch ${prior.batchId}` : '';
|
|
486
|
+
return { lines: [`${tag} not re-served: run ${prior.runId}${where} is this story's ${prior.kind === 'later' ? 'newer' : 'earlier, re-served'} run; following it`],
|
|
487
|
+
needsYou: true, batchId: prior.batchId || null, state: 'adopted' };
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
let queued;
|
|
491
|
+
let file = null;
|
|
492
|
+
try {
|
|
493
|
+
file = io.writeStoriesFile(`${formatStoryLine({ storyId: action.storyId, paths: action.paths })}\n`);
|
|
494
|
+
queued = await io.runVerb(buildReserveArgs(file.file, { repoName: action.repoName, branch: action.branch }), { capture: true });
|
|
495
|
+
} catch (e) {
|
|
496
|
+
return { lines: [`${tag} not re-served: ${firstLine(e && e.message)}; the paused run was left alone`], needsYou: true, batchId: null, state: 'queue-failed' };
|
|
497
|
+
} finally {
|
|
498
|
+
if (file) file.remove();
|
|
499
|
+
}
|
|
500
|
+
const batchId = queued.code === 0 ? batchIdFromQueueOutput(queued.stdout) : null;
|
|
501
|
+
if (!batchId) {
|
|
502
|
+
const why = queued.code === 0 ? 'printed no batch id' : `exited ${queued.code}`;
|
|
503
|
+
return { lines: [`${tag} not re-served: queue-stories ${why}; the paused run was left alone`], needsYou: true, batchId: null, state: 'queue-failed' };
|
|
504
|
+
}
|
|
505
|
+
const lines = [`${tag} ${action.stepKey} named files it was not given: queued batch ${batchId} with paths:${action.paths.join(',')}`];
|
|
506
|
+
try {
|
|
507
|
+
await io.killRun(action.runId, reserveReason(batchId, action.stepKey));
|
|
508
|
+
} catch (e) {
|
|
509
|
+
const status = e && e.response && e.response.status;
|
|
510
|
+
lines.push(`${tag} could NOT kill the paused run (${status || firstLine(e && e.message)}); it is still paused beside batch ${batchId}: kill it with hone run-story ${action.runId} --kill`);
|
|
511
|
+
return { lines, needsYou: true, batchId, state: 'kill-failed' };
|
|
512
|
+
}
|
|
513
|
+
lines.push(`${tag} killed the paused run: ${reserveReason(batchId, action.stepKey)}`);
|
|
514
|
+
return { lines, needsYou: false, batchId, state: 'served' };
|
|
218
515
|
}
|
|
219
516
|
|
|
220
517
|
/**
|
|
@@ -229,6 +526,7 @@ function planTick(batch, details, memory, ctx = {}) {
|
|
|
229
526
|
*/
|
|
230
527
|
async function runTask(action, io, opts = {}) {
|
|
231
528
|
const tag = `[follow-batch] ${action.storyId} ${action.runId}`;
|
|
529
|
+
if (action.kind === 'reserve') return reserve(action, io, tag);
|
|
232
530
|
if (action.kind === 'emit' || action.kind === 'draft') {
|
|
233
531
|
// emit-pr re-runs the tests before it commits, even as a local dry run, so it needs the same consent and
|
|
234
532
|
// provenance verify-patch does. Without this, a repository that never set ci.local_command gets `make ci`.
|
|
@@ -255,11 +553,33 @@ async function runTask(action, io, opts = {}) {
|
|
|
255
553
|
return { lines, needsYou: v.code !== 0 };
|
|
256
554
|
}
|
|
257
555
|
|
|
556
|
+
/**
|
|
557
|
+
* What the loop returns once every batch it follows has ended (settled by the server, or stuck on a person).
|
|
558
|
+
* DONE only when every one of them completed and no emit is owed; the reason of each batch that did not is kept.
|
|
559
|
+
* A batch stuck on a person is not over: step_5's gate is mandatory, so this is where most runs end, and the reason
|
|
560
|
+
* says how to continue (`again`: the batches to follow once the person has acted).
|
|
561
|
+
*/
|
|
562
|
+
function settle(stops, memory, again = []) {
|
|
563
|
+
const open = stops.filter((s) => s.exitCode !== EXIT.DONE);
|
|
564
|
+
const owed = memory.owed.size ? `${memory.owed.size} run(s) were not emitted: ${[...memory.owed.values()].join(', ')}` : '';
|
|
565
|
+
if (open.length === 0 && !owed) return { exitCode: EXIT.DONE, reason: stops.map((s) => s.reason).join('; ') };
|
|
566
|
+
const why = [...open.map((s) => s.reason), owed].filter(Boolean);
|
|
567
|
+
const next = again.length ? ` (after you act, run again to emit: hone follow-batch ${again.join('; hone follow-batch ')})` : '';
|
|
568
|
+
return { exitCode: EXIT.NEEDS_YOU, reason: (open.length ? why.join('; ') : `${stops.map((s) => s.reason).join('; ')}, but ${owed}`) + next };
|
|
569
|
+
}
|
|
570
|
+
|
|
258
571
|
/**
|
|
259
572
|
* The loop. Every effect is injected, so a test drives it with a stub server and a recording runVerb.
|
|
573
|
+
*
|
|
574
|
+
* It follows a SET of batches: the one it was given, and every batch a killed run says its story was re-served as
|
|
575
|
+
* (HC-GF-351), found in the same tick. It ends when every one of them has ended, or when one has fired a breaker.
|
|
260
576
|
* @param {string} batchId
|
|
261
577
|
* @param {{fetchBatch: Function, fetchRun: Function, runVerb: Function, verifyDecision: Function,
|
|
262
|
-
* sleep: (ms: number) => Promise<void>, now: () => number, log: (line: string) => void
|
|
578
|
+
* sleep: (ms: number) => Promise<void>, now: () => number, log: (line: string) => void,
|
|
579
|
+
* trackedPaths?: () => Promise<Set<string>>|Set<string>, realPath?: (file: string) => string|null,
|
|
580
|
+
* priorServe?: Function, writeStoriesFile?: Function, killRun?: Function}} io
|
|
581
|
+
* trackedPaths, realPath, priorServe, writeStoriesFile and killRun serve a re-serve; without trackedPaths or
|
|
582
|
+
* realPath every request is refused, and without priorServe nobody asks whether the story was served already.
|
|
263
583
|
* @param {{pollMs?: number, deadlineMs?: number|null, maxOfflineMs?: number, openPr?: boolean,
|
|
264
584
|
* repoName?: string|null}} [opts]
|
|
265
585
|
* @returns {Promise<{exitCode: number, reason: string}>}
|
|
@@ -268,8 +588,10 @@ async function followBatch(batchId, io, opts = {}) {
|
|
|
268
588
|
const pollMs = opts.pollMs || 30_000;
|
|
269
589
|
const maxOfflineMs = opts.maxOfflineMs || 15 * 60_000;
|
|
270
590
|
// owed: runs whose emit-pr needed a person. A completed batch with one of these is not DONE (HC-GF-207's shape:
|
|
271
|
-
// a failure path that exits 0).
|
|
272
|
-
|
|
591
|
+
// a failure path that exits 0). reserved: story → what became of its re-serve ({batchId, state}). trackedFails:
|
|
592
|
+
// consecutive ticks on which listing the tracked files failed.
|
|
593
|
+
const memory = { emitted: new Set(), tried: new Set(), drafted: new Set(), said: new Set(), owed: new Map(), reserved: new Map(), trackedFails: 0 };
|
|
594
|
+
const followed = [batchId];
|
|
273
595
|
const start = io.now();
|
|
274
596
|
let offlineSince = null;
|
|
275
597
|
const say = (line) => { if (!memory.said.has(line)) { memory.said.add(line); io.log(line); } };
|
|
@@ -280,19 +602,36 @@ async function followBatch(batchId, io, opts = {}) {
|
|
|
280
602
|
}
|
|
281
603
|
// A tick reads everything or acts on nothing: a run that could not be read is the server being unreachable,
|
|
282
604
|
// not a run with no details (which would skip its gate, or emit it with no provenance to assess).
|
|
283
|
-
|
|
284
|
-
|
|
605
|
+
const reads = [];
|
|
606
|
+
let reading = batchId;
|
|
285
607
|
try {
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
const
|
|
289
|
-
|
|
608
|
+
for (let i = 0; i < followed.length; i += 1) {
|
|
609
|
+
reading = followed[i];
|
|
610
|
+
const batch = await io.fetchBatch(reading);
|
|
611
|
+
const details = {};
|
|
612
|
+
for (const r of Array.isArray(batch && batch.runs) ? batch.runs : []) {
|
|
613
|
+
const wanted = r && r.runId && (r.status === 'paused' || (r.status === 'completed' && !memory.emitted.has(r.runId)));
|
|
614
|
+
if (wanted) details[r.runId] = await io.fetchRun(r.runId);
|
|
615
|
+
}
|
|
616
|
+
// A run hard-paused on a request is only re-served if nothing depends on it, and a queued run has not said
|
|
617
|
+
// what it depends on: its config is the only place that is written. Read them when a request is waiting.
|
|
618
|
+
if (Object.values(details).some((d) => hardPauseRequest(d))) {
|
|
619
|
+
for (const r of Array.isArray(batch && batch.runs) ? batch.runs : []) {
|
|
620
|
+
if (r && r.runId && r.status === 'queued' && !details[r.runId]) details[r.runId] = await io.fetchRun(r.runId);
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
reads.push({ batch, details });
|
|
624
|
+
// A killed run that says where its story went: follow that batch, and do not serve the story again.
|
|
625
|
+
for (const f of reservedFrom(batch)) {
|
|
626
|
+
memory.reserved.set(f.storyId, { batchId: f.batchId, state: 'served' });
|
|
627
|
+
if (!followed.includes(f.batchId)) followed.push(f.batchId);
|
|
628
|
+
}
|
|
290
629
|
}
|
|
291
630
|
offlineSince = null;
|
|
292
631
|
} catch (e) {
|
|
293
632
|
const status = e && e.response && e.response.status;
|
|
294
633
|
if (status === 401 || status === 403 || status === 404) {
|
|
295
|
-
return { exitCode: EXIT.FATAL, reason: `the server answered ${status} while reading batch ${
|
|
634
|
+
return { exitCode: EXIT.FATAL, reason: `the server answered ${status} while reading batch ${reading}` };
|
|
296
635
|
}
|
|
297
636
|
if (offlineSince === null) offlineSince = io.now();
|
|
298
637
|
if (io.now() - offlineSince >= maxOfflineMs) {
|
|
@@ -302,32 +641,60 @@ async function followBatch(batchId, io, opts = {}) {
|
|
|
302
641
|
continue;
|
|
303
642
|
}
|
|
304
643
|
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
}
|
|
316
|
-
memory.
|
|
644
|
+
// The checkout's tracked files, asked for only when a run is hard-paused on a request. Without them every
|
|
645
|
+
// path is refused: a request cannot be served on a list that could not be read. A listing that fails is
|
|
646
|
+
// worth three more polls (git can be busy), not an exit.
|
|
647
|
+
let tracked = null;
|
|
648
|
+
let trackedRetry = null;
|
|
649
|
+
const asks = reads.some((x) => Object.values(x.details).some((d) => hardPauseRequest(d)));
|
|
650
|
+
if (asks && io.trackedPaths) {
|
|
651
|
+
try {
|
|
652
|
+
tracked = await io.trackedPaths();
|
|
653
|
+
memory.trackedFails = 0;
|
|
654
|
+
} catch (e) {
|
|
655
|
+
memory.trackedFails += 1;
|
|
656
|
+
if (memory.trackedFails <= 3) trackedRetry = firstLine(e && e.message) || 'git ls-files failed';
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
const ctx = { repoName: opts.repoName || null, openPr: !!opts.openPr, tracked, trackedRetry, realPath: io.realPath || null };
|
|
660
|
+
const plans = reads.map((x) => planTick(x.batch, x.details, memory, ctx));
|
|
661
|
+
for (const plan of plans) for (const n of plan.notes) say(`[follow-batch] ${n}`);
|
|
662
|
+
// A fired breaker stops the loop BEFORE any batch's action runs.
|
|
663
|
+
const halted = plans.find((plan) => plan.stop && plan.stop.kind === 'breaker');
|
|
664
|
+
if (halted) return { exitCode: halted.stop.exitCode, reason: halted.stop.reason };
|
|
665
|
+
|
|
666
|
+
let grew = false;
|
|
667
|
+
for (const plan of plans) {
|
|
668
|
+
for (const action of plan.actions) {
|
|
669
|
+
const out = await runTask(action, io, { openPr: opts.openPr });
|
|
670
|
+
if (action.kind === 'reserve') {
|
|
671
|
+
if (!out.retry) {
|
|
672
|
+
memory.reserved.set(action.storyId, { batchId: out.batchId, state: out.state });
|
|
673
|
+
if (out.batchId && !followed.includes(out.batchId)) { followed.push(out.batchId); grew = true; }
|
|
674
|
+
}
|
|
675
|
+
} else if (action.kind === 'emit' || action.kind === 'draft') {
|
|
676
|
+
if (action.kind === 'emit') memory.emitted.add(action.runId);
|
|
677
|
+
else memory.drafted.add(`${action.runId}#${action.attempt}`);
|
|
678
|
+
// A run is owed until an emit for it succeeds: a draft that failed is retried by the completed-run emit.
|
|
679
|
+
if (out.needsYou) memory.owed.set(action.runId, `${action.storyId} ${action.runId}`);
|
|
680
|
+
else if (action.kind === 'emit') memory.owed.delete(action.runId);
|
|
681
|
+
} else {
|
|
682
|
+
memory.tried.add(`${action.runId}#${action.attempt}`);
|
|
683
|
+
}
|
|
684
|
+
for (const line of out.lines) io.log(line);
|
|
317
685
|
}
|
|
318
|
-
for (const line of out.lines) io.log(line);
|
|
319
686
|
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
687
|
+
// Ended means every batch read this tick has: settled by the server, or stuck on a person. A batch queued
|
|
688
|
+
// this tick has not been read yet.
|
|
689
|
+
if (!grew && plans.every((plan) => plan.stop)) {
|
|
690
|
+
return settle(plans.map((plan) => plan.stop), memory, followed.filter((id, i) => plans[i].stop.kind === 'stuck'));
|
|
323
691
|
}
|
|
324
|
-
if (plan.stop) return plan.stop;
|
|
325
692
|
await io.sleep(pollMs);
|
|
326
693
|
}
|
|
327
694
|
}
|
|
328
695
|
|
|
329
696
|
module.exports = {
|
|
330
697
|
EXIT, FORBIDDEN_FLAGS, NOT_A_GATE,
|
|
331
|
-
breakerStop, isTerminal, priorVerdict, step4Action, buildArgs, planTick, runTask, followBatch,
|
|
332
|
-
currentStep4Attempt, verifiedStep4Attempt, draftBranch,
|
|
698
|
+
breakerStop, isTerminal, priorVerdict, step4Action, buildArgs, buildReserveArgs, planTick, runTask, followBatch,
|
|
699
|
+
currentStep4Attempt, verifiedStep4Attempt, draftBranch, reservedFrom, otherPause, stuckReason, gateEnforced, stepRank,
|
|
333
700
|
};
|