omp-conductor 0.20.0 → 0.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/schema/config.schema.json +24 -0
- package/src/admission.ts +58 -14
- package/src/arm-challenge.ts +54 -3
- package/src/briefs/console.md +10 -5
- package/src/commands/arm.ts +7 -5
- package/src/commands/companion.ts +52 -16
- package/src/commands/drain.ts +12 -5
- package/src/commands/worker.ts +5 -0
- package/src/config-schema.ts +8 -0
- package/src/config.ts +17 -2
- package/src/daemon/drain.ts +33 -0
- package/src/daemon/groom-pass.ts +16 -6
- package/src/daemon/http.ts +28 -0
- package/src/daemon/review.ts +11 -11
- package/src/daemon/runtime.ts +55 -3
- package/src/daemon/settle-pass.ts +19 -2
- package/src/daemon/supervision.ts +144 -1
- package/src/daemon/tick.ts +31 -14
- package/src/daemon/views.ts +17 -0
- package/src/daemon.ts +4 -3
- package/src/decisions.ts +20 -9
- package/src/diff-flags.ts +204 -28
- package/src/doctor.ts +27 -9
- package/src/escalate.ts +187 -15
- package/src/failure-class.ts +358 -5
- package/src/fleet.ts +55 -25
- package/src/graph-health.ts +17 -1
- package/src/groom.ts +11 -0
- package/src/orchestrator-tick.ts +302 -24
- package/src/reports.ts +4 -1
- package/src/settlement.ts +133 -12
- package/src/status-render.ts +38 -9
- package/src/store.ts +64 -15
- package/src/to-spec.ts +285 -24
- package/src/tracker/github.ts +132 -10
- package/src/types.ts +49 -3
- package/src/verbs/server.ts +16 -12
- package/src/worker.ts +149 -35
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "omp-conductor",
|
|
3
|
-
"version": "0.20.
|
|
3
|
+
"version": "0.20.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",
|
|
@@ -92,6 +92,18 @@
|
|
|
92
92
|
"minimum": 0,
|
|
93
93
|
"description": "Wall-clock ceiling for one worker (default 5400000)"
|
|
94
94
|
},
|
|
95
|
+
"workerStallSilenceMs": {
|
|
96
|
+
"anyOf": [
|
|
97
|
+
{
|
|
98
|
+
"type": "number",
|
|
99
|
+
"minimum": 0
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
"type": "null"
|
|
103
|
+
}
|
|
104
|
+
],
|
|
105
|
+
"description": "How long a live worker's transcript may stay unwritten before the daemon settles it as a progress stall; null derives a third of workerWallClockMs"
|
|
106
|
+
},
|
|
95
107
|
"maxAttemptsPerIssue": {
|
|
96
108
|
"type": "number",
|
|
97
109
|
"minimum": 0,
|
|
@@ -357,6 +369,18 @@
|
|
|
357
369
|
"minimum": 0,
|
|
358
370
|
"description": "Wall-clock ceiling for one worker (default 5400000)"
|
|
359
371
|
},
|
|
372
|
+
"workerStallSilenceMs": {
|
|
373
|
+
"anyOf": [
|
|
374
|
+
{
|
|
375
|
+
"type": "number",
|
|
376
|
+
"minimum": 0
|
|
377
|
+
},
|
|
378
|
+
{
|
|
379
|
+
"type": "null"
|
|
380
|
+
}
|
|
381
|
+
],
|
|
382
|
+
"description": "How long a live worker's transcript may stay unwritten before the daemon settles it as a progress stall; null derives a third of workerWallClockMs"
|
|
383
|
+
},
|
|
360
384
|
"maxAttemptsPerIssue": {
|
|
361
385
|
"type": "number",
|
|
362
386
|
"minimum": 0,
|
package/src/admission.ts
CHANGED
|
@@ -270,7 +270,9 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
|
|
|
270
270
|
* optional bold/heading markers) whose rest carries the paths as
|
|
271
271
|
* backtick-delimited spans (the brief form), with a bare
|
|
272
272
|
* comma/space-separated fallback that keeps tokens that look like relative
|
|
273
|
-
* paths.
|
|
273
|
+
* paths. The paths are the *leading* backticked run: a comma/space-separated
|
|
274
|
+
* list at the line's start, and any backticked path later on the line is
|
|
275
|
+
* prose, not a write target (#1073).
|
|
274
276
|
* - The write-lane section (#825): a markdown heading "Exact write lane" (or
|
|
275
277
|
* "Write lane" / "write-lane") whose immediately following bullet items
|
|
276
278
|
* carry the paths — the package-floor decomposition format, where a groomed
|
|
@@ -278,7 +280,9 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
|
|
|
278
280
|
* the contiguous bullet run under that heading is read: a following
|
|
279
281
|
* paragraph (like a "Read only:" caveat) or a later section ends it, so
|
|
280
282
|
* read-only entry points, proof commands and acceptance bullets elsewhere in
|
|
281
|
-
* the issue are never captured.
|
|
283
|
+
* the issue are never captured. Each bullet contributes its leading
|
|
284
|
+
* backticked path run alone — a filename named in a bullet's prose (say, to
|
|
285
|
+
* note that it does not exist) is prose, never a lane entry (#1073).
|
|
282
286
|
*
|
|
283
287
|
* Inline wins when both are present — the machine-shaped sentence has held
|
|
284
288
|
* since #555, and the mediated label echo shows exactly which declaration
|
|
@@ -295,20 +299,59 @@ export function laneDeclaration(text: string): LaneDeclaration | undefined {
|
|
|
295
299
|
return inlineLaneDeclaration(text) ?? sectionLaneDeclaration(text);
|
|
296
300
|
}
|
|
297
301
|
|
|
302
|
+
/** One declared surface's paths (#1073). The canonical spelling is a leading
|
|
303
|
+
* run of backticked paths — a comma/space-separated list at the surface's
|
|
304
|
+
* start. A backticked path later in the same surface is prose, not part of
|
|
305
|
+
* the lane: a parenthetical like "(there is no `settle-pass.test.ts`)" is a
|
|
306
|
+
* sentence for the reader, and reading it as a write target makes a correct
|
|
307
|
+
* issue permanently refuse. A surface with no leading backticked path keeps
|
|
308
|
+
* the fallback: every backticked pathlike token, else bare
|
|
309
|
+
* comma/space-separated pathlike tokens (the unbackticked spelling). */
|
|
310
|
+
function writeLanePaths(text: string): string[] {
|
|
311
|
+
const fromRun = leadingBacktickedRun(text).filter(isPathLike);
|
|
312
|
+
if (fromRun.length > 0) return fromRun;
|
|
313
|
+
const backticked = [...text.matchAll(/`([^`]+)`/g)]
|
|
314
|
+
.map((m) => m[1]!.trim())
|
|
315
|
+
.filter(isPathLike);
|
|
316
|
+
return backticked.length > 0
|
|
317
|
+
? [...new Set(backticked)]
|
|
318
|
+
: [...new Set(text.split(/[,\s]+/).map((s) => s.trim()).filter(isPathLike))];
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** The leading backticked run of a surface (or bullet): consecutive
|
|
322
|
+
* backtick-delimited tokens separated by commas or whitespace, starting at
|
|
323
|
+
* the first character. A run is a *list*; the first non-backtick,
|
|
324
|
+
* non-separator character ends it, so "`a.ts`, `b.ts`" yields both while
|
|
325
|
+
* "`a.ts` — see `b.ts`" yields only `a.ts`. Empty when the surface does not
|
|
326
|
+
* open with a backtick. */
|
|
327
|
+
function leadingBacktickedRun(text: string): string[] {
|
|
328
|
+
const trimmed = text.trim();
|
|
329
|
+
if (!trimmed.startsWith("`")) return [];
|
|
330
|
+
const run: string[] = [];
|
|
331
|
+
let pos = 0;
|
|
332
|
+
while (true) {
|
|
333
|
+
if (trimmed[pos] !== "`") break;
|
|
334
|
+
const close = trimmed.indexOf("`", pos + 1);
|
|
335
|
+
if (close < 0) break;
|
|
336
|
+
const token = trimmed.slice(pos + 1, close).trim();
|
|
337
|
+
if (token === "") break;
|
|
338
|
+
run.push(token);
|
|
339
|
+
pos = close + 1;
|
|
340
|
+
const sep = /^[,\s]+/.exec(trimmed.slice(pos));
|
|
341
|
+
if (sep === null) break;
|
|
342
|
+
pos += sep[0].length;
|
|
343
|
+
}
|
|
344
|
+
return run;
|
|
345
|
+
}
|
|
346
|
+
|
|
298
347
|
/** The inline `File lane:`/`File-lane=` sentence grammar (see {@link laneDeclaration}). */
|
|
299
348
|
function inlineLaneDeclaration(text: string): LaneDeclaration | undefined {
|
|
300
349
|
const match = text.match(
|
|
301
|
-
/^\s*(?:[#>*-]\s*)*file[- ]lane\s*[:=]\s*([^\n]*)$/im,
|
|
350
|
+
/^\s*(?:[#>*-]\s*)*file[- ]lane\s*[:=]\s*(?:(?:\*\*|\*|__|_)\s*)?([^\n]*)$/im,
|
|
302
351
|
);
|
|
303
352
|
if (match === null) return undefined;
|
|
304
353
|
const rest = match[1] ?? "";
|
|
305
|
-
const
|
|
306
|
-
.map((m) => m[1]!.trim())
|
|
307
|
-
.filter(isPathLike);
|
|
308
|
-
const files =
|
|
309
|
-
backticked.length > 0
|
|
310
|
-
? [...new Set(backticked)]
|
|
311
|
-
: [...new Set(rest.split(/[,\s]+/).map((s) => s.trim()).filter(isPathLike))];
|
|
354
|
+
const files = writeLanePaths(rest);
|
|
312
355
|
if (files.length === 0) return undefined;
|
|
313
356
|
return { files, source: match[0].trim() };
|
|
314
357
|
}
|
|
@@ -351,10 +394,11 @@ function sectionLaneDeclaration(text: string): LaneDeclaration | undefined {
|
|
|
351
394
|
const marker = line.match(/^[ \t]*(?:[-*+]|\d+[.)])[ \t]+(?:\[[ xX]\][ \t]+)?/);
|
|
352
395
|
if (marker === null) break;
|
|
353
396
|
const rest = line.slice(marker[0].length);
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
397
|
+
// One bullet contributes its leading backticked path run — what it
|
|
398
|
+
// declares it writes, not every backticked token its prose mentions.
|
|
399
|
+
// A bullet without a leading backticked path keeps the fallback
|
|
400
|
+
// (bare pathlike tokens), so unbackticked lanes are unaffected.
|
|
401
|
+
files.push(...writeLanePaths(rest));
|
|
358
402
|
bullets.push(line.trim());
|
|
359
403
|
}
|
|
360
404
|
if (files.length === 0) return undefined;
|
package/src/arm-challenge.ts
CHANGED
|
@@ -5,9 +5,12 @@
|
|
|
5
5
|
* orchestrator-workflow redesign).
|
|
6
6
|
*
|
|
7
7
|
* `armTicks` (fleet.ts) sends a short-lived `FLEET-…` code to the operator and
|
|
8
|
-
* files the challenge here. Verification
|
|
9
|
-
*
|
|
10
|
-
*
|
|
8
|
+
* files the challenge here. Verification classifies the operator's message
|
|
9
|
+
* against these records and arms the projects the challenge named — either
|
|
10
|
+
* mechanically in the orchestrator session, which owns the project topic the
|
|
11
|
+
* challenge is sent to (#1061), or through the CLI step
|
|
12
|
+
* `omp-conductor arm --reply "<the operator's message>"` from a console host
|
|
13
|
+
* whose DM the operator answered in.
|
|
11
14
|
*
|
|
12
15
|
* Nothing waits for the reply any more. The console session owns the operator
|
|
13
16
|
* DM, and no tick extension runs there — so the in-session acknowledgement
|
|
@@ -498,3 +501,51 @@ export function observeArmChallenge(project: string | undefined): ArmChallengeSi
|
|
|
498
501
|
...(ack === undefined ? {} : { acknowledgedAt: ack.acknowledgedAt }),
|
|
499
502
|
};
|
|
500
503
|
}
|
|
504
|
+
|
|
505
|
+
/** One expired, unsettled ceremony, with the key the notice clears it under. */
|
|
506
|
+
export interface ExpiredArmChallenge {
|
|
507
|
+
/**
|
|
508
|
+
* The record's own state key — the project key or {@link FLEET_ARM_KEY} —
|
|
509
|
+
* passed straight back to {@link clearArmTransaction} after the notice, so
|
|
510
|
+
* the clear is addressed to the record that actually expired.
|
|
511
|
+
*/
|
|
512
|
+
key: string;
|
|
513
|
+
/** The expired transaction's id. */
|
|
514
|
+
id: string;
|
|
515
|
+
/** The sighting, acknowledgement included when a reply was seen. */
|
|
516
|
+
sighting: ArmChallengeSighting;
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
/**
|
|
520
|
+
* Every pending challenge that is no longer a proof — the project's own record
|
|
521
|
+
* and the fleet-wide one, in the same consultation order as
|
|
522
|
+
* {@link resolveArmReply} — whose window has passed without being settled.
|
|
523
|
+
* An acknowledgement may already exist (the reply was seen but the settle
|
|
524
|
+
* never completed), which is still unsettled: the notice must name that
|
|
525
|
+
* reason, and {@link doctor} does the same. This is what lets the fleet
|
|
526
|
+
* session tell the operator a ceremony died instead of letting the record sit
|
|
527
|
+
* invisible until the next arm replaces it.
|
|
528
|
+
*/
|
|
529
|
+
export function expiredArmChallenges(
|
|
530
|
+
project: string | undefined,
|
|
531
|
+
now: number,
|
|
532
|
+
): ExpiredArmChallenge[] {
|
|
533
|
+
const keys = projectKey(project) === FLEET_ARM_KEY ? [FLEET_ARM_KEY] : [projectKey(project), FLEET_ARM_KEY];
|
|
534
|
+
const expired: ExpiredArmChallenge[] = [];
|
|
535
|
+
for (const key of keys) {
|
|
536
|
+
const pending = readPendingFor(key);
|
|
537
|
+
if (pending === undefined || pending.expiresAt > now) continue;
|
|
538
|
+
const ack = readArmAcknowledgement(pending.id);
|
|
539
|
+
expired.push({
|
|
540
|
+
key,
|
|
541
|
+
id: pending.id,
|
|
542
|
+
sighting: {
|
|
543
|
+
id: pending.id,
|
|
544
|
+
...(typeof pending.sentAt === "number" ? { sentAt: pending.sentAt } : {}),
|
|
545
|
+
...(typeof pending.expiresAt === "number" ? { expiresAt: pending.expiresAt } : {}),
|
|
546
|
+
...(ack === undefined ? {} : { acknowledgedAt: ack.acknowledgedAt }),
|
|
547
|
+
},
|
|
548
|
+
});
|
|
549
|
+
}
|
|
550
|
+
return expired;
|
|
551
|
+
}
|
package/src/briefs/console.md
CHANGED
|
@@ -156,13 +156,18 @@ same turn — an approved answer is work to execute, not a proposal to re-open.
|
|
|
156
156
|
|
|
157
157
|
## Arm ceremony
|
|
158
158
|
|
|
159
|
-
Ticks are gated on an operator-owned marker, and
|
|
160
|
-
that writes it. Two mechanical
|
|
159
|
+
Ticks are gated on an operator-owned marker, and something real always owns
|
|
160
|
+
the ceremony that writes it. Two mechanical halves, and nothing waits
|
|
161
|
+
anywhere:
|
|
161
162
|
|
|
162
163
|
1. `omp-conductor arm [--project {{PROJECT}}]` — records the challenge, sends it
|
|
163
|
-
to the
|
|
164
|
-
It writes no marker.
|
|
165
|
-
2.
|
|
164
|
+
to the project topic (this fleet's own chat), and returns immediately,
|
|
165
|
+
naming the exact follow-up command. It writes no marker.
|
|
166
|
+
2. A reply settles the ceremony. A reply sent in the project topic is consumed
|
|
167
|
+
mechanically by the orchestrator session itself — the challenge is sent to
|
|
168
|
+
the topic that session claims, so on a host with no console session the
|
|
169
|
+
ceremony still completes on its own. A reply sent here in the DM is yours:
|
|
170
|
+
`omp-conductor arm --reply "<the operator's message, verbatim>" [--project
|
|
166
171
|
{{PROJECT}}]` — classifies that reply and, on a match, writes the marker for
|
|
167
172
|
the targets the ceremony recorded.
|
|
168
173
|
|
package/src/commands/arm.ts
CHANGED
|
@@ -4,10 +4,11 @@
|
|
|
4
4
|
* `arm` files a challenge, sends it, and returns immediately naming the
|
|
5
5
|
* follow-up command. `arm --reply "<the operator's message>"` verifies that
|
|
6
6
|
* message and writes the arm marker for exactly the projects the challenge
|
|
7
|
-
* recorded. Nothing waits anywhere:
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
7
|
+
* recorded. Nothing waits anywhere: a reply sent to the project topic is
|
|
8
|
+
* consumed mechanically by the orchestrator pane itself (#1061), and this
|
|
9
|
+
* command remains the console's route for a reply that landed in the operator
|
|
10
|
+
* DM — the in-session acknowledgement wait this replaced could never be
|
|
11
|
+
* satisfied, because the console session runs no tick extension.
|
|
11
12
|
*
|
|
12
13
|
* Both halves print copy-pasteable commands, because the reader is usually an
|
|
13
14
|
* agent in a console pane rather than a human at a prompt.
|
|
@@ -95,7 +96,8 @@ function challengeReceipt(sent: ArmChallengeSent): string {
|
|
|
95
96
|
return (
|
|
96
97
|
`CHALLENGE SENT — a code went to owner ${sent.owner}, valid for ${sent.validFor} ` +
|
|
97
98
|
`(challenge ${sent.challengeId}).\n` +
|
|
98
|
-
`NOTHING IS ARMED YET.
|
|
99
|
+
`NOTHING IS ARMED YET. A reply in the challenge's chat settles it automatically; ` +
|
|
100
|
+
`on a console host run:\n` +
|
|
99
101
|
` ${sent.followUp}\n` +
|
|
100
102
|
`That reply will arm ${String(sent.targets.length)} project(s):\n` +
|
|
101
103
|
sent.targets
|
|
@@ -53,6 +53,52 @@ export function drawFrame(lines: readonly string[], previous: number): string {
|
|
|
53
53
|
return lines.length < previous ? `${HOME}${body}\n${ERASE_BELOW}` : `${HOME}${body}\n`;
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
+
/**
|
|
57
|
+
* How the pane sleeps out the interval between renders. Production is
|
|
58
|
+
* {@link timerSleep}; tests inject a fake (#1085) so no real timer runs and
|
|
59
|
+
* the wait's scheduling is asserted directly.
|
|
60
|
+
*/
|
|
61
|
+
export type CompanionSleep = (ms: number, signal: AbortSignal) => Promise<void>;
|
|
62
|
+
|
|
63
|
+
/** The production sleep: one ref'd timeout for the whole interval, ended
|
|
64
|
+
* early by an abort. Ref'd deliberately (#1085): the pending deadline is the
|
|
65
|
+
* event loop's reason to sleep out the minute. The old wait unref'd this
|
|
66
|
+
* timer and polled `stopped` every 250 ms instead — with nothing ref'd left
|
|
67
|
+
* to wait on, the loop never slept, and the pane burned ~99% of a core
|
|
68
|
+
* between renders (~10k epoll wakes a second, measured). The unrefs' real
|
|
69
|
+
* job — never outlive the pane — belongs to cleanup, not to the handle flag:
|
|
70
|
+
* every exit path here clears the timer and detaches its listener, SIGINT
|
|
71
|
+
* and SIGTERM abort within microseconds of delivery, and a pane closed by
|
|
72
|
+
* its terminal still kills the process by signal regardless of all this.
|
|
73
|
+
* Same shape as `sleepUntilAbort` in commands/tail.ts and board.ts's
|
|
74
|
+
* `waitForInput`, which never had the spin because its deadline stayed
|
|
75
|
+
* ref'd. */
|
|
76
|
+
export const timerSleep: CompanionSleep = (ms, signal) => {
|
|
77
|
+
// An already-aborted signal can never fire its listener.
|
|
78
|
+
if (signal.aborted) return Promise.resolve();
|
|
79
|
+
return new Promise<void>((resolve) => {
|
|
80
|
+
const done = (): void => {
|
|
81
|
+
clearTimeout(timer);
|
|
82
|
+
signal.removeEventListener("abort", done);
|
|
83
|
+
resolve();
|
|
84
|
+
};
|
|
85
|
+
const timer = setTimeout(done, ms);
|
|
86
|
+
signal.addEventListener("abort", done, { once: true });
|
|
87
|
+
});
|
|
88
|
+
};
|
|
89
|
+
/** Wait out one refresh interval, ending early the moment `stopped` aborts.
|
|
90
|
+
* Exactly one sleep is scheduled per render — the poll this replaced
|
|
91
|
+
* scheduled four a second and spun the process hot between them (#1085).
|
|
92
|
+
* A stop that has already landed ends the wait before anything is scheduled.
|
|
93
|
+
* Tests drive the wait through an injected {@link CompanionSleep}. */
|
|
94
|
+
export async function waitOutRefreshInterval(
|
|
95
|
+
stopped: AbortSignal,
|
|
96
|
+
sleep: CompanionSleep = timerSleep,
|
|
97
|
+
): Promise<void> {
|
|
98
|
+
if (stopped.aborted) return;
|
|
99
|
+
await sleep(COMPANION_REFRESH_SECONDS * 1_000, stopped);
|
|
100
|
+
}
|
|
101
|
+
|
|
56
102
|
export async function companionCommand(ctx: CommandContext): Promise<void> {
|
|
57
103
|
const sub = ctx.argv[1];
|
|
58
104
|
if (sub !== "decisions") {
|
|
@@ -65,14 +111,14 @@ export async function companionCommand(ctx: CommandContext): Promise<void> {
|
|
|
65
111
|
const once = ctx.argv.includes("--once");
|
|
66
112
|
|
|
67
113
|
let previous = 0;
|
|
68
|
-
|
|
114
|
+
const stopped = new AbortController();
|
|
69
115
|
const stop = (): void => {
|
|
70
|
-
stopped
|
|
116
|
+
stopped.abort();
|
|
71
117
|
};
|
|
72
118
|
process.on("SIGINT", stop);
|
|
73
119
|
process.on("SIGTERM", stop);
|
|
74
120
|
|
|
75
|
-
while (!stopped) {
|
|
121
|
+
while (!stopped.signal.aborted) {
|
|
76
122
|
// Opened per render, closed immediately: a companion pane lives for days,
|
|
77
123
|
// and a handle held that long across daemon restarts and db snapshots is a
|
|
78
124
|
// handle to a file that may no longer be the store.
|
|
@@ -86,18 +132,8 @@ export async function companionCommand(ctx: CommandContext): Promise<void> {
|
|
|
86
132
|
process.stdout.write(drawFrame(lines, previous));
|
|
87
133
|
previous = lines.length;
|
|
88
134
|
if (once) return;
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
// closing the pane, and a lingering timer would outlive it.
|
|
93
|
-
timer.unref?.();
|
|
94
|
-
const poll = setInterval(() => {
|
|
95
|
-
if (!stopped) return;
|
|
96
|
-
clearInterval(poll);
|
|
97
|
-
clearTimeout(timer);
|
|
98
|
-
resolve();
|
|
99
|
-
}, 250);
|
|
100
|
-
poll.unref?.();
|
|
101
|
-
});
|
|
135
|
+
// One ref'd sleep for the whole minute, aborted early on SIGINT/SIGTERM
|
|
136
|
+
// — see timerSleep above for why it must stay ref'd (#1085).
|
|
137
|
+
await waitOutRefreshInterval(stopped.signal);
|
|
102
138
|
}
|
|
103
139
|
}
|
package/src/commands/drain.ts
CHANGED
|
@@ -24,13 +24,15 @@ usage:
|
|
|
24
24
|
omp-conductor drain cancel [--project NAME]
|
|
25
25
|
|
|
26
26
|
A drain is a durable, self-expiring admission fence: new claims pause while
|
|
27
|
-
existing runs settle,
|
|
27
|
+
existing runs settle, an emptied fleet OPENS the release window rather than
|
|
28
|
+
ending the drain (#1078), and admission resumes automatically at the absolute
|
|
28
29
|
deadline — even if the orchestrator crashes. start replaces any prior drain of
|
|
29
30
|
the project; --until takes an ISO instant or a relative duration (90s, 45m,
|
|
30
31
|
2h, 1d) that must be bounded and in the future. status reports the active
|
|
31
|
-
drain's creation time, absolute expiry, reason,
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
drain's creation time, absolute expiry, reason, remaining active runs, and
|
|
33
|
+
whether the window is still draining or already drained and holding. cancel
|
|
34
|
+
removes the project's drain and is idempotent. A drain never touches the
|
|
35
|
+
pause sentinel, the arm marker, or any queue label — it is the file record
|
|
34
36
|
that expires on its own.`;
|
|
35
37
|
|
|
36
38
|
/** The flags each drain subcommand accepts, after the subcommand itself. */
|
|
@@ -143,7 +145,7 @@ export async function drainCommand(ctx: CommandContext): Promise<void> {
|
|
|
143
145
|
);
|
|
144
146
|
process.stdout.write(
|
|
145
147
|
dim(
|
|
146
|
-
"the drain is a durable record: it survives crashes and resumes admission at the deadline on its own",
|
|
148
|
+
"the drain is a durable record: it survives crashes, holds the window open once the fleet empties, and resumes admission at the deadline on its own",
|
|
147
149
|
) + "\n",
|
|
148
150
|
);
|
|
149
151
|
return;
|
|
@@ -161,6 +163,11 @@ export async function drainCommand(ctx: CommandContext): Promise<void> {
|
|
|
161
163
|
` expires at ${new Date(drain.expiresAt).toISOString()}`,
|
|
162
164
|
...(drain.reason === undefined ? [] : [` reason ${drain.reason}`]),
|
|
163
165
|
` remaining ${drain.remainingRuns} active run${drain.remainingRuns === 1 ? "" : "s"}`,
|
|
166
|
+
...(drain.remainingRuns === 0
|
|
167
|
+
? [
|
|
168
|
+
` state drained and holding until ${new Date(drain.expiresAt).toISOString()} — admission resumes at the deadline`,
|
|
169
|
+
]
|
|
170
|
+
: [" state draining — the window holds while runs settle"]),
|
|
164
171
|
];
|
|
165
172
|
process.stdout.write(`${lines.join("\n")}\n`);
|
|
166
173
|
return;
|
package/src/commands/worker.ts
CHANGED
|
@@ -33,6 +33,10 @@ if (sub === "stop" && (reason === undefined || reason === "" || reason.length >
|
|
|
33
33
|
);
|
|
34
34
|
process.exit(2);
|
|
35
35
|
}
|
|
36
|
+
// A bare presence flag, parsed like every other boolean flag in this CLI:
|
|
37
|
+
// the daemon refuses a stop that would strand an open PR (#1101), and this is
|
|
38
|
+
// the operator's explicit record of choosing to do so anyway.
|
|
39
|
+
const allowOpenPr = sub === "stop" && ctx.argv.includes("--allow-open-pr");
|
|
36
40
|
const project = findProject(loadConfig(), ctx.projectFlag);
|
|
37
41
|
// The pidfile is authoritative when it names a live process; a missing or
|
|
38
42
|
// stale record falls back to the systemd unit, proved through its own
|
|
@@ -51,6 +55,7 @@ const response = await fetch(
|
|
|
51
55
|
project: project.name,
|
|
52
56
|
source: "cli",
|
|
53
57
|
...(reason === undefined ? {} : { reason }),
|
|
58
|
+
...(allowOpenPr ? { allowOpenPr: true } : {}),
|
|
54
59
|
}),
|
|
55
60
|
},
|
|
56
61
|
);
|
package/src/config-schema.ts
CHANGED
|
@@ -137,6 +137,14 @@ const capsSchema = z
|
|
|
137
137
|
workerMaxTurns: z.number().min(0).describe(`Turn ceiling for one worker (default ${DEFAULT_CAPS.workerMaxTurns})`),
|
|
138
138
|
workerMaxTurnsCeiling: z.number().min(0).describe(`Maximum turn budget assignable to one issue's next attempt`),
|
|
139
139
|
workerWallClockMs: z.number().min(0).describe(`Wall-clock ceiling for one worker (default ${DEFAULT_CAPS.workerWallClockMs})`),
|
|
140
|
+
workerStallSilenceMs: z
|
|
141
|
+
.number()
|
|
142
|
+
.min(0)
|
|
143
|
+
.nullable()
|
|
144
|
+
.describe(
|
|
145
|
+
"How long a live worker's transcript may stay unwritten before the daemon settles it as a " +
|
|
146
|
+
"progress stall; null derives a third of workerWallClockMs",
|
|
147
|
+
),
|
|
140
148
|
maxAttemptsPerIssue: z.number().min(0).describe(`Failed attempts allowed before escalation (default ${DEFAULT_CAPS.maxAttemptsPerIssue})`),
|
|
141
149
|
maxContinuationsPerIssue: z.number().min(0).describe(`Operational continuations allowed before crash/resume escalation (default ${DEFAULT_CAPS.maxContinuationsPerIssue})`),
|
|
142
150
|
})
|
package/src/config.ts
CHANGED
|
@@ -293,6 +293,11 @@ export function resolveCaps(p: ProjectConfig, defaults: Caps): Caps {
|
|
|
293
293
|
o.workerMaxTurnsCeiling ??
|
|
294
294
|
(o.workerMaxTurns === undefined ? defaults.workerMaxTurnsCeiling : workerMaxTurns * 2),
|
|
295
295
|
workerWallClockMs: o.workerWallClockMs ?? defaults.workerWallClockMs,
|
|
296
|
+
// Same `!== undefined` reading as `maxRunSpendUsd`: an explicit `null`
|
|
297
|
+
// means "derive from workerWallClockMs", not "inherit a global override"
|
|
298
|
+
// (#1086).
|
|
299
|
+
workerStallSilenceMs:
|
|
300
|
+
o.workerStallSilenceMs !== undefined ? o.workerStallSilenceMs : defaults.workerStallSilenceMs,
|
|
296
301
|
maxAttemptsPerIssue: o.maxAttemptsPerIssue ?? defaults.maxAttemptsPerIssue,
|
|
297
302
|
maxContinuationsPerIssue:
|
|
298
303
|
o.maxContinuationsPerIssue ?? defaults.maxContinuationsPerIssue,
|
|
@@ -1109,7 +1114,9 @@ function capProblem(key: string, found: string): string {
|
|
|
1109
1114
|
}
|
|
1110
1115
|
return key === "dailySpendUsd"
|
|
1111
1116
|
? `caps.dailySpendUsd must be a non-negative finite number or null (no cap), found ${found}`
|
|
1112
|
-
:
|
|
1117
|
+
: key === "workerStallSilenceMs"
|
|
1118
|
+
? `caps.workerStallSilenceMs must be a non-negative finite number or null (derive it from workerWallClockMs), found ${found}`
|
|
1119
|
+
: `caps.${key} must be a non-negative finite number, found ${found}`;
|
|
1113
1120
|
}
|
|
1114
1121
|
|
|
1115
1122
|
/** Quted fallback key lists for policy unknown-key errors. */
|
|
@@ -1729,10 +1736,18 @@ function reconcileCaps(
|
|
|
1729
1736
|
out.dailySpendUsd = null;
|
|
1730
1737
|
continue;
|
|
1731
1738
|
}
|
|
1739
|
+
if (key === "workerStallSilenceMs" && v === null) {
|
|
1740
|
+
// Null is the derive-it value (#1086): a third of the wall-clock ceiling,
|
|
1741
|
+
// computed where the window is read, so it passes through untouched.
|
|
1742
|
+
out.workerStallSilenceMs = null;
|
|
1743
|
+
continue;
|
|
1744
|
+
}
|
|
1732
1745
|
problems.push(
|
|
1733
1746
|
key === "dailySpendUsd"
|
|
1734
1747
|
? `${label}.${key} must be a non-negative finite number or null (no cap), found ${JSON.stringify(v)}`
|
|
1735
|
-
:
|
|
1748
|
+
: key === "workerStallSilenceMs"
|
|
1749
|
+
? `${label}.${key} must be a non-negative finite number or null (derive it from workerWallClockMs), found ${JSON.stringify(v)}`
|
|
1750
|
+
: `${label}.${key} must be a non-negative finite number, found ${JSON.stringify(v)}`,
|
|
1736
1751
|
);
|
|
1737
1752
|
}
|
|
1738
1753
|
|
package/src/daemon/drain.ts
CHANGED
|
@@ -35,6 +35,10 @@ export interface DrainRecord {
|
|
|
35
35
|
createdAt: string;
|
|
36
36
|
/** Absolute ISO deadline: admission resumes automatically at or after it. */
|
|
37
37
|
expiresAt: string;
|
|
38
|
+
/** ISO instant a dispatch pass first observed an empty active set under
|
|
39
|
+
* this drain — the moment the release window opened (#1078). Written by
|
|
40
|
+
* the tick's drain pass, never by the operator; absent until then. */
|
|
41
|
+
drainedAt?: string;
|
|
38
42
|
/** Purpose recorded at creation (release window, maintenance…). */
|
|
39
43
|
reason?: string;
|
|
40
44
|
}
|
|
@@ -54,6 +58,7 @@ export type DrainProblem =
|
|
|
54
58
|
| "invalid-project"
|
|
55
59
|
| "invalid-created-at"
|
|
56
60
|
| "invalid-expires-at"
|
|
61
|
+
| "invalid-drained-at"
|
|
57
62
|
| "expiry-not-future"
|
|
58
63
|
| "invalid-reason";
|
|
59
64
|
|
|
@@ -143,6 +148,13 @@ export function readDrain(project: string, now = Date.now()): DrainVerdict {
|
|
|
143
148
|
if (reason !== undefined && typeof reason !== "string") {
|
|
144
149
|
return { kind: "error", problem: "invalid-reason" };
|
|
145
150
|
}
|
|
151
|
+
const drainedAt = rec["drainedAt"];
|
|
152
|
+
if (
|
|
153
|
+
drainedAt !== undefined &&
|
|
154
|
+
(typeof drainedAt !== "string" || Number.isNaN(Date.parse(drainedAt)))
|
|
155
|
+
) {
|
|
156
|
+
return { kind: "error", problem: "invalid-drained-at" };
|
|
157
|
+
}
|
|
146
158
|
if (rec["project"] !== project) {
|
|
147
159
|
// A record persisted at this project's path but naming another project is
|
|
148
160
|
// either a copy or a rename mishap; it fences nobody (that project's drain
|
|
@@ -157,6 +169,7 @@ export function readDrain(project: string, now = Date.now()): DrainVerdict {
|
|
|
157
169
|
createdAt: rec["createdAt"],
|
|
158
170
|
expiresAt: rec["expiresAt"],
|
|
159
171
|
...(reason === undefined ? {} : { reason }),
|
|
172
|
+
...(drainedAt === undefined ? {} : { drainedAt }),
|
|
160
173
|
};
|
|
161
174
|
return { kind: "active", drain };
|
|
162
175
|
}
|
|
@@ -179,6 +192,26 @@ export function consumeDrain(project: string, now = Date.now()): DrainVerdict {
|
|
|
179
192
|
return verdict;
|
|
180
193
|
}
|
|
181
194
|
|
|
195
|
+
/**
|
|
196
|
+
* Marks an active drain with the instant its active set first reached zero —
|
|
197
|
+
* the moment the release window opened (#1078). A no-op for anything that is
|
|
198
|
+
* not a fresh, unmarked fence: absent, expired, malformed and already-marked
|
|
199
|
+
* records are left exactly as they were, so only the opening transition ever
|
|
200
|
+
* writes. The rewrite is atomic (tmp + rename, exactly like
|
|
201
|
+
* {@link createDrain}), so a crash mid-write can never leave a half-record.
|
|
202
|
+
* The annotation is what makes the window's opening durable and keeps the
|
|
203
|
+
* tick's log line a once-per-drain event instead of once-per-pass noise.
|
|
204
|
+
*/
|
|
205
|
+
export function markDrained(project: string, now = Date.now()): void {
|
|
206
|
+
const verdict = readDrain(project, now);
|
|
207
|
+
if (verdict.kind !== "active" || verdict.drain.drainedAt !== undefined) return;
|
|
208
|
+
const marked: DrainRecord = { ...verdict.drain, drainedAt: new Date(now).toISOString() };
|
|
209
|
+
const path = drainPath(project);
|
|
210
|
+
const tmp = `${path}.${process.pid.toString(36)}.${Date.now().toString(36)}.tmp`;
|
|
211
|
+
writeFileSync(tmp, `${JSON.stringify(marked, null, 2)}\n`);
|
|
212
|
+
renameSync(tmp, path);
|
|
213
|
+
}
|
|
214
|
+
|
|
182
215
|
/** Removes this project's drain, idempotently — a second cancel is a no-op. */
|
|
183
216
|
export function cancelDrain(project: string): void {
|
|
184
217
|
rmSync(drainPath(project), { force: true });
|
package/src/daemon/groom-pass.ts
CHANGED
|
@@ -245,14 +245,23 @@ export async function handleToSpecGrooming(
|
|
|
245
245
|
at: Date.now(),
|
|
246
246
|
});
|
|
247
247
|
|
|
248
|
-
// The scout's answer, unjudged, through the one validator.
|
|
249
|
-
//
|
|
250
|
-
//
|
|
251
|
-
//
|
|
248
|
+
// The scout's answer, unjudged, through the one validator (#1064). Only the
|
|
249
|
+
// structured payload is graded: a session that produced no payload at all is
|
|
250
|
+
// its own refusal class (`no-answer`), with the narration recorded as
|
|
251
|
+
// context, and a payload recovered from the text carries its `via` marker so
|
|
252
|
+
// the record says it did not come through the yield contract. `raw: ""`
|
|
253
|
+
// therefore means "no answer", and the pass never grades `result.report` as
|
|
254
|
+
// if it were one.
|
|
252
255
|
const outcome = recordToSpecGrooming(d.store, {
|
|
253
256
|
project: project.name,
|
|
254
257
|
issue,
|
|
255
|
-
input: result.raw
|
|
258
|
+
input: result.raw,
|
|
259
|
+
via: result.via === "text" ? "text" : result.raw === "" ? undefined : "yield",
|
|
260
|
+
report: result.report,
|
|
261
|
+
repaired: result.repaired,
|
|
262
|
+
killedAtCeiling: result.killedAtCeiling,
|
|
263
|
+
maxTurns: TO_SPEC_MAX_TURNS,
|
|
264
|
+
turns: result.turns,
|
|
256
265
|
launchedAt: batch.launchedAt,
|
|
257
266
|
});
|
|
258
267
|
const ran = result.model === undefined ? "" : ` by ${result.model}`;
|
|
@@ -261,9 +270,10 @@ export async function handleToSpecGrooming(
|
|
|
261
270
|
// stores and what the stats lane renders.
|
|
262
271
|
const cost =
|
|
263
272
|
result.turns > 0 && result.spendUsd === 0 ? "unmetered" : `$${result.spendUsd.toFixed(2)}`;
|
|
273
|
+
const repairNote = result.repaired === true ? " (repaired)" : "";
|
|
264
274
|
log(
|
|
265
275
|
`#${issue} groomed ${outcome.record.verdict}(${outcome.record.reason})${ran} in batch ${batch.id} — ` +
|
|
266
|
-
`${result.turns} turn(s), ${cost}`,
|
|
276
|
+
`${result.turns} turn(s), ${cost}${repairNote}`,
|
|
267
277
|
);
|
|
268
278
|
if (outcome.kind !== "persisted" || outcome.record.verdict !== "promotable") return;
|
|
269
279
|
await promoteGroomedVerdict(d, outcome.record);
|
package/src/daemon/http.ts
CHANGED
|
@@ -176,6 +176,34 @@ export async function workerControlResponse(
|
|
|
176
176
|
|
|
177
177
|
|
|
178
178
|
const issue = Number(match[1]);
|
|
179
|
+
// The #1101 front door: stopping a live run that has already pushed a PR
|
|
180
|
+
// strands that PR — the review verb could not return it and dispatch will
|
|
181
|
+
// not re-claim an issue with an open PR. Refuse and name the PR so the
|
|
182
|
+
// operator chooses knowingly; the explicit flag records that choice. A row
|
|
183
|
+
// that is not live settles as already-terminal below, where no strand is
|
|
184
|
+
// possible, and a run that never pushed has nothing to strand.
|
|
185
|
+
const rawAllowOpenPr = Reflect.get(body, "allowOpenPr");
|
|
186
|
+
if (rawAllowOpenPr !== undefined && typeof rawAllowOpenPr !== "boolean") {
|
|
187
|
+
return Response.json({ error: "allowOpenPr must be a boolean when present" }, { status: 400 });
|
|
188
|
+
}
|
|
189
|
+
const controlled = action === "stop" ? store.latestRun(project, issue) : undefined;
|
|
190
|
+
if (
|
|
191
|
+
action === "stop" &&
|
|
192
|
+
rawAllowOpenPr !== true &&
|
|
193
|
+
controlled !== undefined &&
|
|
194
|
+
controlled.prUrl !== undefined &&
|
|
195
|
+
LIVE_STATES.includes(controlled.state)
|
|
196
|
+
) {
|
|
197
|
+
return Response.json(
|
|
198
|
+
{
|
|
199
|
+
error:
|
|
200
|
+
`#${issue} run ${controlled.id} already pushed ${controlled.prUrl} — stopping now would strand it: ` +
|
|
201
|
+
"a stopped run cannot be returned for review, and dispatch will not re-claim an issue with an open PR. " +
|
|
202
|
+
"Pass allowOpenPr (worker stop --allow-open-pr) to stop it anyway.",
|
|
203
|
+
},
|
|
204
|
+
{ status: 409 },
|
|
205
|
+
);
|
|
206
|
+
}
|
|
179
207
|
const outcome =
|
|
180
208
|
action === "pause"
|
|
181
209
|
? await registry.pause(project, issue, source)
|