owenloop 0.5.24 → 0.5.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/dist/packages/work/src/exec/loop.js +110 -6
- package/dist/packages/work/src/exec/payload.d.ts +107 -5
- package/dist/packages/work/src/exec/payload.js +222 -17
- package/dist/packages/work/src/exec/receipt.d.ts +12 -3
- package/dist/packages/work/src/exec/receipt.js +9 -9
- package/dist/packages/work/src/shift/loop.d.ts +76 -12
- package/dist/packages/work/src/shift/loop.js +157 -24
- package/dist/src/bundle/runtime.d.ts +1 -1
- package/dist/src/bundle/runtime.js +13 -0
- package/dist/src/defs.js +106 -11
- package/dist/src/engine.d.ts +2 -8
- package/dist/src/engine.js +40 -28
- package/dist/src/model.d.ts +12 -2
- package/dist/src/model.js +29 -22
- package/dist/src/order-resolver.js +1 -0
- package/dist/src/types.d.ts +8 -0
- package/docs/authoring.md +31 -1
- package/docs/bundles.md +66 -0
- package/docs/cli.md +6 -2
- package/docs/design.md +36 -5
- package/docs/routing.md +3 -2
- package/docs/wire-contracts.md +22 -1
- package/package.json +1 -1
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/plugin/.mcp.json +1 -1
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
- package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
2
|
+
* Fold a raw `CommandResult` and its order context into the submitted receipt.
|
|
3
|
+
*
|
|
4
|
+
* `parsedPayload` is required rather than defaulted. A default could only parse
|
|
5
|
+
* the stdout marker, which stopped being the whole story when the payload file
|
|
6
|
+
* was added: it would silently drop a file payload, or publish the marker half
|
|
7
|
+
* of a conflict, and in both cases produce a receipt with no `payloadError` to
|
|
8
|
+
* show anything went wrong. Resolving both transports needs the file read, which
|
|
9
|
+
* only the caller holds, so the caller states the answer.
|
|
8
10
|
*/
|
|
9
|
-
|
|
10
|
-
/** Fold a raw `CommandResult` and its order context into the submitted receipt. */
|
|
11
|
-
export function buildReceipt(result, ctx, parsedPayload = parsePayloadLine(result.payloadLine, result.payloadOverCap)) {
|
|
11
|
+
export function buildReceipt(result, ctx, parsedPayload) {
|
|
12
12
|
return {
|
|
13
13
|
kind: 'command-receipt',
|
|
14
14
|
command: ctx.command,
|
|
@@ -4,7 +4,7 @@ import { type WorkOrder } from '../hub/types.ts';
|
|
|
4
4
|
import type { FetchedStep } from '../bundle/types.ts';
|
|
5
5
|
import { type ReconcileOptions, type Liveness } from './state.ts';
|
|
6
6
|
import { sweepWorkDirs as sweepWorkDirsImpl } from '../agent/workdir.ts';
|
|
7
|
-
import type { Spawner } from './spawn.ts';
|
|
7
|
+
import type { Spawner, WorkerFailure } from './spawn.ts';
|
|
8
8
|
import { type ShiftEvent } from './protocol.ts';
|
|
9
9
|
export interface ShiftLoopOptions {
|
|
10
10
|
hub: HubClient;
|
|
@@ -172,11 +172,18 @@ export interface ShiftLoop {
|
|
|
172
172
|
/**
|
|
173
173
|
* Run-ended reap (metering condition (a)): drop the in-flight record for
|
|
174
174
|
* `run` NOW, so its slot frees immediately instead of waiting for the next
|
|
175
|
-
* reconcile to notice the child exited.
|
|
176
|
-
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
*
|
|
175
|
+
* reconcile to notice the child exited.
|
|
176
|
+
*
|
|
177
|
+
* NOTHING IN `src/` CALLS THIS. It is reachable only from tests. The intended
|
|
178
|
+
* caller was a submit path that told the shift the hub had closed a run, and
|
|
179
|
+
* that wiring was never made, so the slot is always freed the slower way
|
|
180
|
+
* instead: the child exits, and the next reconcile's pid probe frees it.
|
|
181
|
+
*
|
|
182
|
+
* Stated plainly because the gap is invisible from the type — this is on the
|
|
183
|
+
* `ShiftLoop` interface, so it reads as live. Do not reason from its
|
|
184
|
+
* existence about when `runBrakeKey` entries are cleared; in production the
|
|
185
|
+
* per-sweep prune is the only clearer. Wire it or delete it, but do not build
|
|
186
|
+
* a guard on top of it.
|
|
180
187
|
*/
|
|
181
188
|
noteRunEnded(run: string): void;
|
|
182
189
|
/**
|
|
@@ -203,10 +210,44 @@ export interface ShiftLoop {
|
|
|
203
210
|
* A failure for a run this loop never dispatched (a foreign or already-reaped
|
|
204
211
|
* run) is ignored rather than guessed at: the fan-out key is not on the
|
|
205
212
|
* failure record, so there is no sound way to pick a brake key for it.
|
|
213
|
+
*
|
|
214
|
+
* ALSO HANDS THE CLAIM BACK, with the failure as the reason. A worker that
|
|
215
|
+
* dies before it submits leaves its order INFLIGHT on the hub with a frozen
|
|
216
|
+
* heartbeat: no alert, no re-offer, and no other shift able to take it,
|
|
217
|
+
* because this shift still holds the claim. Measured once at 34 minutes,
|
|
218
|
+
* indistinguishable from work in progress, until an operator released it by
|
|
219
|
+
* hand. The failure is known here and already named precisely, so the name
|
|
220
|
+
* travels with the release instead of dying in a local log line.
|
|
221
|
+
*
|
|
222
|
+
* The brake-key guard scopes the release to runs THIS shift dispatched and
|
|
223
|
+
* has not yet pruned. It does NOT mean "the run is still open": the only
|
|
224
|
+
* production clearer is the per-sweep prune below, and a child's exit event
|
|
225
|
+
* always beats the next sweep. So this fires for ordinary worker failures
|
|
226
|
+
* too, on top of the specific release the child's own final breath sends.
|
|
227
|
+
*
|
|
228
|
+
* That redundancy is deliberate and it is cheap, because the hub answers a
|
|
229
|
+
* release on an unheld claim by returning BEFORE it normalizes the reason or
|
|
230
|
+
* publishes anything (`verbs/release.ts`: "Only a currently held targeted
|
|
231
|
+
* release gets a diagnostic side effect"). A worker whose own final breath
|
|
232
|
+
* reached the hub therefore costs exactly one no-op request here — it cannot
|
|
233
|
+
* overwrite the reason that worker already recorded.
|
|
234
|
+
*
|
|
235
|
+
* Covering the ordinary exit path is the point, not an accident. The
|
|
236
|
+
* incident this exists for exited 1 with no signal, killed by ENOSPC inside
|
|
237
|
+
* its own error logging, so a predicate keyed on "could not have released
|
|
238
|
+
* for itself" — a signal death, or a spawn that never started — would have
|
|
239
|
+
* missed it entirely. A supervisor cannot tell those apart from outside, so
|
|
240
|
+
* it reports what it saw and lets the hub deduplicate.
|
|
241
|
+
*
|
|
242
|
+
* One narrow race remains, and it is worth taking. `finalBreath` gives up
|
|
243
|
+
* after `RELEASE_CAP_MS` while its request is still in flight; if that
|
|
244
|
+
* request is slow enough for this one to overtake it, the supervisor's
|
|
245
|
+
* account is recorded and the worker's more specific one is dropped as
|
|
246
|
+
* not-held. The reason string names its own provenance so that outcome still
|
|
247
|
+
* reads honestly, and a slightly vaguer alert beats the alternative this
|
|
248
|
+
* replaces, which was no alert at all for 34 minutes.
|
|
206
249
|
*/
|
|
207
|
-
noteWorkerFailure(failure:
|
|
208
|
-
run: string;
|
|
209
|
-
}): void;
|
|
250
|
+
noteWorkerFailure(failure: WorkerFailure): void;
|
|
210
251
|
}
|
|
211
252
|
/** The hub reaps a never-contacted claim after this long. Every client-side
|
|
212
253
|
* re-offer latency must stay strictly under it. */
|
|
@@ -238,9 +279,12 @@ export declare const MAX_PENDING_CANDIDATE_AGE_MS: number;
|
|
|
238
279
|
* each other.
|
|
239
280
|
*
|
|
240
281
|
* WHAT IS COUNTED, AND WHY IT IS NOT DISPATCHES. The event counted against that
|
|
241
|
-
* key is a
|
|
242
|
-
*
|
|
243
|
-
*
|
|
282
|
+
* key is a FAILED ATTEMPT TO RUN THE STEP, in either of the two shapes that
|
|
283
|
+
* exist: a child that started and exited non-zero, reported into the loop
|
|
284
|
+
* through `noteWorkerFailure`; or a dispatch that never produced a running
|
|
285
|
+
* child at all, which throws out of `dispatchCandidate` and is charged there.
|
|
286
|
+
* What is NOT counted is a dispatch that succeeds. Counting those would
|
|
287
|
+
* conflate a storm with legitimate concurrency: the hub can offer several
|
|
244
288
|
* DISTINCT runs of one step in a single sweep, and metering the second through
|
|
245
289
|
* fifth of those would throttle healthy work. A failure is direct evidence that
|
|
246
290
|
* running this step again right now is unlikely to help; a dispatch is evidence
|
|
@@ -250,4 +294,24 @@ export declare const MAX_PENDING_CANDIDATE_AGE_MS: number;
|
|
|
250
294
|
* with zero delay, however many orders arrive for it at once.
|
|
251
295
|
*/
|
|
252
296
|
export declare const STEP_BRAKE_DELAYS_MS: readonly [2000, 8000, 30000, 60000, 90000];
|
|
297
|
+
/**
|
|
298
|
+
* Treat a failure this long after the previous one as the start of a NEW
|
|
299
|
+
* streak, resetting the count to zero first. Comfortably longer than the
|
|
300
|
+
* longest delay in the table (90s), so a step failing at maximum backoff never
|
|
301
|
+
* decays out of it, while an isolated transient long after an old streak starts
|
|
302
|
+
* again at the shortest delay instead of inheriting a stale penalty.
|
|
303
|
+
*/
|
|
304
|
+
export declare const STEP_BRAKE_DECAY_MS = 600000;
|
|
305
|
+
/**
|
|
306
|
+
* Longest release reason this shift will put on the wire, in code points.
|
|
307
|
+
*
|
|
308
|
+
* Mirrors the hub's own `MAX_RELEASE_REASON_CODE_POINTS`, which trims, treats
|
|
309
|
+
* blank as absent, and truncates without splitting a surrogate pair. Clamping
|
|
310
|
+
* on this side too means the reason we send is the reason an operator reads,
|
|
311
|
+
* with no silent server-side shortening in between.
|
|
312
|
+
*
|
|
313
|
+
* Exported so a test can assert the clamp against the constant rather than
|
|
314
|
+
* against a copy of the number, which would drift the day the hub's limit moves.
|
|
315
|
+
*/
|
|
316
|
+
export declare const MAX_RELEASE_REASON_POINTS = 1024;
|
|
253
317
|
export declare function createShiftLoop(opts: ShiftLoopOptions): ShiftLoop;
|
|
@@ -144,9 +144,12 @@ const CAPACITY_RELEASE_COOLDOWN_MS = 30_000;
|
|
|
144
144
|
* each other.
|
|
145
145
|
*
|
|
146
146
|
* WHAT IS COUNTED, AND WHY IT IS NOT DISPATCHES. The event counted against that
|
|
147
|
-
* key is a
|
|
148
|
-
*
|
|
149
|
-
*
|
|
147
|
+
* key is a FAILED ATTEMPT TO RUN THE STEP, in either of the two shapes that
|
|
148
|
+
* exist: a child that started and exited non-zero, reported into the loop
|
|
149
|
+
* through `noteWorkerFailure`; or a dispatch that never produced a running
|
|
150
|
+
* child at all, which throws out of `dispatchCandidate` and is charged there.
|
|
151
|
+
* What is NOT counted is a dispatch that succeeds. Counting those would
|
|
152
|
+
* conflate a storm with legitimate concurrency: the hub can offer several
|
|
150
153
|
* DISTINCT runs of one step in a single sweep, and metering the second through
|
|
151
154
|
* fifth of those would throttle healthy work. A failure is direct evidence that
|
|
152
155
|
* running this step again right now is unlikely to help; a dispatch is evidence
|
|
@@ -163,13 +166,25 @@ export const STEP_BRAKE_DELAYS_MS = [2_000, 8_000, 30_000, 60_000, 90_000];
|
|
|
163
166
|
* decays out of it, while an isolated transient long after an old streak starts
|
|
164
167
|
* again at the shortest delay instead of inheriting a stale penalty.
|
|
165
168
|
*/
|
|
166
|
-
const STEP_BRAKE_DECAY_MS = 600_000;
|
|
169
|
+
export const STEP_BRAKE_DECAY_MS = 600_000;
|
|
167
170
|
/**
|
|
168
171
|
* Forget a step's brake entry once this long has passed since its window
|
|
169
172
|
* closed, so the map cannot grow for the life of the shift. Pure housekeeping —
|
|
170
173
|
* `STEP_BRAKE_DECAY_MS` is what governs behaviour.
|
|
171
174
|
*/
|
|
172
175
|
const STEP_BRAKE_FORGET_MS = 1_800_000;
|
|
176
|
+
/**
|
|
177
|
+
* Longest release reason this shift will put on the wire, in code points.
|
|
178
|
+
*
|
|
179
|
+
* Mirrors the hub's own `MAX_RELEASE_REASON_CODE_POINTS`, which trims, treats
|
|
180
|
+
* blank as absent, and truncates without splitting a surrogate pair. Clamping
|
|
181
|
+
* on this side too means the reason we send is the reason an operator reads,
|
|
182
|
+
* with no silent server-side shortening in between.
|
|
183
|
+
*
|
|
184
|
+
* Exported so a test can assert the clamp against the constant rather than
|
|
185
|
+
* against a copy of the number, which would drift the day the hub's limit moves.
|
|
186
|
+
*/
|
|
187
|
+
export const MAX_RELEASE_REASON_POINTS = 1024;
|
|
173
188
|
/**
|
|
174
189
|
* How long to refuse the next dispatch of a step after `count` consecutive
|
|
175
190
|
* worker failures. `count` is at least 1 at every call site (this is reached
|
|
@@ -386,6 +401,60 @@ export function createShiftLoop(opts) {
|
|
|
386
401
|
opts.err(`shift event sink failed: ${errMsg(e)} (continuing)`);
|
|
387
402
|
}
|
|
388
403
|
}
|
|
404
|
+
/**
|
|
405
|
+
* Bound a release reason to what the hub will actually store.
|
|
406
|
+
*
|
|
407
|
+
* Applied to EVERY reason this supervisor sends, not only the ones that look
|
|
408
|
+
* long. The two senders differ sharply in how bounded they are, which is
|
|
409
|
+
* exactly why the clamp belongs in one shared place rather than at whichever
|
|
410
|
+
* call site looked risky on the day:
|
|
411
|
+
*
|
|
412
|
+
* - `describeWorkerFailure` is provably short. Its longest constructible
|
|
413
|
+
* output is under 100 code points, because every part is generated here or
|
|
414
|
+
* is one of two literals in `spawn.ts`.
|
|
415
|
+
* - The dispatch catch interpolates `errMsg(e)`, which is NOT bounded. A
|
|
416
|
+
* `FileLockTimeoutError` from `acquireDispatchLock` embeds a filesystem
|
|
417
|
+
* path, and a path is bounded only by PATH_MAX.
|
|
418
|
+
*
|
|
419
|
+
* Slices by code point, never by UTF-16 unit, so a clamp cannot halve a
|
|
420
|
+
* surrogate pair and put a lone surrogate on the wire. The hub truncates at
|
|
421
|
+
* `MAX_RELEASE_REASON_CODE_POINTS` too; clamping here as well means the
|
|
422
|
+
* reason we send is the reason an operator reads, with no silent server-side
|
|
423
|
+
* shortening in between.
|
|
424
|
+
*/
|
|
425
|
+
const clampReason = (reason) => {
|
|
426
|
+
const points = Array.from(reason);
|
|
427
|
+
return points.length > MAX_RELEASE_REASON_POINTS
|
|
428
|
+
? points.slice(0, MAX_RELEASE_REASON_POINTS).join('')
|
|
429
|
+
: reason;
|
|
430
|
+
};
|
|
431
|
+
/**
|
|
432
|
+
* The operator-facing account of a dead worker, for the release reason.
|
|
433
|
+
*
|
|
434
|
+
* It names the supervisor explicitly. A worker that fails through its own
|
|
435
|
+
* error path already sends a precise reason of its own, and in a narrow race
|
|
436
|
+
* this account can land first and take the slot the hub keeps for the first
|
|
437
|
+
* observation. Saying who is speaking keeps that outcome readable instead of
|
|
438
|
+
* looking like a worker's own vague self-report.
|
|
439
|
+
*
|
|
440
|
+
* `message` is the SPAWNER'S OWN bounded lifecycle string, never worker
|
|
441
|
+
* output: agent-run stderr is untrusted and is deliberately never quoted back
|
|
442
|
+
* (see `spawn.ts`), and the spawn error object is dropped there rather than
|
|
443
|
+
* interpolated, so no local path reaches the wire.
|
|
444
|
+
*
|
|
445
|
+
* The exit status or signal is what tells an operator which failure this was:
|
|
446
|
+
* a crash, an external kill, or a child that never launched at all. It is the
|
|
447
|
+
* difference between "this step is broken" and "this host is broken", so it
|
|
448
|
+
* travels with the message rather than staying in a local log line.
|
|
449
|
+
*/
|
|
450
|
+
const describeWorkerFailure = (failure) => {
|
|
451
|
+
const cause = failure.signal !== null
|
|
452
|
+
? `signal ${failure.signal}`
|
|
453
|
+
: failure.exitStatus !== null
|
|
454
|
+
? `exitStatus ${failure.exitStatus}`
|
|
455
|
+
: 'no exit status';
|
|
456
|
+
return clampReason(`shift supervisor: ${failure.kind} ${failure.message} (${cause})`);
|
|
457
|
+
};
|
|
389
458
|
/**
|
|
390
459
|
* Hand a claim back so another shift can be offered it immediately. This
|
|
391
460
|
* deliberately stays off the dispatch critical path; a failed release falls
|
|
@@ -445,6 +514,29 @@ export function createShiftLoop(opts) {
|
|
|
445
514
|
* monotonic instant the next spawn of that step becomes permitted.
|
|
446
515
|
*/
|
|
447
516
|
const stepBrake = new Map();
|
|
517
|
+
/**
|
|
518
|
+
* Charge one failure against a step's brake, extending the current streak or
|
|
519
|
+
* starting a new one.
|
|
520
|
+
*
|
|
521
|
+
* Shared by both places a dispatch can die: the worker failing after it
|
|
522
|
+
* started (`noteWorkerFailure`) and the spawn itself failing before it ever
|
|
523
|
+
* did (`dispatchCandidate`'s catch). Both now hand the claim back, and a
|
|
524
|
+
* released claim is re-offered immediately instead of lapsing after the
|
|
525
|
+
* 120s reap — so the brake, not the reap, is what throttles a step that
|
|
526
|
+
* keeps dying. Charging from only one of the two paths would leave the other
|
|
527
|
+
* free-running against the hub.
|
|
528
|
+
*/
|
|
529
|
+
const chargeStepBrake = (brakeKey) => {
|
|
530
|
+
const at = monotonicNow();
|
|
531
|
+
const prior = stepBrake.get(brakeKey);
|
|
532
|
+
// A failure long after the previous one starts a NEW streak rather than
|
|
533
|
+
// extending a stale one — see `STEP_BRAKE_DECAY_MS`.
|
|
534
|
+
const priorCount = prior !== undefined && at - prior.failedAt < STEP_BRAKE_DECAY_MS
|
|
535
|
+
? prior.count
|
|
536
|
+
: 0;
|
|
537
|
+
const count = priorCount + 1;
|
|
538
|
+
stepBrake.set(brakeKey, { count, failedAt: at, nextAllowedAt: at + stepBrakeDelayMs(count) });
|
|
539
|
+
};
|
|
448
540
|
/**
|
|
449
541
|
* Brake key of the step each dispatched run belongs to.
|
|
450
542
|
*
|
|
@@ -456,9 +548,11 @@ export function createShiftLoop(opts) {
|
|
|
456
548
|
* precedes the child's `exit` event (that fires on a later tick), so a
|
|
457
549
|
* failure can never arrive before its own entry exists.
|
|
458
550
|
*
|
|
459
|
-
* Entries are removed when the run's failure is noted,
|
|
460
|
-
*
|
|
461
|
-
* map
|
|
551
|
+
* Entries are removed when the run's failure is noted and, in production,
|
|
552
|
+
* for any run no longer live at sweep time — that prune is what bounds the
|
|
553
|
+
* map. (`noteRunEnded` also clears one, but nothing in `src/` calls it; see
|
|
554
|
+
* its declaration.) So an entry's presence means "this shift dispatched this
|
|
555
|
+
* run and has not pruned it yet", NOT "the run is still open".
|
|
462
556
|
*/
|
|
463
557
|
const runBrakeKey = new Map();
|
|
464
558
|
/**
|
|
@@ -596,7 +690,40 @@ export function createShiftLoop(opts) {
|
|
|
596
690
|
message,
|
|
597
691
|
});
|
|
598
692
|
const prefix = c.kind === 'command' ? 'spawn' : 'agent-run spawn';
|
|
693
|
+
// Two different failures reach this one catch, and an operator reading the
|
|
694
|
+
// release reason needs to know which. `cancel` is assigned only after
|
|
695
|
+
// `opts.spawner` returns, so it is the exact witness: undefined means the
|
|
696
|
+
// spawn itself threw and no child ever existed; defined means a child WAS
|
|
697
|
+
// running and something after it threw — the reservation write, the
|
|
698
|
+
// dispatch lock, or `opts.out` on a full disk — and `cancel()` above has
|
|
699
|
+
// just killed it. Calling both "failed" told an operator to look for a
|
|
700
|
+
// fork problem that never happened.
|
|
701
|
+
const startedThenAborted = cancel !== undefined;
|
|
599
702
|
opts.err(`${prefix} for ${c.workflow}/${c.order.run} failed: ${message}`);
|
|
703
|
+
// Hand the claim back. `whats_next` took it before this function ran, and
|
|
704
|
+
// reaching here means no worker exists to hand it back for itself — either
|
|
705
|
+
// no child was ever started, or `cancel()` has just killed the one that
|
|
706
|
+
// was. Without this the order sits INFLIGHT with a heartbeat that will
|
|
707
|
+
// never tick again, which is the same invisible stall a dead worker
|
|
708
|
+
// leaves.
|
|
709
|
+
//
|
|
710
|
+
// No `WorkerFailure` is ever reported for this path — the spawn never
|
|
711
|
+
// returned, or the child was killed before it could report — so
|
|
712
|
+
// `noteWorkerFailure` cannot cover it.
|
|
713
|
+
//
|
|
714
|
+
// Charge the brake from here too. A released claim is re-offered at once
|
|
715
|
+
// rather than lapsing after the 120s reap, so on a host that cannot fork
|
|
716
|
+
// at all the brake is now the only thing standing between this shift and
|
|
717
|
+
// a claim-fail-release loop against the hub.
|
|
718
|
+
//
|
|
719
|
+
// Drop the run's brake-key mapping first. On the aborted-child arm
|
|
720
|
+
// `runBrakeKey` may already be set, and leaving it would let a later
|
|
721
|
+
// `noteWorkerFailure` for the same run release a second time and charge
|
|
722
|
+
// the brake twice for one logical failure. Nothing depends on the entry
|
|
723
|
+
// once the claim is released, so deleting it is unconditional.
|
|
724
|
+
releaseClaim(c.workflow, c.order.run, clampReason(`shift supervisor: ${prefix} ${startedThenAborted ? 'started then aborted' : 'failed'} — ${message}`));
|
|
725
|
+
runBrakeKey.delete(c.order.run);
|
|
726
|
+
chargeStepBrake(brakeKey);
|
|
600
727
|
return 'failed';
|
|
601
728
|
}
|
|
602
729
|
}
|
|
@@ -688,8 +815,18 @@ export function createShiftLoop(opts) {
|
|
|
688
815
|
// exited CLEANLY reports no failure, so nothing else would ever remove its
|
|
689
816
|
// entry. A non-zero exit is charged by `noteWorkerFailure`, which the
|
|
690
817
|
// child's own `exit` event drives on the next event-loop tick — always well
|
|
691
|
-
// before the next sweep, which waits a whole poll interval.
|
|
692
|
-
//
|
|
818
|
+
// before the next sweep, which waits a whole poll interval.
|
|
819
|
+
//
|
|
820
|
+
// This prune is the only clearer for a run that exits CLEANLY, and is
|
|
821
|
+
// therefore what bounds the map — a failing run is cleared by
|
|
822
|
+
// `noteWorkerFailure`, and an aborted dispatch clears its own entry. That
|
|
823
|
+
// makes the ordering matter more than it used to. `noteWorkerFailure` now also
|
|
824
|
+
// releases the claim, so an inversion here no longer costs one uncharged
|
|
825
|
+
// failure: it costs a claim nobody hands back, which is the stall this
|
|
826
|
+
// release exists to end. `iteration()` closes the window — `reconcile()`
|
|
827
|
+
// snapshots `live` and then awaits `hub.wake`, and a queued `exit` event is
|
|
828
|
+
// delivered during that await, so a child that died before the snapshot has
|
|
829
|
+
// already been charged and one that died after is still inside it.
|
|
693
830
|
for (const [key, entry] of stepBrake) {
|
|
694
831
|
if (monotonicNow() - entry.nextAllowedAt > STEP_BRAKE_FORGET_MS)
|
|
695
832
|
stepBrake.delete(key);
|
|
@@ -1012,7 +1149,14 @@ export function createShiftLoop(opts) {
|
|
|
1012
1149
|
// 'braked' and 'failed' are deliberately NOT queued into pendingCandidates:
|
|
1013
1150
|
// both mean this shift is not going to run that order now, and re-queuing a
|
|
1014
1151
|
// braked candidate would just re-dispatch it on the next drain, defeating
|
|
1015
|
-
// the brake.
|
|
1152
|
+
// the brake.
|
|
1153
|
+
//
|
|
1154
|
+
// What happens to the claim differs between the two, and the difference is
|
|
1155
|
+
// the point of this change. A BRAKED candidate was never dispatched, so its
|
|
1156
|
+
// claim lapses through the hub's pickup window — the delay is deliberate,
|
|
1157
|
+
// and shedding it is what the brake is for. A FAILED one hands its claim
|
|
1158
|
+
// back explicitly in `dispatchCandidate`, so it is re-offered at once
|
|
1159
|
+
// instead of sitting INFLIGHT behind a heartbeat that will never tick.
|
|
1016
1160
|
for (const candidate of candidates) {
|
|
1017
1161
|
if (discardExpiredCandidate(candidate, false))
|
|
1018
1162
|
continue;
|
|
@@ -1342,21 +1486,10 @@ export function createShiftLoop(opts) {
|
|
|
1342
1486
|
noteWorkerFailure: (failure) => {
|
|
1343
1487
|
const brakeKey = runBrakeKey.get(failure.run);
|
|
1344
1488
|
if (brakeKey === undefined)
|
|
1345
|
-
return; // not
|
|
1489
|
+
return; // not a run this shift still tracks
|
|
1346
1490
|
runBrakeKey.delete(failure.run);
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
// A failure long after the previous one starts a NEW streak rather than
|
|
1350
|
-
// extending a stale one — see `STEP_BRAKE_DECAY_MS`.
|
|
1351
|
-
const priorCount = prior !== undefined && at - prior.failedAt < STEP_BRAKE_DECAY_MS
|
|
1352
|
-
? prior.count
|
|
1353
|
-
: 0;
|
|
1354
|
-
const count = priorCount + 1;
|
|
1355
|
-
stepBrake.set(brakeKey, {
|
|
1356
|
-
count,
|
|
1357
|
-
failedAt: at,
|
|
1358
|
-
nextAllowedAt: at + stepBrakeDelayMs(count),
|
|
1359
|
-
});
|
|
1491
|
+
releaseClaim(failure.workflow, failure.run, describeWorkerFailure(failure));
|
|
1492
|
+
chargeStepBrake(brakeKey);
|
|
1360
1493
|
},
|
|
1361
1494
|
};
|
|
1362
1495
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { BundleRuntimeRequirements } from './types.ts';
|
|
2
2
|
/** Runtime features implemented by this Owenloop release. */
|
|
3
|
-
export declare const SUPPORTED_RUNTIME_FEATURES: readonly ["harness-policy-enforcement.v1", "native-judge-policy-inheritance.v1", "neutral-approval-modes.v1"];
|
|
3
|
+
export declare const SUPPORTED_RUNTIME_FEATURES: readonly ["harness-policy-enforcement.v1", "native-judge-policy-inheritance.v1", "neutral-approval-modes.v1", "command-payload-file.v1"];
|
|
4
4
|
export interface RuntimeCompatibilityEnvironment {
|
|
5
5
|
/** The running Owenloop package version. `0.0.0` is the unavailable-version sentinel. */
|
|
6
6
|
version: string;
|
|
@@ -15,6 +15,19 @@ export const SUPPORTED_RUNTIME_FEATURES = Object.freeze([
|
|
|
15
15
|
// than about an old runtime. Requiring the id moves the diagnosis to the
|
|
16
16
|
// bundle check, before any order is offered.
|
|
17
17
|
'neutral-approval-modes.v1',
|
|
18
|
+
// A command step may return its payload by writing JSON to the path in
|
|
19
|
+
// `$OWENLOOP_PAYLOAD_FILE` instead of printing an `##owenloop:payload##` line
|
|
20
|
+
// on stdout, which lifts the result ceiling from the 64 KiB stdout marker to
|
|
21
|
+
// the hub's artifact cap. A def whose command writes that file MUST require
|
|
22
|
+
// this id, because a CLI that predates it fails in the worst available way:
|
|
23
|
+
// it leaves the variable unset, and the natural defensive spelling
|
|
24
|
+
// `> "${OWENLOOP_PAYLOAD_FILE:-/dev/null}"` then EXITS ZERO having written the
|
|
25
|
+
// payload to /dev/null. There is no error anywhere — the step submits a
|
|
26
|
+
// receipt with no payload, and the first sign of trouble is a downstream
|
|
27
|
+
// `from: payload.value` bind failing at the hub, one step away from the cause.
|
|
28
|
+
// Requiring the id moves the diagnosis to the bundle check, before any order
|
|
29
|
+
// is offered.
|
|
30
|
+
'command-payload-file.v1',
|
|
18
31
|
]);
|
|
19
32
|
/** True only for one canonical strict SemVer value, with no prefix, range, or whitespace. */
|
|
20
33
|
export function isCanonicalSemver(value) {
|
package/dist/src/defs.js
CHANGED
|
@@ -39,7 +39,7 @@ const RAW_PRODUCE_KEYS = ['name', 'schema', 'judges', 'maxAttempts', 'maxSchemaF
|
|
|
39
39
|
* fields, so it gets its own allowlist rather than folding into
|
|
40
40
|
* RAW_PRODUCE_KEYS (§27 unknown-key rejection). */
|
|
41
41
|
const RAW_GROUP_KEYS = ['group', 'mode', 'of'];
|
|
42
|
-
const RAW_JUDGE_KEYS = ['name', 'body', 'bodyFile', 'model', 'inputs', 'cadence', 'maxRunsPerDay', 'executor', 'command', 'spec', 'capabilities'];
|
|
42
|
+
const RAW_JUDGE_KEYS = ['name', 'body', 'bodyFile', 'model', 'inputs', 'cadence', 'maxRunsPerDay', 'executor', 'command', 'spec', 'capabilities', 'modifiers'];
|
|
43
43
|
/** Keys valid on a normal (non-calls:, non-include:) step entry. */
|
|
44
44
|
const RAW_STEP_KEYS = [
|
|
45
45
|
'name', 'consumes', 'produces', 'generates', 'invalidates', 'cadence',
|
|
@@ -371,6 +371,10 @@ function parseJudges(v, ctx, baseDir) {
|
|
|
371
371
|
assertAuthoredCapability(cap, `judge '${name}'.capabilities`);
|
|
372
372
|
judge.capabilities = caps;
|
|
373
373
|
}
|
|
374
|
+
if (raw.modifiers !== undefined) {
|
|
375
|
+
// Keep the raw YAML object isolated from the compiled definition.
|
|
376
|
+
judge.modifiers = [...parseModifiers(raw.modifiers, `judge '${name}'.modifiers`)];
|
|
377
|
+
}
|
|
374
378
|
return judge;
|
|
375
379
|
});
|
|
376
380
|
}
|
|
@@ -791,7 +795,16 @@ function prefixStep(step, prefix, defInputs = []) {
|
|
|
791
795
|
// map
|
|
792
796
|
raw = `${stem}[$${p.binder}]${p.suffix}`;
|
|
793
797
|
}
|
|
794
|
-
|
|
798
|
+
const result = { ...p, stem, raw };
|
|
799
|
+
// Includes are independent materializations. Judge declarations carry
|
|
800
|
+
// optional scope arrays, which must not alias between child/aliases.
|
|
801
|
+
if (p.judges !== undefined) {
|
|
802
|
+
result.judges = p.judges.map((judge) => ({
|
|
803
|
+
...judge,
|
|
804
|
+
...(judge.modifiers !== undefined ? { modifiers: [...judge.modifiers] } : {}),
|
|
805
|
+
}));
|
|
806
|
+
}
|
|
807
|
+
return result;
|
|
795
808
|
};
|
|
796
809
|
const newProduces = step.produces.map(prefixProduce);
|
|
797
810
|
const newGenerates = step.generates ? step.generates.map(prefixProduce) : undefined;
|
|
@@ -836,6 +849,8 @@ function prefixStep(step, prefix, defInputs = []) {
|
|
|
836
849
|
result.effect = newEffect;
|
|
837
850
|
if (newJudges !== undefined)
|
|
838
851
|
result.judges = newJudges;
|
|
852
|
+
if (step.judgeModifiers !== undefined)
|
|
853
|
+
result.judgeModifiers = [...step.judgeModifiers];
|
|
839
854
|
if (newOnCancel !== undefined)
|
|
840
855
|
result.onCancel = newOnCancel;
|
|
841
856
|
if (newWorkdirFrom !== undefined)
|
|
@@ -1040,6 +1055,8 @@ function synthesizeJudgeSteps(producerStepName, pat, producerConsumeStems, produ
|
|
|
1040
1055
|
step.command = j.command;
|
|
1041
1056
|
if (j.spec !== undefined)
|
|
1042
1057
|
step.spec = j.spec;
|
|
1058
|
+
if (j.modifiers !== undefined)
|
|
1059
|
+
step.judgeModifiers = [...j.modifiers];
|
|
1043
1060
|
return step;
|
|
1044
1061
|
});
|
|
1045
1062
|
}
|
|
@@ -1580,6 +1597,34 @@ export function validateDef(def) {
|
|
|
1580
1597
|
if (!l.consumes.some((c) => c.mode === 'plain' && c.stem === judgedStem)) {
|
|
1581
1598
|
errors.push(`judge step '${l.name}' does not consume its judged stem '${judgedStem}'`);
|
|
1582
1599
|
}
|
|
1600
|
+
// A direct/in-memory WorkflowDef can bypass parse+synthesis, so keep the
|
|
1601
|
+
// compiled carrier fail-closed: it must exactly mirror the authored judge
|
|
1602
|
+
// declaration it represents (including true absence for unconditional).
|
|
1603
|
+
const marker = '.judges.';
|
|
1604
|
+
const markerIndex = l.name.lastIndexOf(marker);
|
|
1605
|
+
const judgeName = markerIndex === -1 ? l.name : l.name.slice(markerIndex + marker.length);
|
|
1606
|
+
const owner = def.steps
|
|
1607
|
+
.flatMap((candidate) => candidate.produces)
|
|
1608
|
+
.find((produce) => produce.kind === 'singleton' && produce.stem === judgedStem);
|
|
1609
|
+
const authored = owner?.judges?.find((judge) => judge.name === judgeName);
|
|
1610
|
+
if (authored === undefined) {
|
|
1611
|
+
errors.push(`judge step '${l.name}' has no matching judge declaration on produce '${judgedStem}'`);
|
|
1612
|
+
}
|
|
1613
|
+
else {
|
|
1614
|
+
const expected = authored.modifiers;
|
|
1615
|
+
const actual = l.judgeModifiers;
|
|
1616
|
+
const sameScope = expected === undefined
|
|
1617
|
+
? actual === undefined
|
|
1618
|
+
: actual !== undefined && expected.length === actual.length && expected.every((value, i) => value === actual[i]);
|
|
1619
|
+
if (!sameScope) {
|
|
1620
|
+
errors.push(`judge step '${l.name}' judgeModifiers must match the scope declared for judge '${judgeName}' on '${judgedStem}'`);
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
}
|
|
1624
|
+
for (const l of def.steps) {
|
|
1625
|
+
if (l.judges === undefined && l.judgeModifiers !== undefined) {
|
|
1626
|
+
errors.push(`step '${l.name}' sets judgeModifiers but is not a synthesized judge step`);
|
|
1627
|
+
}
|
|
1583
1628
|
}
|
|
1584
1629
|
// G25-VALIDATE: declarative exclusive produce-groups per-def rules.
|
|
1585
1630
|
for (const l of def.steps) {
|
|
@@ -1642,6 +1687,36 @@ export function validateDef(def) {
|
|
|
1642
1687
|
// "defs without `modifiers:` run exactly as today". Opting into the
|
|
1643
1688
|
// new routing vocabulary is what turns the stricter rule on.
|
|
1644
1689
|
const declaredModifiers = new Set(def.modifiers ?? []);
|
|
1690
|
+
// J24 modifier scopes are authored only on judge declarations. Validate the
|
|
1691
|
+
// vocabulary here, where the complete definition is available, rather than
|
|
1692
|
+
// letting an unknown scope silently make a gate disappear at runtime.
|
|
1693
|
+
for (const step of def.steps) {
|
|
1694
|
+
for (const produce of step.produces) {
|
|
1695
|
+
for (const judge of produce.judges ?? []) {
|
|
1696
|
+
if (judge.modifiers === undefined)
|
|
1697
|
+
continue;
|
|
1698
|
+
if (judge.modifiers.length === 0) {
|
|
1699
|
+
errors.push(`judge '${judge.name}' on produce '${produce.stem}' has an empty modifiers list`);
|
|
1700
|
+
}
|
|
1701
|
+
const seen = new Set();
|
|
1702
|
+
for (const modifier of judge.modifiers) {
|
|
1703
|
+
if (modifier.trim().length === 0 || /\s/.test(modifier) || modifier.includes(MODIFIER_SEPARATOR)) {
|
|
1704
|
+
errors.push(`judge '${judge.name}' on produce '${produce.stem}' has malformed modifier '${modifier}'`);
|
|
1705
|
+
}
|
|
1706
|
+
if (seen.has(modifier)) {
|
|
1707
|
+
errors.push(`judge '${judge.name}' on produce '${produce.stem}' lists modifier '${modifier}' more than once`);
|
|
1708
|
+
}
|
|
1709
|
+
seen.add(modifier);
|
|
1710
|
+
if (declaredModifiers.size === 0) {
|
|
1711
|
+
errors.push(`judge '${judge.name}' on produce '${produce.stem}' scopes modifiers but workflow '${def.name}' declares no modifiers:`);
|
|
1712
|
+
}
|
|
1713
|
+
else if (!declaredModifiers.has(modifier)) {
|
|
1714
|
+
errors.push(`judge '${judge.name}' on produce '${produce.stem}' modifier '${modifier}' is not in workflow '${def.name}'.modifiers (${[...declaredModifiers].join(', ')})`);
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
}
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1645
1720
|
const modifierBinds = [];
|
|
1646
1721
|
for (const l of def.steps) {
|
|
1647
1722
|
for (const p of l.produces) {
|
|
@@ -1669,6 +1744,15 @@ export function validateDef(def) {
|
|
|
1669
1744
|
if (modifierBinds.length > 1) {
|
|
1670
1745
|
errors.push(`workflow '${def.name}' binds modifier more than once: ${modifierBinds.map((b) => `'${b.step}.${b.artifact}'`).join(', ')}`);
|
|
1671
1746
|
}
|
|
1747
|
+
if (modifierBinds.length === 1) {
|
|
1748
|
+
const { boundArtifact, downstream } = modifierBoundDownstreamSteps(def);
|
|
1749
|
+
for (const step of def.steps) {
|
|
1750
|
+
const hasScopedJudge = step.produces.some((produce) => produce.judges?.some((judge) => judge.modifiers !== undefined));
|
|
1751
|
+
if (hasScopedJudge && !downstream.has(step.name)) {
|
|
1752
|
+
errors.push(`step '${step.name}' produces modifier-scoped judges but is not strictly downstream of artifact '${boundArtifact?.raw ?? modifierBinds[0].artifact}' bound to modifier`);
|
|
1753
|
+
}
|
|
1754
|
+
}
|
|
1755
|
+
}
|
|
1672
1756
|
for (const l of def.steps) {
|
|
1673
1757
|
if (l.escalation === undefined)
|
|
1674
1758
|
continue;
|
|
@@ -1922,8 +2006,14 @@ function consumesProducedPattern(consume, produce) {
|
|
|
1922
2006
|
}
|
|
1923
2007
|
return consume.mode !== 'plain' && consume.suffix === produce.suffix;
|
|
1924
2008
|
}
|
|
1925
|
-
/**
|
|
1926
|
-
|
|
2009
|
+
/**
|
|
2010
|
+
* Find the one modifier-binding artifact (when present) and every step that
|
|
2011
|
+
* is strictly downstream of it. The binding step itself is deliberately not
|
|
2012
|
+
* in the result: a sibling output could otherwise commit before the bind.
|
|
2013
|
+
*
|
|
2014
|
+
* This preserves the exact map-suffix matching used by the routing lint.
|
|
2015
|
+
*/
|
|
2016
|
+
function modifierBoundDownstreamSteps(def) {
|
|
1927
2017
|
let bindingStep;
|
|
1928
2018
|
let boundArtifact;
|
|
1929
2019
|
for (const step of def.steps) {
|
|
@@ -1935,29 +2025,34 @@ function unroutedCapabilityWarnings(def) {
|
|
|
1935
2025
|
}
|
|
1936
2026
|
}
|
|
1937
2027
|
if (bindingStep === undefined || boundArtifact === undefined)
|
|
1938
|
-
return
|
|
1939
|
-
|
|
1940
|
-
// exempt, but its unbound sibling outputs must not make a branch downstream.
|
|
1941
|
-
const reachedSteps = new Set([bindingStep.name]);
|
|
2028
|
+
return { downstream: new Set() };
|
|
2029
|
+
const downstream = new Set();
|
|
1942
2030
|
const reachedArtifacts = [boundArtifact];
|
|
1943
2031
|
let changed = true;
|
|
1944
2032
|
while (changed) {
|
|
1945
2033
|
changed = false;
|
|
1946
2034
|
for (const step of def.steps) {
|
|
1947
|
-
if (
|
|
2035
|
+
if (step.name === bindingStep.name || downstream.has(step.name))
|
|
1948
2036
|
continue;
|
|
1949
2037
|
if (!step.consumes.some((consume) => reachedArtifacts.some((produce) => consumesProducedPattern(consume, produce))))
|
|
1950
2038
|
continue;
|
|
1951
|
-
|
|
2039
|
+
downstream.add(step.name);
|
|
1952
2040
|
reachedArtifacts.push(...step.produces);
|
|
1953
2041
|
changed = true;
|
|
1954
2042
|
}
|
|
1955
2043
|
}
|
|
2044
|
+
return { bindingStep, boundArtifact, downstream };
|
|
2045
|
+
}
|
|
2046
|
+
/** Warn when routed work can run before the artifact that writes the modifier. */
|
|
2047
|
+
function unroutedCapabilityWarnings(def) {
|
|
2048
|
+
const { bindingStep, boundArtifact, downstream } = modifierBoundDownstreamSteps(def);
|
|
2049
|
+
if (bindingStep === undefined || boundArtifact === undefined)
|
|
2050
|
+
return [];
|
|
1956
2051
|
const warnings = [];
|
|
1957
2052
|
for (const step of def.steps) {
|
|
1958
2053
|
if (step.capabilities === undefined || step.capabilities.length === 0)
|
|
1959
2054
|
continue;
|
|
1960
|
-
if (
|
|
2055
|
+
if (step.name === bindingStep.name || downstream.has(step.name))
|
|
1961
2056
|
continue;
|
|
1962
2057
|
warnings.push(`step '${step.name}' declares capabilities (${step.capabilities.join(', ')}) but is not downstream of artifact '${boundArtifact.raw}' bound to modifier`);
|
|
1963
2058
|
}
|