owenloop 0.5.7 → 0.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/README.md +7 -8
- package/dist/packages/work/src/agent/capability-model.d.ts +64 -82
- package/dist/packages/work/src/agent/capability-model.js +94 -91
- package/dist/packages/work/src/agent/loop.d.ts +55 -26
- package/dist/packages/work/src/agent/loop.js +144 -50
- package/dist/packages/work/src/bundle/types.d.ts +3 -3
- package/dist/packages/work/src/consumed-verifier.js +6 -17
- package/dist/packages/work/src/exec/loop.d.ts +1 -1
- package/dist/packages/work/src/exec/loop.js +206 -35
- package/dist/packages/work/src/exec/receipt.d.ts +7 -7
- package/dist/packages/work/src/exec/receipt.js +6 -6
- package/dist/packages/work/src/harness/child-env.js +4 -5
- package/dist/packages/work/src/harness/claude.js +100 -0
- package/dist/packages/work/src/harness/codex.js +3 -4
- package/dist/packages/work/src/harness/session-store.d.ts +63 -3
- package/dist/packages/work/src/harness/session-store.js +104 -9
- package/dist/packages/work/src/hub/client.d.ts +19 -4
- package/dist/packages/work/src/hub/client.js +7 -3
- package/dist/packages/work/src/hub/types.d.ts +87 -3
- package/dist/packages/work/src/main.js +1 -0
- package/dist/packages/work/src/roles/agent-run.js +46 -43
- package/dist/packages/work/src/roles/exec.js +2 -0
- package/dist/packages/work/src/roles/sessions.d.ts +3 -2
- package/dist/packages/work/src/roles/sessions.js +9 -2
- package/dist/packages/work/src/roles/settings.js +3 -1
- package/dist/packages/work/src/roles/shift.js +14 -2
- package/dist/packages/work/src/roles/util.d.ts +1 -0
- package/dist/packages/work/src/roles/util.js +10 -0
- package/dist/packages/work/src/settings/hub-roster-cache.d.ts +68 -0
- package/dist/packages/work/src/settings/hub-roster-cache.js +370 -0
- package/dist/packages/work/src/settings/roster.d.ts +105 -0
- package/dist/packages/work/src/settings/roster.js +439 -0
- package/dist/packages/work/src/settings/serving.d.ts +17 -0
- package/dist/packages/work/src/settings/serving.js +79 -0
- package/dist/packages/work/src/settings/settings.d.ts +25 -24
- package/dist/packages/work/src/settings/settings.js +29 -33
- package/dist/packages/work/src/shift/loop.d.ts +108 -5
- package/dist/packages/work/src/shift/loop.js +299 -51
- package/dist/packages/work/src/shift/protocol.d.ts +7 -5
- package/dist/packages/work/src/shift/runtime.d.ts +15 -0
- package/dist/packages/work/src/shift/runtime.js +95 -13
- package/dist/packages/work/src/shift/server.js +2 -0
- package/dist/packages/work/src/shift/spawn.d.ts +18 -50
- package/dist/packages/work/src/shift/spawn.js +30 -65
- package/dist/packages/work/src/shift/state.d.ts +29 -2
- package/dist/packages/work/src/shift/state.js +45 -15
- package/dist/packages/work/src/usage.d.ts +1 -1
- package/dist/packages/work/src/usage.js +10 -5
- package/dist/packages/work/src/util/modifier-init.d.ts +7 -0
- package/dist/packages/work/src/util/modifier-init.js +73 -0
- package/dist/src/bundle/tar.d.ts +9 -4
- package/dist/src/bundle/tar.js +9 -4
- package/dist/src/capabilities.d.ts +50 -0
- package/dist/src/capabilities.js +38 -0
- package/dist/src/cli.d.ts +15 -1
- package/dist/src/cli.js +1099 -32
- package/dist/src/config-dir.d.ts +14 -19
- package/dist/src/config-dir.js +16 -24
- package/dist/src/credentials.d.ts +1 -1
- package/dist/src/credentials.js +1 -1
- package/dist/src/crypto/index.d.ts +1 -1
- package/dist/src/crypto/index.js +1 -1
- package/dist/src/crypto/org-root.d.ts +30 -12
- package/dist/src/crypto/org-root.js +162 -14
- package/dist/src/crypto/records.d.ts +3 -0
- package/dist/src/crypto/records.js +8 -0
- package/dist/src/crypto/trust-roots.d.ts +2 -2
- package/dist/src/crypto/trust-roots.js +2 -2
- package/dist/src/defs.d.ts +6 -1
- package/dist/src/defs.js +203 -8
- package/dist/src/engine.d.ts +55 -6
- package/dist/src/engine.js +172 -22
- package/dist/src/global-config.d.ts +4 -7
- package/dist/src/global-config.js +4 -7
- package/dist/src/hub.d.ts +207 -5
- package/dist/src/hub.js +263 -3
- package/dist/src/index.d.ts +5 -5
- package/dist/src/index.js +3 -3
- package/dist/src/mcp/serve.d.ts +22 -0
- package/dist/src/mcp/serve.js +88 -8
- package/dist/src/model.d.ts +2 -0
- package/dist/src/model.js +32 -25
- package/dist/src/order-resolver.d.ts +1 -0
- package/dist/src/order-resolver.js +8 -1
- package/dist/src/schemas/index.d.ts +19 -0
- package/dist/src/schemas/order.v1.schema.json +13 -2
- package/dist/src/store/pre-commit-verifier.js +1 -1
- package/dist/src/store.d.ts +5 -0
- package/dist/src/store.js +43 -8
- package/dist/src/types.d.ts +64 -7
- package/dist/src/work-settings.d.ts +2 -4
- package/dist/src/work-settings.js +2 -4
- package/docs/agent-runner.md +2 -2
- package/docs/authoring.md +90 -2
- package/docs/bundles.md +29 -3
- package/docs/cli.md +456 -120
- package/docs/crypto.md +32 -16
- package/docs/design.md +70 -1
- package/docs/routing.md +762 -0
- package/docs/shift-logs.md +37 -10
- package/docs/wire-contracts.md +50 -7
- package/examples/workflows/delivery.yaml +31 -14
- package/examples/workflows/ship.yaml +10 -2
- package/package.json +1 -1
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/plugin/.mcp.json +1 -1
- package/plugins/claude-code/plugin/skills/author/SKILL.md +1 -2
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
- package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
- package/plugins/codex/plugins/owenloop/skills/author/SKILL.md +1 -2
|
@@ -27,9 +27,11 @@
|
|
|
27
27
|
* liveInFlight counts every live child record, each one pid-probed (see
|
|
28
28
|
* state.ts). When `k <= 0` the loop skips `whats_next` but remembers that the
|
|
29
29
|
* adopted wake still needs a sweep. Orders already returned by `whats_next`
|
|
30
|
-
* are already claimed
|
|
31
|
-
*
|
|
32
|
-
*
|
|
30
|
+
* are already claimed. Orders this shift cannot dispatch are released back to
|
|
31
|
+
* the hub immediately, so another daemon can take them; `localQueueHoldMs`
|
|
32
|
+
* optionally retains them locally for a bounded window instead. The agent lane
|
|
33
|
+
* is additionally bounded by `cap − execReserve`, preserving room for an exec
|
|
34
|
+
* order while agents are saturated.
|
|
33
35
|
*
|
|
34
36
|
* RESILIENCE: presence/wake/whats_next failures are logged and never kill the
|
|
35
37
|
* loop. `stop()` flips a flag checked between awaits; the in-flight sweep
|
|
@@ -53,21 +55,66 @@
|
|
|
53
55
|
*/
|
|
54
56
|
import { join } from 'node:path';
|
|
55
57
|
import { performance } from 'node:perf_hooks';
|
|
56
|
-
import { acquireFileLockSync, releaseFileLock } from "../../../../src/lock.js";
|
|
58
|
+
import { acquireFileLockSync, FileLockTimeoutError, releaseFileLock, } from "../../../../src/lock.js";
|
|
57
59
|
import { readDispatchBundle } from "../bundle/cache.js";
|
|
58
60
|
import { HubError } from "../hub/types.js";
|
|
59
61
|
import { isCommandStep, resolveCommandRouting } from "./routing.js";
|
|
60
|
-
import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reserveChild, startReservedChild, removeChildRecord, ensureStateDir, } from "./state.js";
|
|
62
|
+
import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reserveChild, startReservedChild, removeChildRecord, settleChildGate, ensureStateDir, } from "./state.js";
|
|
61
63
|
import { DEFAULT_WORK_DIR_TTL_MS, sweepWorkDirs as sweepWorkDirsImpl } from "../agent/workdir.js";
|
|
64
|
+
import { withHubRosterSyncTimeout } from "../settings/hub-roster-cache.js";
|
|
62
65
|
import { sessionsPath } from "../harness/session-store.js";
|
|
63
66
|
import { stampShiftEvent, } from "./protocol.js";
|
|
67
|
+
/** Run a state mutation while holding the lock that serializes dispatch state. */
|
|
68
|
+
export function withDispatchLock(stateDir, options = {}, fn) {
|
|
69
|
+
const dispatchLock = acquireFileLockSync(join(stateDir, '.dispatch.lock'), {
|
|
70
|
+
...options.dispatchLockOptions,
|
|
71
|
+
waitMs: options.waitMs ?? 30_000,
|
|
72
|
+
label: options.label ?? 'owenloop Shift dispatch-state removal',
|
|
73
|
+
});
|
|
74
|
+
try {
|
|
75
|
+
return fn();
|
|
76
|
+
}
|
|
77
|
+
finally {
|
|
78
|
+
releaseFileLock(dispatchLock);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Remove one child record while holding the same lock that serializes durable
|
|
83
|
+
* reservation creation. The record is deliberately re-read by
|
|
84
|
+
* `removeChildRecord` only after acquisition, so a stale observer cannot erase
|
|
85
|
+
* a newer dispatch for the same run.
|
|
86
|
+
*/
|
|
87
|
+
export function removeChildRecordUnderDispatchLock(stateDir, run, recordOptions, options = {}) {
|
|
88
|
+
return withDispatchLock(stateDir, options, () => removeChildRecord(stateDir, run, recordOptions));
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Reconciliation callbacks for every unlocked Shift daemon path. Keep these
|
|
92
|
+
* together with the lock-held removal primitive so startup and the poll loop
|
|
93
|
+
* cannot silently diverge.
|
|
94
|
+
*/
|
|
95
|
+
export function createLockedRemovalCallbacks(stateDir, options = {}) {
|
|
96
|
+
return {
|
|
97
|
+
removeAbandonedReservation: (reservation) => withDispatchLock(stateDir, options, () => cancelReservedChild(stateDir, reservation)),
|
|
98
|
+
removeDeadChild: (record) => withDispatchLock(stateDir, options, () => removeChildRecord(stateDir, record.run, { pid: record.pid })),
|
|
99
|
+
settleLiveChild: (record) => withDispatchLock(stateDir, options, () => settleChildGate(stateDir, record)),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
64
102
|
/** The `maxConcurrentAgents` fallback when the option is absent. */
|
|
65
103
|
const DEFAULT_MAX_AGENTS = 4;
|
|
104
|
+
/** The `execReserve` fallback when the option is absent. */
|
|
105
|
+
const DEFAULT_EXEC_RESERVE = 1;
|
|
106
|
+
/** The `localQueueHoldMs` fallback when the option is absent. */
|
|
107
|
+
const DEFAULT_LOCAL_QUEUE_HOLD_MS = 0;
|
|
108
|
+
/** The hub reaps a never-contacted claim after this long. Every client-side
|
|
109
|
+
* re-offer latency must stay strictly under it. */
|
|
110
|
+
export const HUB_PICKUP_WINDOW_MS = 120_000;
|
|
111
|
+
/** Headroom for a detached child to make first contact after dispatch. */
|
|
112
|
+
const CHILD_FIRST_CONTACT_MARGIN_MS = 30_000;
|
|
66
113
|
/**
|
|
67
114
|
* The hub reaps a never-contacted claim after 120 seconds. Stop local queueing
|
|
68
115
|
* after 90 seconds so a detached child retains 30 seconds to make first contact.
|
|
69
116
|
*/
|
|
70
|
-
export const MAX_PENDING_CANDIDATE_AGE_MS =
|
|
117
|
+
export const MAX_PENDING_CANDIDATE_AGE_MS = HUB_PICKUP_WINDOW_MS - CHILD_FIRST_CONTACT_MARGIN_MS;
|
|
71
118
|
/**
|
|
72
119
|
* THE PER-STEP FAILURE BRAKE.
|
|
73
120
|
*
|
|
@@ -101,11 +148,11 @@ export const MAX_PENDING_CANDIDATE_AGE_MS = 90_000;
|
|
|
101
148
|
* It is a rate limit, never a ban. A step that has not failed is dispatched
|
|
102
149
|
* with zero delay, however many orders arrive for it at once.
|
|
103
150
|
*/
|
|
104
|
-
const STEP_BRAKE_DELAYS_MS = [2_000, 8_000, 30_000,
|
|
151
|
+
export const STEP_BRAKE_DELAYS_MS = [2_000, 8_000, 30_000, 60_000, 90_000];
|
|
105
152
|
/**
|
|
106
153
|
* Treat a failure this long after the previous one as the start of a NEW
|
|
107
154
|
* streak, resetting the count to zero first. Comfortably longer than the
|
|
108
|
-
* longest delay in the table (
|
|
155
|
+
* longest delay in the table (90s), so a step failing at maximum backoff never
|
|
109
156
|
* decays out of it, while an isolated transient long after an old streak starts
|
|
110
157
|
* again at the shortest delay instead of inheriting a stale penalty.
|
|
111
158
|
*/
|
|
@@ -121,7 +168,7 @@ const STEP_BRAKE_FORGET_MS = 1_800_000;
|
|
|
121
168
|
* worker failures. `count` is at least 1 at every call site (this is reached
|
|
122
169
|
* only from the failure path), so the FIRST failure already costs 2s; past the
|
|
123
170
|
* end of the table the last entry repeats, so a permanently-failing step
|
|
124
|
-
* settles at one dispatch per
|
|
171
|
+
* settles at one dispatch per 90 seconds instead of one per poll.
|
|
125
172
|
*/
|
|
126
173
|
function stepBrakeDelayMs(count) {
|
|
127
174
|
const index = Math.min(count, STEP_BRAKE_DELAYS_MS.length) - 1;
|
|
@@ -146,6 +193,23 @@ function isNonServableRace(e) {
|
|
|
146
193
|
export function createShiftLoop(opts) {
|
|
147
194
|
let stopped = false;
|
|
148
195
|
let cap = opts.cap;
|
|
196
|
+
/**
|
|
197
|
+
* The agent lane's live budget. Recomputed per call because `setCap`
|
|
198
|
+
* rewrites `cap` at runtime.
|
|
199
|
+
*/
|
|
200
|
+
function agentLane() {
|
|
201
|
+
const requested = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
|
|
202
|
+
const reserve = Math.min(opts.execReserve ?? DEFAULT_EXEC_RESERVE, Math.max(0, cap - 1));
|
|
203
|
+
return {
|
|
204
|
+
ceiling: Math.max(0, Math.min(requested, cap - reserve)),
|
|
205
|
+
requested,
|
|
206
|
+
reserve,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
/** Effective local hold, capped below the hub pickup window. */
|
|
210
|
+
function localQueueHoldMs() {
|
|
211
|
+
return Math.min(Math.max(0, opts.localQueueHoldMs ?? DEFAULT_LOCAL_QUEUE_HOLD_MS), MAX_PENDING_CANDIDATE_AGE_MS);
|
|
212
|
+
}
|
|
149
213
|
const isAlive = opts.isAlive;
|
|
150
214
|
const monotonicNow = opts.monotonicNow ?? performance.now.bind(performance);
|
|
151
215
|
// Live shift identity (MCP `clock_in`, D3-D7 of the plan). Seeded from opts,
|
|
@@ -153,9 +217,32 @@ export function createShiftLoop(opts) {
|
|
|
153
217
|
// nor a caller of getShift/setShift can mutate the other's state afterward.
|
|
154
218
|
let shiftName = opts.name;
|
|
155
219
|
let serveCrews = [...opts.serveCrews];
|
|
220
|
+
let serveCapabilities = [];
|
|
221
|
+
let initial = true;
|
|
222
|
+
function refreshServeCapabilities() {
|
|
223
|
+
if (opts.computeServeCapabilities === undefined)
|
|
224
|
+
return;
|
|
225
|
+
let next;
|
|
226
|
+
try {
|
|
227
|
+
next = [...opts.computeServeCapabilities(serveCrews)];
|
|
228
|
+
}
|
|
229
|
+
catch {
|
|
230
|
+
return; // Keep the last known-good advertisement on a transient failure.
|
|
231
|
+
}
|
|
232
|
+
const changed = next.length !== serveCapabilities.length ||
|
|
233
|
+
next.some((capability, index) => capability !== serveCapabilities[index]);
|
|
234
|
+
serveCapabilities = next;
|
|
235
|
+
if (changed && !initial)
|
|
236
|
+
opts.out(`serving ${next.length === 0 ? '(nothing)' : next.join(', ')}`);
|
|
237
|
+
}
|
|
238
|
+
refreshServeCapabilities();
|
|
239
|
+
initial = false;
|
|
156
240
|
// Park state persists across `iterate()` calls (the MCP park reuses it).
|
|
157
241
|
let cursor;
|
|
158
242
|
let lastPresence = Number.NEGATIVE_INFINITY;
|
|
243
|
+
// Runtime performs the required shift-start refresh before creating this
|
|
244
|
+
// loop. Start this cadence now so it is not immediately fetched twice.
|
|
245
|
+
let lastRosterSync = monotonicNow();
|
|
159
246
|
let attendedAt;
|
|
160
247
|
/**
|
|
161
248
|
* Orders returned by `whats_next` are already claimed. If the hub returns more
|
|
@@ -168,6 +255,9 @@ export function createShiftLoop(opts) {
|
|
|
168
255
|
/** A changed wake whose sweep was skipped or failed must be retried after the
|
|
169
256
|
* cursor is adopted; otherwise the next unchanged wake hides that work. */
|
|
170
257
|
let sweepOwed = false;
|
|
258
|
+
/** Earliest monotonic time at which a braked step becomes dispatchable again.
|
|
259
|
+
* Drives a sweep without a hub cursor move, so a brake never waits forever. */
|
|
260
|
+
let brakeSweepDueAt;
|
|
171
261
|
/**
|
|
172
262
|
* Has the current at-capacity EPISODE already produced a `capacity` event?
|
|
173
263
|
*
|
|
@@ -213,10 +303,41 @@ export function createShiftLoop(opts) {
|
|
|
213
303
|
opts.err(`shift event sink failed: ${errMsg(e)} (continuing)`);
|
|
214
304
|
}
|
|
215
305
|
}
|
|
306
|
+
/**
|
|
307
|
+
* Hand a claim back so another shift can be offered it immediately. This
|
|
308
|
+
* deliberately stays off the dispatch critical path; a failed release falls
|
|
309
|
+
* back to the existing pickup-window behavior and is observable.
|
|
310
|
+
*/
|
|
311
|
+
function releaseClaim(workflow, run) {
|
|
312
|
+
void opts.hub.release({ workflow, run }).catch((e) => {
|
|
313
|
+
noteServerBackoff(e);
|
|
314
|
+
const message = errMsg(e);
|
|
315
|
+
opts.err(`[${workflow}/${run}] release failed: ${message} — leaving the hub pickup window to re-offer it`);
|
|
316
|
+
emit({ type: 'hub-error', op: 'release', workflow, message });
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
function acquireDispatchLock(waitMs, label) {
|
|
320
|
+
return acquireFileLockSync(join(opts.stateDir, '.dispatch.lock'), {
|
|
321
|
+
...opts.dispatchLockOptions,
|
|
322
|
+
waitMs,
|
|
323
|
+
label,
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
const lockedRemovalCallbacks = createLockedRemovalCallbacks(opts.stateDir, {
|
|
327
|
+
dispatchLockOptions: opts.dispatchLockOptions,
|
|
328
|
+
});
|
|
329
|
+
function removeRecordUnderDispatchLock(run, recordOptions, waitMs = 30_000, label = 'owenloop Shift dispatch-state removal') {
|
|
330
|
+
return removeChildRecordUnderDispatchLock(opts.stateDir, run, recordOptions, {
|
|
331
|
+
dispatchLockOptions: opts.dispatchLockOptions,
|
|
332
|
+
waitMs,
|
|
333
|
+
label,
|
|
334
|
+
});
|
|
335
|
+
}
|
|
216
336
|
function reconcile() {
|
|
217
337
|
const result = reconcileInFlight(opts.stateDir, {
|
|
218
338
|
...(isAlive !== undefined ? { isAlive } : {}),
|
|
219
339
|
now: opts.now(),
|
|
340
|
+
...lockedRemovalCallbacks,
|
|
220
341
|
});
|
|
221
342
|
for (const rec of result.reaped) {
|
|
222
343
|
emit({
|
|
@@ -263,10 +384,7 @@ export function createShiftLoop(opts) {
|
|
|
263
384
|
*/
|
|
264
385
|
function reserveCandidate(c) {
|
|
265
386
|
const childKind = c.kind === 'command' ? 'exec' : 'agent-run';
|
|
266
|
-
const dispatchLock =
|
|
267
|
-
waitMs: 30_000,
|
|
268
|
-
label: 'owenloop Shift dispatch',
|
|
269
|
-
});
|
|
387
|
+
const dispatchLock = acquireDispatchLock(30_000, 'owenloop Shift dispatch');
|
|
270
388
|
try {
|
|
271
389
|
const fresh = reconcileInFlight(opts.stateDir, {
|
|
272
390
|
...(isAlive === undefined ? {} : { isAlive }),
|
|
@@ -280,7 +398,7 @@ export function createShiftLoop(opts) {
|
|
|
280
398
|
if (childKind === 'agent-run') {
|
|
281
399
|
const agents = fresh.live.filter((record) => record.kind === 'agent-run').length +
|
|
282
400
|
fresh.reserved.filter((record) => record.childKind === 'agent-run').length;
|
|
283
|
-
if (agents >= (
|
|
401
|
+
if (agents >= agentLane().ceiling)
|
|
284
402
|
return 'agent-capacity';
|
|
285
403
|
}
|
|
286
404
|
return reserveChild(opts.stateDir, {
|
|
@@ -312,6 +430,9 @@ export function createShiftLoop(opts) {
|
|
|
312
430
|
opts.err(`[${c.workflow}/${c.order.run}] step '${c.order.step}' has failed ` +
|
|
313
431
|
`${brake.count} time(s) in a row — braking for ${waitMs}ms and ` +
|
|
314
432
|
`leaving this claim to lapse`);
|
|
433
|
+
brakeSweepDueAt = brakeSweepDueAt === undefined
|
|
434
|
+
? brake.nextAllowedAt
|
|
435
|
+
: Math.min(brakeSweepDueAt, brake.nextAllowedAt);
|
|
315
436
|
return 'braked';
|
|
316
437
|
}
|
|
317
438
|
let reservation;
|
|
@@ -331,15 +452,22 @@ export function createShiftLoop(opts) {
|
|
|
331
452
|
startGate: reserved.gatePath,
|
|
332
453
|
});
|
|
333
454
|
cancel = spawned.cancel ?? spawned.terminate;
|
|
334
|
-
const
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
455
|
+
const dispatchLock = acquireDispatchLock(30_000, 'owenloop Shift dispatch');
|
|
456
|
+
let rec;
|
|
457
|
+
try {
|
|
458
|
+
rec = finalizeChildReservation(opts.stateDir, reservation, {
|
|
459
|
+
pid: spawned.pid,
|
|
460
|
+
spawnedAt: opts.now(),
|
|
461
|
+
kind: childKind,
|
|
462
|
+
...(childKind === 'agent-run' && c.defName !== undefined ? { def: c.defName } : {}),
|
|
463
|
+
...(childKind === 'agent-run' && c.defHash !== undefined ? { hash: c.defHash } : {}),
|
|
464
|
+
...(childKind === 'agent-run' ? { step: c.order.step } : {}),
|
|
465
|
+
});
|
|
466
|
+
startReservedChild(opts.stateDir, rec);
|
|
467
|
+
}
|
|
468
|
+
finally {
|
|
469
|
+
releaseFileLock(dispatchLock);
|
|
470
|
+
}
|
|
343
471
|
// Remember which step this run belongs to, so a later worker failure —
|
|
344
472
|
// which names the run but not the fan-out key — can be charged to the
|
|
345
473
|
// right brake key. NOTHING is counted here: a dispatch is not evidence of
|
|
@@ -362,7 +490,13 @@ export function createShiftLoop(opts) {
|
|
|
362
490
|
cancel?.();
|
|
363
491
|
if (reservation !== undefined) {
|
|
364
492
|
try {
|
|
365
|
-
|
|
493
|
+
const dispatchLock = acquireDispatchLock(30_000, 'owenloop Shift dispatch');
|
|
494
|
+
try {
|
|
495
|
+
cancelReservedChild(opts.stateDir, reservation);
|
|
496
|
+
}
|
|
497
|
+
finally {
|
|
498
|
+
releaseFileLock(dispatchLock);
|
|
499
|
+
}
|
|
366
500
|
}
|
|
367
501
|
catch (cleanupError) {
|
|
368
502
|
opts.err(`[${c.workflow}/${c.order.run}] failed to cancel dispatch reservation: ${errMsg(cleanupError)}`);
|
|
@@ -383,26 +517,35 @@ export function createShiftLoop(opts) {
|
|
|
383
517
|
}
|
|
384
518
|
}
|
|
385
519
|
function discardExpiredCandidate(candidate, queued) {
|
|
386
|
-
|
|
520
|
+
const holdMs = queued ? localQueueHoldMs() : MAX_PENDING_CANDIDATE_AGE_MS;
|
|
521
|
+
if (monotonicNow() - candidate.requestStartedAt < holdMs)
|
|
387
522
|
return false;
|
|
388
523
|
const prefix = queued ? 'queued claim' : 'claim';
|
|
389
|
-
|
|
524
|
+
const message = `${prefix} expired before local dispatch — handing the claim back to the hub`;
|
|
525
|
+
opts.err(`[${candidate.workflow}/${candidate.order.run}] ${message}`);
|
|
526
|
+
emit({
|
|
527
|
+
type: 'order-dropped',
|
|
528
|
+
workflow: candidate.workflow,
|
|
529
|
+
run: candidate.order.run,
|
|
530
|
+
step: candidate.order.step,
|
|
531
|
+
reason: 'claim-expired',
|
|
532
|
+
message,
|
|
533
|
+
});
|
|
534
|
+
releaseClaim(candidate.workflow, candidate.order.run);
|
|
390
535
|
return true;
|
|
391
536
|
}
|
|
392
537
|
/** Dispatch already-claimed orders when local child capacity becomes free. */
|
|
393
538
|
function drainPending(live, reserved) {
|
|
394
539
|
let remaining = cap - live.length - reserved.length;
|
|
395
|
-
if (
|
|
540
|
+
if (pendingCandidates.size === 0)
|
|
396
541
|
return 0;
|
|
397
|
-
const
|
|
398
|
-
let agentRoom =
|
|
542
|
+
const { ceiling } = agentLane();
|
|
543
|
+
let agentRoom = ceiling -
|
|
399
544
|
live.filter((r) => r.kind === 'agent-run').length -
|
|
400
545
|
reserved.filter((r) => r.childKind === 'agent-run').length;
|
|
401
546
|
const liveRuns = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
|
|
402
547
|
let dispatched = 0;
|
|
403
548
|
for (const [run, candidate] of pendingCandidates) {
|
|
404
|
-
if (remaining <= 0)
|
|
405
|
-
break;
|
|
406
549
|
if (discardExpiredCandidate(candidate, true)) {
|
|
407
550
|
pendingCandidates.delete(run);
|
|
408
551
|
continue;
|
|
@@ -411,6 +554,10 @@ export function createShiftLoop(opts) {
|
|
|
411
554
|
pendingCandidates.delete(run);
|
|
412
555
|
continue;
|
|
413
556
|
}
|
|
557
|
+
// Even at full capacity, keep scanning so a configured local hold is a
|
|
558
|
+
// real deadline rather than one deferred until another child exits.
|
|
559
|
+
if (remaining <= 0)
|
|
560
|
+
continue;
|
|
414
561
|
// The agent lane's second budget can be full while the dispatch cap still
|
|
415
562
|
// has room. Skip this entry and KEEP it queued (a command entry further
|
|
416
563
|
// down the map is still dispatchable) — `continue`, never `break`, so one
|
|
@@ -418,10 +565,14 @@ export function createShiftLoop(opts) {
|
|
|
418
565
|
if (candidate.kind === 'agent' && agentRoom <= 0)
|
|
419
566
|
continue;
|
|
420
567
|
const result = dispatchCandidate(candidate);
|
|
421
|
-
if (result === 'total-capacity')
|
|
422
|
-
|
|
423
|
-
if (result === 'agent-capacity')
|
|
568
|
+
if (result === 'total-capacity') {
|
|
569
|
+
remaining = 0;
|
|
424
570
|
continue;
|
|
571
|
+
}
|
|
572
|
+
if (result === 'agent-capacity') {
|
|
573
|
+
agentRoom = 0;
|
|
574
|
+
continue;
|
|
575
|
+
}
|
|
425
576
|
pendingCandidates.delete(run);
|
|
426
577
|
if (result !== 'dispatched')
|
|
427
578
|
continue;
|
|
@@ -482,8 +633,11 @@ export function createShiftLoop(opts) {
|
|
|
482
633
|
// The agent lane's SECOND budget (D7). Counted from live records and durable
|
|
483
634
|
// reservations, so a crash between reservation and spawn cannot overbook it.
|
|
484
635
|
// A child or reservation that is reaped frees the slot on the next reconcile.
|
|
485
|
-
const
|
|
486
|
-
|
|
636
|
+
const { ceiling, requested, reserve } = agentLane();
|
|
637
|
+
const agentCapLabel = reserve > 0 && ceiling < requested
|
|
638
|
+
? `${ceiling}, ${requested} requested minus a ${reserve}-slot exec reserve`
|
|
639
|
+
: `${ceiling}`;
|
|
640
|
+
let agentRoom = ceiling -
|
|
487
641
|
live.filter((r) => r.kind === 'agent-run').length -
|
|
488
642
|
reserved.filter((r) => r.childKind === 'agent-run').length;
|
|
489
643
|
/**
|
|
@@ -504,7 +658,7 @@ export function createShiftLoop(opts) {
|
|
|
504
658
|
}
|
|
505
659
|
else {
|
|
506
660
|
try {
|
|
507
|
-
const inbox = await opts.hub.whatsNext();
|
|
661
|
+
const inbox = await opts.hub.whatsNext({ serve_capabilities: [...serveCapabilities] });
|
|
508
662
|
instances = (inbox.instances ?? []).map((i) => i.workflow);
|
|
509
663
|
}
|
|
510
664
|
catch (e) {
|
|
@@ -525,7 +679,8 @@ export function createShiftLoop(opts) {
|
|
|
525
679
|
const claimed = new Set();
|
|
526
680
|
const candidates = [];
|
|
527
681
|
/**
|
|
528
|
-
* Refuse one order
|
|
682
|
+
* Refuse one order. Some reasons intentionally leave the claim for the hub
|
|
683
|
+
* pickup window; capacity reasons use `releaseOrder` below instead.
|
|
529
684
|
*
|
|
530
685
|
* Every refusal below is a DROPPED UNIT OF WORK, not a debug aside: the
|
|
531
686
|
* shift declines an order the hub already handed it, and until now the only
|
|
@@ -539,11 +694,23 @@ export function createShiftLoop(opts) {
|
|
|
539
694
|
emit({ type: 'order-dropped', workflow, run: order.run, step: order.step, reason, message });
|
|
540
695
|
claimed.add(order.run);
|
|
541
696
|
};
|
|
697
|
+
/**
|
|
698
|
+
* Refuse an order because this shift lacks capacity, then hand its claim
|
|
699
|
+
* back so a sibling daemon may dispatch it immediately.
|
|
700
|
+
*/
|
|
701
|
+
const releaseOrder = (workflow, order, reason, message) => {
|
|
702
|
+
dropOrder(workflow, order, reason, message);
|
|
703
|
+
releaseClaim(workflow, order.run);
|
|
704
|
+
};
|
|
542
705
|
for (const wf of instances) {
|
|
543
706
|
let res;
|
|
544
707
|
const requestStartedAt = monotonicNow();
|
|
545
708
|
try {
|
|
546
|
-
res = await opts.hub.whatsNext({
|
|
709
|
+
res = await opts.hub.whatsNext({
|
|
710
|
+
workflow: wf,
|
|
711
|
+
serve_crews: serveCrews,
|
|
712
|
+
serve_capabilities: [...serveCapabilities],
|
|
713
|
+
});
|
|
547
714
|
}
|
|
548
715
|
catch (e) {
|
|
549
716
|
if (isNonServableRace(e)) {
|
|
@@ -666,8 +833,13 @@ export function createShiftLoop(opts) {
|
|
|
666
833
|
continue;
|
|
667
834
|
}
|
|
668
835
|
if (remaining <= 0) {
|
|
669
|
-
|
|
670
|
-
|
|
836
|
+
if (localQueueHoldMs() > 0) {
|
|
837
|
+
pendingCandidates.set(order.run, candidate);
|
|
838
|
+
opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
|
|
839
|
+
}
|
|
840
|
+
else {
|
|
841
|
+
releaseOrder(wf, order, 'dispatch-cap-full', `at the dispatch cap (${cap}) — handing the claim back to the hub`);
|
|
842
|
+
}
|
|
671
843
|
}
|
|
672
844
|
else {
|
|
673
845
|
remaining--;
|
|
@@ -678,6 +850,7 @@ export function createShiftLoop(opts) {
|
|
|
678
850
|
}
|
|
679
851
|
// Agent wire routing is authoritative. The detached agent-run resolves its
|
|
680
852
|
// own exact step and harness from the order digest, never from this cache.
|
|
853
|
+
// A re-offer already has a live child record; never release it.
|
|
681
854
|
if (workerRuns.has(order.run))
|
|
682
855
|
continue;
|
|
683
856
|
const candidate = {
|
|
@@ -694,13 +867,27 @@ export function createShiftLoop(opts) {
|
|
|
694
867
|
claimed.add(order.run);
|
|
695
868
|
continue;
|
|
696
869
|
}
|
|
870
|
+
if (ceiling === 0) {
|
|
871
|
+
releaseOrder(wf, order, 'agent-lane-closed', 'this shift runs no agent-run children (agent ceiling 0) — handing the claim back to the hub');
|
|
872
|
+
continue;
|
|
873
|
+
}
|
|
697
874
|
if (!reoffer && remaining <= 0) {
|
|
698
|
-
|
|
699
|
-
|
|
875
|
+
if (localQueueHoldMs() > 0) {
|
|
876
|
+
pendingCandidates.set(order.run, candidate);
|
|
877
|
+
opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
|
|
878
|
+
}
|
|
879
|
+
else {
|
|
880
|
+
releaseOrder(wf, order, 'dispatch-cap-full', `at the dispatch cap (${cap}) — handing the claim back to the hub`);
|
|
881
|
+
}
|
|
700
882
|
}
|
|
701
883
|
else if (!reoffer && agentRoom <= 0) {
|
|
702
|
-
|
|
703
|
-
|
|
884
|
+
if (localQueueHoldMs() > 0) {
|
|
885
|
+
pendingCandidates.set(order.run, candidate);
|
|
886
|
+
opts.out(`[${wf}/${order.run}] at the agent-run cap (${agentCapLabel}) — queued for local dispatch`);
|
|
887
|
+
}
|
|
888
|
+
else {
|
|
889
|
+
releaseOrder(wf, order, 'agent-cap-full', `at the agent-run cap (${agentCapLabel}) — handing the claim back to the hub`);
|
|
890
|
+
}
|
|
704
891
|
}
|
|
705
892
|
else {
|
|
706
893
|
if (!reoffer) {
|
|
@@ -725,11 +912,16 @@ export function createShiftLoop(opts) {
|
|
|
725
912
|
continue;
|
|
726
913
|
}
|
|
727
914
|
if (result === 'total-capacity' || result === 'agent-capacity') {
|
|
728
|
-
pendingCandidates.set(candidate.order.run, candidate);
|
|
729
915
|
const limit = result === 'total-capacity'
|
|
730
916
|
? `dispatch cap (${cap})`
|
|
731
|
-
: `agent-run cap (${
|
|
732
|
-
|
|
917
|
+
: `agent-run cap (${agentCapLabel})`;
|
|
918
|
+
if (localQueueHoldMs() > 0) {
|
|
919
|
+
pendingCandidates.set(candidate.order.run, candidate);
|
|
920
|
+
opts.out(`[${candidate.workflow}/${candidate.order.run}] lost a shared-capacity race at the ${limit} — queued for local dispatch`);
|
|
921
|
+
}
|
|
922
|
+
else {
|
|
923
|
+
releaseOrder(candidate.workflow, candidate.order, result === 'total-capacity' ? 'dispatch-cap-full' : 'agent-cap-full', `lost a shared-capacity race at the ${limit} — handing the claim back to the hub`);
|
|
924
|
+
}
|
|
733
925
|
}
|
|
734
926
|
}
|
|
735
927
|
return { dispatched, polled, openRuns, complete };
|
|
@@ -788,10 +980,33 @@ export function createShiftLoop(opts) {
|
|
|
788
980
|
async function iteration() {
|
|
789
981
|
const backoffActiveAtStart = monotonicNow() < backoffUntil;
|
|
790
982
|
let rateLimitedThisIteration = false;
|
|
983
|
+
// Hub roster cache refresh is low-priority operational work. Its absence is
|
|
984
|
+
// always a degraded read for children, never a reason to stop dispatch.
|
|
985
|
+
if (!backoffActiveAtStart &&
|
|
986
|
+
opts.syncRosters !== undefined &&
|
|
987
|
+
opts.rosterSyncIntervalMs !== undefined &&
|
|
988
|
+
monotonicNow() - lastRosterSync >= opts.rosterSyncIntervalMs) {
|
|
989
|
+
try {
|
|
990
|
+
await withHubRosterSyncTimeout((signal) => opts.syncRosters(signal), opts.rosterSyncTimeoutMs);
|
|
991
|
+
}
|
|
992
|
+
catch (e) {
|
|
993
|
+
rateLimitedThisIteration = noteServerBackoff(e);
|
|
994
|
+
opts.err(`roster sync failed: ${errMsg(e)} (continuing)`);
|
|
995
|
+
emit({ type: 'hub-error', op: 'roster_sync', message: `roster sync failed: ${errMsg(e)} (continuing)` });
|
|
996
|
+
}
|
|
997
|
+
finally {
|
|
998
|
+
// This is a periodic attempt cadence, not a success-only retry loop. A
|
|
999
|
+
// persistent non-429 failure must not make every 5s poll issue two more
|
|
1000
|
+
// hub calls and append another durable failure record.
|
|
1001
|
+
lastRosterSync = monotonicNow();
|
|
1002
|
+
refreshServeCapabilities();
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
791
1005
|
// Presence when due (starts immediately — this shift exists to conduct).
|
|
792
1006
|
// A server backoff suppresses every hub poll, including presence, while local
|
|
793
1007
|
// reconciliation and queued dispatch continue below.
|
|
794
1008
|
if (!backoffActiveAtStart &&
|
|
1009
|
+
!rateLimitedThisIteration &&
|
|
795
1010
|
monotonicNow() - lastPresence >= opts.presenceIntervalMs) {
|
|
796
1011
|
// `setShift` and `noteAttended` force the next ping by setting
|
|
797
1012
|
// lastPresence to -Infinity. Either can land WHILE this ping is awaiting
|
|
@@ -804,6 +1019,7 @@ export function createShiftLoop(opts) {
|
|
|
804
1019
|
await opts.hub.presencePing({
|
|
805
1020
|
name: shiftName,
|
|
806
1021
|
serve_crews: serveCrews,
|
|
1022
|
+
serve_capabilities: [...serveCapabilities],
|
|
807
1023
|
...(opts.shiftId !== undefined ? { shift_id: opts.shiftId } : {}),
|
|
808
1024
|
...(opts.startedAt !== undefined ? { started_at: opts.startedAt } : {}),
|
|
809
1025
|
...(attendedAt !== undefined ? { attended_at: attendedAt } : {}),
|
|
@@ -859,7 +1075,11 @@ export function createShiftLoop(opts) {
|
|
|
859
1075
|
// had no local capacity, or its sweep failed, the adopted cursor will be
|
|
860
1076
|
// unchanged next time. `sweepOwed` preserves that unconsumed work signal.
|
|
861
1077
|
let swept;
|
|
862
|
-
|
|
1078
|
+
const brakeDue = brakeSweepDueAt !== undefined && monotonicNow() >= brakeSweepDueAt;
|
|
1079
|
+
if (wakeSucceeded && (changed || sweepOwed || brakeDue) && k > 0) {
|
|
1080
|
+
// Clear before awaiting: a new brake armed during sweep must survive.
|
|
1081
|
+
if (brakeDue)
|
|
1082
|
+
brakeSweepDueAt = undefined;
|
|
863
1083
|
swept = await sweep(k, live, reserved);
|
|
864
1084
|
sweepOwed = !swept.complete;
|
|
865
1085
|
}
|
|
@@ -936,15 +1156,19 @@ export function createShiftLoop(opts) {
|
|
|
936
1156
|
return cap - current.live.length - current.reserved.length;
|
|
937
1157
|
},
|
|
938
1158
|
getCap: () => cap,
|
|
1159
|
+
agentCeiling: () => agentLane().ceiling,
|
|
939
1160
|
setCap: (next) => {
|
|
940
1161
|
cap = next;
|
|
941
1162
|
},
|
|
942
1163
|
getShift: () => ({ name: shiftName, serveCrews: [...serveCrews] }),
|
|
1164
|
+
getServeCapabilities: () => [...serveCapabilities],
|
|
943
1165
|
setShift: (next) => {
|
|
944
1166
|
if (next.name !== undefined)
|
|
945
1167
|
shiftName = next.name;
|
|
946
|
-
if (next.serveCrews !== undefined)
|
|
1168
|
+
if (next.serveCrews !== undefined) {
|
|
947
1169
|
serveCrews = [...next.serveCrews];
|
|
1170
|
+
refreshServeCapabilities();
|
|
1171
|
+
}
|
|
948
1172
|
lastPresence = Number.NEGATIVE_INFINITY; // D6: next iterate() pings immediately
|
|
949
1173
|
presenceGeneration++; // survive an in-flight ping completing after this
|
|
950
1174
|
return { name: shiftName, serveCrews: [...serveCrews] };
|
|
@@ -966,7 +1190,31 @@ export function createShiftLoop(opts) {
|
|
|
966
1190
|
runBrakeKey.delete(run);
|
|
967
1191
|
stepBrake.delete(brakeKey);
|
|
968
1192
|
}
|
|
969
|
-
|
|
1193
|
+
// The hub's closed-run report is authoritative but carries no record identity.
|
|
1194
|
+
removeRecordUnderDispatchLock(run);
|
|
1195
|
+
},
|
|
1196
|
+
noteChildExited: (exit) => {
|
|
1197
|
+
pendingCandidates.delete(exit.run);
|
|
1198
|
+
let removed = false;
|
|
1199
|
+
try {
|
|
1200
|
+
removed = removeRecordUnderDispatchLock(exit.run, { pid: exit.pid }, 1_000, 'owenloop Shift exit reaper');
|
|
1201
|
+
}
|
|
1202
|
+
catch (e) {
|
|
1203
|
+
const retry = e instanceof FileLockTimeoutError
|
|
1204
|
+
? ' (the pid reconciliation will retry)'
|
|
1205
|
+
: '';
|
|
1206
|
+
opts.err(`[${exit.workflow}/${exit.run}] failed to free the dispatch slot after the worker exited: ${errMsg(e)}${retry}`);
|
|
1207
|
+
return;
|
|
1208
|
+
}
|
|
1209
|
+
if (!removed)
|
|
1210
|
+
return;
|
|
1211
|
+
emit({
|
|
1212
|
+
type: 'reaped',
|
|
1213
|
+
workflow: exit.workflow,
|
|
1214
|
+
run: exit.run,
|
|
1215
|
+
kind: exit.kind,
|
|
1216
|
+
pid: exit.pid,
|
|
1217
|
+
});
|
|
970
1218
|
},
|
|
971
1219
|
noteWorkerFailure: (failure) => {
|
|
972
1220
|
const brakeKey = runBrakeKey.get(failure.run);
|
|
@@ -119,10 +119,10 @@ export interface CapacityEvent {
|
|
|
119
119
|
inFlight: number;
|
|
120
120
|
cap: number;
|
|
121
121
|
}
|
|
122
|
-
/** A hub call failed. `workflow` is present
|
|
122
|
+
/** A hub call failed. `workflow` is present for per-workflow whats_next and release calls. */
|
|
123
123
|
export interface HubErrorEvent {
|
|
124
124
|
type: 'hub-error';
|
|
125
|
-
op: 'wake' | 'whats_next';
|
|
125
|
+
op: 'wake' | 'whats_next' | 'roster_sync' | 'release';
|
|
126
126
|
workflow?: string;
|
|
127
127
|
message: string;
|
|
128
128
|
}
|
|
@@ -133,8 +133,9 @@ export interface BundleMissEvent {
|
|
|
133
133
|
def: string;
|
|
134
134
|
}
|
|
135
135
|
/**
|
|
136
|
-
* The shift refused one order and
|
|
137
|
-
*
|
|
136
|
+
* The shift refused one order. Capacity and expiry reasons hand its claim back
|
|
137
|
+
* to the hub; malformed and unsupported reasons leave it for the pickup
|
|
138
|
+
* window. A dropped unit of work is a record, not a debug aside. `reason` is
|
|
138
139
|
* the stable machine discriminator; `message` is the human text.
|
|
139
140
|
*/
|
|
140
141
|
export interface OrderDroppedEvent {
|
|
@@ -142,7 +143,7 @@ export interface OrderDroppedEvent {
|
|
|
142
143
|
workflow: string;
|
|
143
144
|
run: string;
|
|
144
145
|
step: string;
|
|
145
|
-
reason: 'malformed-digest' | 'malformed-worker' | 'unsupported-worker' | 'verification-failed' | 'metadata-unavailable';
|
|
146
|
+
reason: 'malformed-digest' | 'malformed-worker' | 'unsupported-worker' | 'verification-failed' | 'metadata-unavailable' | 'agent-lane-closed' | 'dispatch-cap-full' | 'agent-cap-full' | 'claim-expired';
|
|
146
147
|
message: string;
|
|
147
148
|
}
|
|
148
149
|
/**
|
|
@@ -205,6 +206,7 @@ export interface ShiftStatus {
|
|
|
205
206
|
cap: number;
|
|
206
207
|
free: number;
|
|
207
208
|
running: number;
|
|
209
|
+
agent_ceiling: number;
|
|
208
210
|
attended_at: number | null;
|
|
209
211
|
started_at: number;
|
|
210
212
|
}
|