owenloop 0.5.8 → 0.5.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +81 -39
- package/bin/owenloop.mjs +16 -1
- package/dist/packages/work/src/agent/loop.d.ts +23 -8
- package/dist/packages/work/src/agent/loop.js +55 -6
- package/dist/packages/work/src/bundle/pull.d.ts +21 -0
- package/dist/packages/work/src/bundle/pull.js +118 -0
- package/dist/packages/work/src/consumed-verifier.js +6 -17
- package/dist/packages/work/src/exec/instructions.d.ts +4 -1
- package/dist/packages/work/src/exec/instructions.js +3 -0
- package/dist/packages/work/src/exec/loop.d.ts +2 -2
- package/dist/packages/work/src/exec/loop.js +273 -46
- package/dist/packages/work/src/exec/receipt.d.ts +7 -7
- package/dist/packages/work/src/exec/receipt.js +6 -6
- package/dist/packages/work/src/harness/claude.js +116 -0
- package/dist/packages/work/src/harness/codex.d.ts +8 -0
- package/dist/packages/work/src/harness/codex.js +27 -1
- package/dist/packages/work/src/harness/contract.d.ts +9 -0
- package/dist/packages/work/src/hub/client.d.ts +19 -4
- package/dist/packages/work/src/hub/client.js +7 -3
- package/dist/packages/work/src/hub/types.d.ts +84 -0
- package/dist/packages/work/src/main.js +1 -0
- package/dist/packages/work/src/roles/agent-run.js +57 -32
- package/dist/packages/work/src/roles/exec.js +27 -8
- package/dist/packages/work/src/roles/settings.js +2 -0
- package/dist/packages/work/src/roles/shift.js +14 -2
- package/dist/packages/work/src/roles/util.d.ts +1 -0
- package/dist/packages/work/src/roles/util.js +10 -0
- package/dist/packages/work/src/settings/hub-roster-cache.d.ts +68 -0
- package/dist/packages/work/src/settings/hub-roster-cache.js +370 -0
- package/dist/packages/work/src/settings/roster.d.ts +84 -0
- package/dist/packages/work/src/settings/roster.js +364 -5
- package/dist/packages/work/src/settings/serving.d.ts +17 -0
- package/dist/packages/work/src/settings/serving.js +79 -0
- package/dist/packages/work/src/settings/settings.d.ts +14 -1
- package/dist/packages/work/src/settings/settings.js +14 -0
- package/dist/packages/work/src/shift/loop.d.ts +108 -5
- package/dist/packages/work/src/shift/loop.js +416 -54
- package/dist/packages/work/src/shift/protocol.d.ts +7 -5
- package/dist/packages/work/src/shift/runtime.d.ts +18 -0
- package/dist/packages/work/src/shift/runtime.js +104 -5
- package/dist/packages/work/src/shift/server.js +2 -0
- package/dist/packages/work/src/shift/spawn.d.ts +17 -2
- package/dist/packages/work/src/shift/spawn.js +16 -13
- package/dist/packages/work/src/shift/state.d.ts +29 -2
- package/dist/packages/work/src/shift/state.js +45 -15
- package/dist/packages/work/src/submit-proof.d.ts +11 -8
- package/dist/packages/work/src/submit-proof.js +11 -8
- package/dist/packages/work/src/usage.d.ts +1 -1
- package/dist/packages/work/src/usage.js +7 -1
- package/dist/packages/work/src/util/modifier-init.d.ts +7 -0
- package/dist/packages/work/src/util/modifier-init.js +73 -0
- package/dist/src/bundle/tar.d.ts +9 -4
- package/dist/src/bundle/tar.js +9 -4
- package/dist/src/capabilities.d.ts +76 -6
- package/dist/src/capabilities.js +49 -6
- package/dist/src/capability-mapping-client.d.ts +29 -0
- package/dist/src/capability-mapping-client.js +160 -0
- package/dist/src/cli.d.ts +16 -3
- package/dist/src/cli.js +1925 -281
- package/dist/src/config-dir.d.ts +1 -1
- package/dist/src/config-dir.js +1 -1
- package/dist/src/crypto/index.d.ts +1 -1
- package/dist/src/crypto/index.js +1 -1
- package/dist/src/crypto/org-root.d.ts +30 -12
- package/dist/src/crypto/org-root.js +162 -14
- package/dist/src/crypto/records.d.ts +2 -0
- package/dist/src/crypto/records.js +2 -0
- package/dist/src/defs.d.ts +38 -2
- package/dist/src/defs.js +593 -86
- package/dist/src/engine.d.ts +103 -25
- package/dist/src/engine.js +331 -79
- package/dist/src/hub.d.ts +77 -0
- package/dist/src/hub.js +162 -0
- package/dist/src/implements.d.ts +56 -0
- package/dist/src/implements.js +790 -0
- package/dist/src/index.d.ts +11 -9
- package/dist/src/index.js +6 -5
- package/dist/src/mcp/serve.d.ts +22 -0
- package/dist/src/mcp/serve.js +330 -41
- package/dist/src/mcp/server.d.ts +10 -7
- package/dist/src/mcp/server.js +53 -8
- package/dist/src/model.d.ts +48 -10
- package/dist/src/model.js +252 -99
- package/dist/src/order-resolver.d.ts +1 -0
- package/dist/src/order-resolver.js +8 -1
- package/dist/src/schemas/index.d.ts +11 -0
- package/dist/src/schemas/order.v1.schema.json +7 -1
- package/dist/src/store/def-source.d.ts +19 -0
- package/dist/src/store/def-source.js +87 -17
- package/dist/src/store/gc-recovery.d.ts +46 -0
- package/dist/src/store/gc-recovery.js +258 -0
- package/dist/src/store/gc.d.ts +94 -0
- package/dist/src/store/gc.js +562 -0
- package/dist/src/store/index.d.ts +3 -1
- package/dist/src/store/index.js +2 -0
- package/dist/src/store/install.d.ts +34 -5
- package/dist/src/store/install.js +73 -11
- package/dist/src/store/pre-commit-verifier.js +42 -2
- package/dist/src/store/snapshot-guard.d.ts +18 -0
- package/dist/src/store/snapshot-guard.js +94 -0
- package/dist/src/store.d.ts +69 -4
- package/dist/src/store.js +254 -16
- package/dist/src/types.d.ts +101 -20
- package/dist/src/util.js +4 -1
- package/docs/authoring.md +235 -10
- package/docs/bundles.md +45 -2
- package/docs/cli.md +437 -55
- package/docs/crypto.md +32 -10
- package/docs/design.md +177 -47
- package/docs/evals/mcp-charter-baseline.json +428 -0
- package/docs/mcp-charter-eval.md +124 -0
- package/docs/routing.md +189 -34
- package/docs/shift-logs.md +38 -9
- package/docs/wire-contracts.md +78 -14
- package/examples/workflows/compile-dev-playbook.yaml +1 -1
- package/examples/workflows/delivery.yaml +31 -14
- package/examples/workflows/ship.yaml +38 -4
- package/package.json +2 -1
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/plugin/.mcp.json +1 -1
- package/plugins/claude-code/plugin/skills/author/SKILL.md +22 -0
- package/plugins/claude-code/plugin/skills/ephemeral/SKILL.md +95 -0
- package/plugins/claude-code/plugin/skills/graduate/SKILL.md +156 -0
- package/plugins/claude-code/plugin/skills/plan/SKILL.md +236 -0
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
- package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
- package/plugins/codex/plugins/owenloop/skills/author/SKILL.md +22 -0
- package/plugins/codex/plugins/owenloop/skills/ephemeral/SKILL.md +95 -0
- package/plugins/codex/plugins/owenloop/skills/graduate/SKILL.md +156 -0
- package/plugins/codex/plugins/owenloop/skills/plan/SKILL.md +236 -0
|
@@ -27,9 +27,11 @@
|
|
|
27
27
|
* liveInFlight counts every live child record, each one pid-probed (see
|
|
28
28
|
* state.ts). When `k <= 0` the loop skips `whats_next` but remembers that the
|
|
29
29
|
* adopted wake still needs a sweep. Orders already returned by `whats_next`
|
|
30
|
-
* are already claimed
|
|
31
|
-
*
|
|
32
|
-
*
|
|
30
|
+
* are already claimed. Orders this shift cannot dispatch are released back to
|
|
31
|
+
* the hub immediately, so another daemon can take them; `localQueueHoldMs`
|
|
32
|
+
* optionally retains them locally for a bounded window instead. The agent lane
|
|
33
|
+
* is additionally bounded by `cap − execReserve`, preserving room for an exec
|
|
34
|
+
* order while agents are saturated.
|
|
33
35
|
*
|
|
34
36
|
* RESILIENCE: presence/wake/whats_next failures are logged and never kill the
|
|
35
37
|
* loop. `stop()` flips a flag checked between awaits; the in-flight sweep
|
|
@@ -53,21 +55,73 @@
|
|
|
53
55
|
*/
|
|
54
56
|
import { join } from 'node:path';
|
|
55
57
|
import { performance } from 'node:perf_hooks';
|
|
56
|
-
import { acquireFileLockSync, releaseFileLock } from "../../../../src/lock.js";
|
|
58
|
+
import { acquireFileLockSync, FileLockTimeoutError, releaseFileLock, } from "../../../../src/lock.js";
|
|
57
59
|
import { readDispatchBundle } from "../bundle/cache.js";
|
|
58
60
|
import { HubError } from "../hub/types.js";
|
|
59
61
|
import { isCommandStep, resolveCommandRouting } from "./routing.js";
|
|
60
|
-
import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reserveChild, startReservedChild, removeChildRecord, ensureStateDir, } from "./state.js";
|
|
62
|
+
import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reserveChild, startReservedChild, removeChildRecord, settleChildGate, ensureStateDir, } from "./state.js";
|
|
61
63
|
import { DEFAULT_WORK_DIR_TTL_MS, sweepWorkDirs as sweepWorkDirsImpl } from "../agent/workdir.js";
|
|
64
|
+
import { withHubRosterSyncTimeout } from "../settings/hub-roster-cache.js";
|
|
62
65
|
import { sessionsPath } from "../harness/session-store.js";
|
|
63
66
|
import { stampShiftEvent, } from "./protocol.js";
|
|
67
|
+
/** Run a state mutation while holding the lock that serializes dispatch state. */
|
|
68
|
+
export function withDispatchLock(stateDir, options = {}, fn) {
|
|
69
|
+
const dispatchLock = acquireFileLockSync(join(stateDir, '.dispatch.lock'), {
|
|
70
|
+
...options.dispatchLockOptions,
|
|
71
|
+
waitMs: options.waitMs ?? 30_000,
|
|
72
|
+
label: options.label ?? 'owenloop Shift dispatch-state removal',
|
|
73
|
+
});
|
|
74
|
+
try {
|
|
75
|
+
return fn();
|
|
76
|
+
}
|
|
77
|
+
finally {
|
|
78
|
+
releaseFileLock(dispatchLock);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Remove one child record while holding the same lock that serializes durable
|
|
83
|
+
* reservation creation. The record is deliberately re-read by
|
|
84
|
+
* `removeChildRecord` only after acquisition, so a stale observer cannot erase
|
|
85
|
+
* a newer dispatch for the same run.
|
|
86
|
+
*/
|
|
87
|
+
export function removeChildRecordUnderDispatchLock(stateDir, run, recordOptions, options = {}) {
|
|
88
|
+
return withDispatchLock(stateDir, options, () => removeChildRecord(stateDir, run, recordOptions));
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Reconciliation callbacks for every unlocked Shift daemon path. Keep these
|
|
92
|
+
* together with the lock-held removal primitive so startup and the poll loop
|
|
93
|
+
* cannot silently diverge.
|
|
94
|
+
*/
|
|
95
|
+
export function createLockedRemovalCallbacks(stateDir, options = {}) {
|
|
96
|
+
return {
|
|
97
|
+
removeAbandonedReservation: (reservation) => withDispatchLock(stateDir, options, () => cancelReservedChild(stateDir, reservation)),
|
|
98
|
+
removeDeadChild: (record) => withDispatchLock(stateDir, options, () => removeChildRecord(stateDir, record.run, { pid: record.pid })),
|
|
99
|
+
settleLiveChild: (record) => withDispatchLock(stateDir, options, () => settleChildGate(stateDir, record)),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
64
102
|
/** The `maxConcurrentAgents` fallback when the option is absent. */
|
|
65
103
|
const DEFAULT_MAX_AGENTS = 4;
|
|
104
|
+
/** The `execReserve` fallback when the option is absent. */
|
|
105
|
+
const DEFAULT_EXEC_RESERVE = 1;
|
|
106
|
+
/** The `localQueueHoldMs` fallback when the option is absent. */
|
|
107
|
+
const DEFAULT_LOCAL_QUEUE_HOLD_MS = 0;
|
|
108
|
+
/** The hub reaps a never-contacted claim after this long. Every client-side
|
|
109
|
+
* re-offer latency must stay strictly under it. */
|
|
110
|
+
export const HUB_PICKUP_WINDOW_MS = 120_000;
|
|
111
|
+
/** Headroom for a detached child to make first contact after dispatch. */
|
|
112
|
+
const CHILD_FIRST_CONTACT_MARGIN_MS = 30_000;
|
|
66
113
|
/**
|
|
67
114
|
* The hub reaps a never-contacted claim after 120 seconds. Stop local queueing
|
|
68
115
|
* after 90 seconds so a detached child retains 30 seconds to make first contact.
|
|
69
116
|
*/
|
|
70
|
-
export const MAX_PENDING_CANDIDATE_AGE_MS =
|
|
117
|
+
export const MAX_PENDING_CANDIDATE_AGE_MS = HUB_PICKUP_WINDOW_MS - CHILD_FIRST_CONTACT_MARGIN_MS;
|
|
118
|
+
/**
|
|
119
|
+
* A capacity-refused claim is returned to the hub immediately, so sibling
|
|
120
|
+
* shifts can take it. This short, fixed monotonic window stops this shift from
|
|
121
|
+
* claiming that same stable step again on every busy wake while its relevant
|
|
122
|
+
* capacity is still unavailable.
|
|
123
|
+
*/
|
|
124
|
+
const CAPACITY_RELEASE_COOLDOWN_MS = 30_000;
|
|
71
125
|
/**
|
|
72
126
|
* THE PER-STEP FAILURE BRAKE.
|
|
73
127
|
*
|
|
@@ -101,11 +155,11 @@ export const MAX_PENDING_CANDIDATE_AGE_MS = 90_000;
|
|
|
101
155
|
* It is a rate limit, never a ban. A step that has not failed is dispatched
|
|
102
156
|
* with zero delay, however many orders arrive for it at once.
|
|
103
157
|
*/
|
|
104
|
-
const STEP_BRAKE_DELAYS_MS = [2_000, 8_000, 30_000,
|
|
158
|
+
export const STEP_BRAKE_DELAYS_MS = [2_000, 8_000, 30_000, 60_000, 90_000];
|
|
105
159
|
/**
|
|
106
160
|
* Treat a failure this long after the previous one as the start of a NEW
|
|
107
161
|
* streak, resetting the count to zero first. Comfortably longer than the
|
|
108
|
-
* longest delay in the table (
|
|
162
|
+
* longest delay in the table (90s), so a step failing at maximum backoff never
|
|
109
163
|
* decays out of it, while an isolated transient long after an old streak starts
|
|
110
164
|
* again at the shortest delay instead of inheriting a stale penalty.
|
|
111
165
|
*/
|
|
@@ -121,7 +175,7 @@ const STEP_BRAKE_FORGET_MS = 1_800_000;
|
|
|
121
175
|
* worker failures. `count` is at least 1 at every call site (this is reached
|
|
122
176
|
* only from the failure path), so the FIRST failure already costs 2s; past the
|
|
123
177
|
* end of the table the last entry repeats, so a permanently-failing step
|
|
124
|
-
* settles at one dispatch per
|
|
178
|
+
* settles at one dispatch per 90 seconds instead of one per poll.
|
|
125
179
|
*/
|
|
126
180
|
function stepBrakeDelayMs(count) {
|
|
127
181
|
const index = Math.min(count, STEP_BRAKE_DELAYS_MS.length) - 1;
|
|
@@ -146,6 +200,23 @@ function isNonServableRace(e) {
|
|
|
146
200
|
export function createShiftLoop(opts) {
|
|
147
201
|
let stopped = false;
|
|
148
202
|
let cap = opts.cap;
|
|
203
|
+
/**
|
|
204
|
+
* The agent lane's live budget. Recomputed per call because `setCap`
|
|
205
|
+
* rewrites `cap` at runtime.
|
|
206
|
+
*/
|
|
207
|
+
function agentLane() {
|
|
208
|
+
const requested = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
|
|
209
|
+
const reserve = Math.min(opts.execReserve ?? DEFAULT_EXEC_RESERVE, Math.max(0, cap - 1));
|
|
210
|
+
return {
|
|
211
|
+
ceiling: Math.max(0, Math.min(requested, cap - reserve)),
|
|
212
|
+
requested,
|
|
213
|
+
reserve,
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
/** Effective local hold, capped below the hub pickup window. */
|
|
217
|
+
function localQueueHoldMs() {
|
|
218
|
+
return Math.min(Math.max(0, opts.localQueueHoldMs ?? DEFAULT_LOCAL_QUEUE_HOLD_MS), MAX_PENDING_CANDIDATE_AGE_MS);
|
|
219
|
+
}
|
|
149
220
|
const isAlive = opts.isAlive;
|
|
150
221
|
const monotonicNow = opts.monotonicNow ?? performance.now.bind(performance);
|
|
151
222
|
// Live shift identity (MCP `clock_in`, D3-D7 of the plan). Seeded from opts,
|
|
@@ -153,9 +224,32 @@ export function createShiftLoop(opts) {
|
|
|
153
224
|
// nor a caller of getShift/setShift can mutate the other's state afterward.
|
|
154
225
|
let shiftName = opts.name;
|
|
155
226
|
let serveCrews = [...opts.serveCrews];
|
|
227
|
+
let serveCapabilities = [];
|
|
228
|
+
let initial = true;
|
|
229
|
+
function refreshServeCapabilities() {
|
|
230
|
+
if (opts.computeServeCapabilities === undefined)
|
|
231
|
+
return;
|
|
232
|
+
let next;
|
|
233
|
+
try {
|
|
234
|
+
next = [...opts.computeServeCapabilities(serveCrews)];
|
|
235
|
+
}
|
|
236
|
+
catch {
|
|
237
|
+
return; // Keep the last known-good advertisement on a transient failure.
|
|
238
|
+
}
|
|
239
|
+
const changed = next.length !== serveCapabilities.length ||
|
|
240
|
+
next.some((capability, index) => capability !== serveCapabilities[index]);
|
|
241
|
+
serveCapabilities = next;
|
|
242
|
+
if (changed && !initial)
|
|
243
|
+
opts.out(`serving ${next.length === 0 ? '(nothing)' : next.join(', ')}`);
|
|
244
|
+
}
|
|
245
|
+
refreshServeCapabilities();
|
|
246
|
+
initial = false;
|
|
156
247
|
// Park state persists across `iterate()` calls (the MCP park reuses it).
|
|
157
248
|
let cursor;
|
|
158
249
|
let lastPresence = Number.NEGATIVE_INFINITY;
|
|
250
|
+
// Runtime performs the required shift-start refresh before creating this
|
|
251
|
+
// loop. Start this cadence now so it is not immediately fetched twice.
|
|
252
|
+
let lastRosterSync = monotonicNow();
|
|
159
253
|
let attendedAt;
|
|
160
254
|
/**
|
|
161
255
|
* Orders returned by `whats_next` are already claimed. If the hub returns more
|
|
@@ -168,6 +262,18 @@ export function createShiftLoop(opts) {
|
|
|
168
262
|
/** A changed wake whose sweep was skipped or failed must be retried after the
|
|
169
263
|
* cursor is adopted; otherwise the next unchanged wake hides that work. */
|
|
170
264
|
let sweepOwed = false;
|
|
265
|
+
/** Earliest monotonic time at which a braked step becomes dispatchable again.
|
|
266
|
+
* Drives a sweep without a hub cursor move, so a brake never waits forever. */
|
|
267
|
+
let brakeSweepDueAt;
|
|
268
|
+
/**
|
|
269
|
+
* Recently capacity-released stable orders. This is deliberately separate
|
|
270
|
+
* from `stepBrake`: a worker failure leaves its claim to lapse, while a
|
|
271
|
+
* capacity refusal hands the claim back to the hub and needs a reason-aware
|
|
272
|
+
* targeted-poll damper.
|
|
273
|
+
*/
|
|
274
|
+
const capacityCooldown = new Map();
|
|
275
|
+
/** Earliest fixed cooldown deadline, used to re-admit an unchanged wake. */
|
|
276
|
+
let capacityCooldownDueAt;
|
|
171
277
|
/**
|
|
172
278
|
* Has the current at-capacity EPISODE already produced a `capacity` event?
|
|
173
279
|
*
|
|
@@ -182,6 +288,73 @@ export function createShiftLoop(opts) {
|
|
|
182
288
|
/** Bumped whenever a ping starts or is forced due, so a ping that completes
|
|
183
289
|
* after a force does not stamp the cadence over that force. */
|
|
184
290
|
let presenceGeneration = 0;
|
|
291
|
+
function refreshCapacityCooldownDueAt() {
|
|
292
|
+
let earliest;
|
|
293
|
+
for (const entry of capacityCooldown.values()) {
|
|
294
|
+
earliest = earliest === undefined ? entry.retryAt : Math.min(earliest, entry.retryAt);
|
|
295
|
+
}
|
|
296
|
+
capacityCooldownDueAt = earliest;
|
|
297
|
+
}
|
|
298
|
+
/** Remove all expired entries; a real post-expiry retry can arm a new window. */
|
|
299
|
+
function pruneExpiredCapacityCooldown(now = monotonicNow()) {
|
|
300
|
+
let changed = false;
|
|
301
|
+
for (const [key, entry] of capacityCooldown) {
|
|
302
|
+
if (entry.retryAt <= now) {
|
|
303
|
+
capacityCooldown.delete(key);
|
|
304
|
+
changed = true;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
if (changed)
|
|
308
|
+
refreshCapacityCooldownDueAt();
|
|
309
|
+
}
|
|
310
|
+
function armCapacityCooldown(workflow, order, reason) {
|
|
311
|
+
const now = monotonicNow();
|
|
312
|
+
pruneExpiredCapacityCooldown(now);
|
|
313
|
+
const key = stepBrakeKey(workflow, order.step, order.key);
|
|
314
|
+
// Do not let cursor activity or duplicate observations slide an active
|
|
315
|
+
// deadline. Only a genuine retry after expiry creates a fresh window.
|
|
316
|
+
if (capacityCooldown.has(key))
|
|
317
|
+
return;
|
|
318
|
+
capacityCooldown.set(key, { workflow, reason, retryAt: now + CAPACITY_RELEASE_COOLDOWN_MS });
|
|
319
|
+
refreshCapacityCooldownDueAt();
|
|
320
|
+
}
|
|
321
|
+
/** Read-only admission signal from the freshly reconciled capacity snapshot. */
|
|
322
|
+
function capacityCooldownReady(remaining, agentRoom, now = monotonicNow()) {
|
|
323
|
+
for (const entry of capacityCooldown.values()) {
|
|
324
|
+
if (entry.retryAt <= now)
|
|
325
|
+
continue;
|
|
326
|
+
if (entry.reason === 'dispatch-cap-full' ? remaining > 0 : agentRoom > 0)
|
|
327
|
+
return true;
|
|
328
|
+
}
|
|
329
|
+
return false;
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* Remove entries for `workflow` whose own capacity class is now eligible and
|
|
333
|
+
* report whether any active entry still suppresses its targeted request.
|
|
334
|
+
* Expiry is global housekeeping so workflows which disappear from the inbox
|
|
335
|
+
* cannot pin this private map forever.
|
|
336
|
+
*/
|
|
337
|
+
function workflowCapacityCooldownActive(workflow, remaining, agentRoom) {
|
|
338
|
+
const now = monotonicNow();
|
|
339
|
+
pruneExpiredCapacityCooldown(now);
|
|
340
|
+
let changed = false;
|
|
341
|
+
let active = false;
|
|
342
|
+
for (const [key, entry] of capacityCooldown) {
|
|
343
|
+
if (entry.workflow !== workflow)
|
|
344
|
+
continue;
|
|
345
|
+
const ready = entry.reason === 'dispatch-cap-full' ? remaining > 0 : agentRoom > 0;
|
|
346
|
+
if (ready) {
|
|
347
|
+
capacityCooldown.delete(key);
|
|
348
|
+
changed = true;
|
|
349
|
+
}
|
|
350
|
+
else {
|
|
351
|
+
active = true;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
if (changed)
|
|
355
|
+
refreshCapacityCooldownDueAt();
|
|
356
|
+
return active;
|
|
357
|
+
}
|
|
185
358
|
function noteServerBackoff(error) {
|
|
186
359
|
if (!(error instanceof HubError) || error.status !== 429)
|
|
187
360
|
return false;
|
|
@@ -213,10 +386,41 @@ export function createShiftLoop(opts) {
|
|
|
213
386
|
opts.err(`shift event sink failed: ${errMsg(e)} (continuing)`);
|
|
214
387
|
}
|
|
215
388
|
}
|
|
389
|
+
/**
|
|
390
|
+
* Hand a claim back so another shift can be offered it immediately. This
|
|
391
|
+
* deliberately stays off the dispatch critical path; a failed release falls
|
|
392
|
+
* back to the existing pickup-window behavior and is observable.
|
|
393
|
+
*/
|
|
394
|
+
function releaseClaim(workflow, run) {
|
|
395
|
+
void opts.hub.release({ workflow, run }).catch((e) => {
|
|
396
|
+
noteServerBackoff(e);
|
|
397
|
+
const message = errMsg(e);
|
|
398
|
+
opts.err(`[${workflow}/${run}] release failed: ${message} — leaving the hub pickup window to re-offer it`);
|
|
399
|
+
emit({ type: 'hub-error', op: 'release', workflow, message });
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
function acquireDispatchLock(waitMs, label) {
|
|
403
|
+
return acquireFileLockSync(join(opts.stateDir, '.dispatch.lock'), {
|
|
404
|
+
...opts.dispatchLockOptions,
|
|
405
|
+
waitMs,
|
|
406
|
+
label,
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
const lockedRemovalCallbacks = createLockedRemovalCallbacks(opts.stateDir, {
|
|
410
|
+
dispatchLockOptions: opts.dispatchLockOptions,
|
|
411
|
+
});
|
|
412
|
+
function removeRecordUnderDispatchLock(run, recordOptions, waitMs = 30_000, label = 'owenloop Shift dispatch-state removal') {
|
|
413
|
+
return removeChildRecordUnderDispatchLock(opts.stateDir, run, recordOptions, {
|
|
414
|
+
dispatchLockOptions: opts.dispatchLockOptions,
|
|
415
|
+
waitMs,
|
|
416
|
+
label,
|
|
417
|
+
});
|
|
418
|
+
}
|
|
216
419
|
function reconcile() {
|
|
217
420
|
const result = reconcileInFlight(opts.stateDir, {
|
|
218
421
|
...(isAlive !== undefined ? { isAlive } : {}),
|
|
219
422
|
now: opts.now(),
|
|
423
|
+
...lockedRemovalCallbacks,
|
|
220
424
|
});
|
|
221
425
|
for (const rec of result.reaped) {
|
|
222
426
|
emit({
|
|
@@ -263,10 +467,7 @@ export function createShiftLoop(opts) {
|
|
|
263
467
|
*/
|
|
264
468
|
function reserveCandidate(c) {
|
|
265
469
|
const childKind = c.kind === 'command' ? 'exec' : 'agent-run';
|
|
266
|
-
const dispatchLock =
|
|
267
|
-
waitMs: 30_000,
|
|
268
|
-
label: 'owenloop Shift dispatch',
|
|
269
|
-
});
|
|
470
|
+
const dispatchLock = acquireDispatchLock(30_000, 'owenloop Shift dispatch');
|
|
270
471
|
try {
|
|
271
472
|
const fresh = reconcileInFlight(opts.stateDir, {
|
|
272
473
|
...(isAlive === undefined ? {} : { isAlive }),
|
|
@@ -280,7 +481,7 @@ export function createShiftLoop(opts) {
|
|
|
280
481
|
if (childKind === 'agent-run') {
|
|
281
482
|
const agents = fresh.live.filter((record) => record.kind === 'agent-run').length +
|
|
282
483
|
fresh.reserved.filter((record) => record.childKind === 'agent-run').length;
|
|
283
|
-
if (agents >= (
|
|
484
|
+
if (agents >= agentLane().ceiling)
|
|
284
485
|
return 'agent-capacity';
|
|
285
486
|
}
|
|
286
487
|
return reserveChild(opts.stateDir, {
|
|
@@ -312,6 +513,9 @@ export function createShiftLoop(opts) {
|
|
|
312
513
|
opts.err(`[${c.workflow}/${c.order.run}] step '${c.order.step}' has failed ` +
|
|
313
514
|
`${brake.count} time(s) in a row — braking for ${waitMs}ms and ` +
|
|
314
515
|
`leaving this claim to lapse`);
|
|
516
|
+
brakeSweepDueAt = brakeSweepDueAt === undefined
|
|
517
|
+
? brake.nextAllowedAt
|
|
518
|
+
: Math.min(brakeSweepDueAt, brake.nextAllowedAt);
|
|
315
519
|
return 'braked';
|
|
316
520
|
}
|
|
317
521
|
let reservation;
|
|
@@ -331,15 +535,22 @@ export function createShiftLoop(opts) {
|
|
|
331
535
|
startGate: reserved.gatePath,
|
|
332
536
|
});
|
|
333
537
|
cancel = spawned.cancel ?? spawned.terminate;
|
|
334
|
-
const
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
538
|
+
const dispatchLock = acquireDispatchLock(30_000, 'owenloop Shift dispatch');
|
|
539
|
+
let rec;
|
|
540
|
+
try {
|
|
541
|
+
rec = finalizeChildReservation(opts.stateDir, reservation, {
|
|
542
|
+
pid: spawned.pid,
|
|
543
|
+
spawnedAt: opts.now(),
|
|
544
|
+
kind: childKind,
|
|
545
|
+
...(childKind === 'agent-run' && c.defName !== undefined ? { def: c.defName } : {}),
|
|
546
|
+
...(childKind === 'agent-run' && c.defHash !== undefined ? { hash: c.defHash } : {}),
|
|
547
|
+
...(childKind === 'agent-run' ? { step: c.order.step } : {}),
|
|
548
|
+
});
|
|
549
|
+
startReservedChild(opts.stateDir, rec);
|
|
550
|
+
}
|
|
551
|
+
finally {
|
|
552
|
+
releaseFileLock(dispatchLock);
|
|
553
|
+
}
|
|
343
554
|
// Remember which step this run belongs to, so a later worker failure —
|
|
344
555
|
// which names the run but not the fan-out key — can be charged to the
|
|
345
556
|
// right brake key. NOTHING is counted here: a dispatch is not evidence of
|
|
@@ -362,7 +573,13 @@ export function createShiftLoop(opts) {
|
|
|
362
573
|
cancel?.();
|
|
363
574
|
if (reservation !== undefined) {
|
|
364
575
|
try {
|
|
365
|
-
|
|
576
|
+
const dispatchLock = acquireDispatchLock(30_000, 'owenloop Shift dispatch');
|
|
577
|
+
try {
|
|
578
|
+
cancelReservedChild(opts.stateDir, reservation);
|
|
579
|
+
}
|
|
580
|
+
finally {
|
|
581
|
+
releaseFileLock(dispatchLock);
|
|
582
|
+
}
|
|
366
583
|
}
|
|
367
584
|
catch (cleanupError) {
|
|
368
585
|
opts.err(`[${c.workflow}/${c.order.run}] failed to cancel dispatch reservation: ${errMsg(cleanupError)}`);
|
|
@@ -383,26 +600,35 @@ export function createShiftLoop(opts) {
|
|
|
383
600
|
}
|
|
384
601
|
}
|
|
385
602
|
function discardExpiredCandidate(candidate, queued) {
|
|
386
|
-
|
|
603
|
+
const holdMs = queued ? localQueueHoldMs() : MAX_PENDING_CANDIDATE_AGE_MS;
|
|
604
|
+
if (monotonicNow() - candidate.requestStartedAt < holdMs)
|
|
387
605
|
return false;
|
|
388
606
|
const prefix = queued ? 'queued claim' : 'claim';
|
|
389
|
-
|
|
607
|
+
const message = `${prefix} expired before local dispatch — handing the claim back to the hub`;
|
|
608
|
+
opts.err(`[${candidate.workflow}/${candidate.order.run}] ${message}`);
|
|
609
|
+
emit({
|
|
610
|
+
type: 'order-dropped',
|
|
611
|
+
workflow: candidate.workflow,
|
|
612
|
+
run: candidate.order.run,
|
|
613
|
+
step: candidate.order.step,
|
|
614
|
+
reason: 'claim-expired',
|
|
615
|
+
message,
|
|
616
|
+
});
|
|
617
|
+
releaseClaim(candidate.workflow, candidate.order.run);
|
|
390
618
|
return true;
|
|
391
619
|
}
|
|
392
620
|
/** Dispatch already-claimed orders when local child capacity becomes free. */
|
|
393
621
|
function drainPending(live, reserved) {
|
|
394
622
|
let remaining = cap - live.length - reserved.length;
|
|
395
|
-
if (
|
|
623
|
+
if (pendingCandidates.size === 0)
|
|
396
624
|
return 0;
|
|
397
|
-
const
|
|
398
|
-
let agentRoom =
|
|
625
|
+
const { ceiling } = agentLane();
|
|
626
|
+
let agentRoom = ceiling -
|
|
399
627
|
live.filter((r) => r.kind === 'agent-run').length -
|
|
400
628
|
reserved.filter((r) => r.childKind === 'agent-run').length;
|
|
401
629
|
const liveRuns = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
|
|
402
630
|
let dispatched = 0;
|
|
403
631
|
for (const [run, candidate] of pendingCandidates) {
|
|
404
|
-
if (remaining <= 0)
|
|
405
|
-
break;
|
|
406
632
|
if (discardExpiredCandidate(candidate, true)) {
|
|
407
633
|
pendingCandidates.delete(run);
|
|
408
634
|
continue;
|
|
@@ -411,6 +637,10 @@ export function createShiftLoop(opts) {
|
|
|
411
637
|
pendingCandidates.delete(run);
|
|
412
638
|
continue;
|
|
413
639
|
}
|
|
640
|
+
// Even at full capacity, keep scanning so a configured local hold is a
|
|
641
|
+
// real deadline rather than one deferred until another child exits.
|
|
642
|
+
if (remaining <= 0)
|
|
643
|
+
continue;
|
|
414
644
|
// The agent lane's second budget can be full while the dispatch cap still
|
|
415
645
|
// has room. Skip this entry and KEEP it queued (a command entry further
|
|
416
646
|
// down the map is still dispatchable) — `continue`, never `break`, so one
|
|
@@ -418,10 +648,14 @@ export function createShiftLoop(opts) {
|
|
|
418
648
|
if (candidate.kind === 'agent' && agentRoom <= 0)
|
|
419
649
|
continue;
|
|
420
650
|
const result = dispatchCandidate(candidate);
|
|
421
|
-
if (result === 'total-capacity')
|
|
422
|
-
|
|
423
|
-
if (result === 'agent-capacity')
|
|
651
|
+
if (result === 'total-capacity') {
|
|
652
|
+
remaining = 0;
|
|
424
653
|
continue;
|
|
654
|
+
}
|
|
655
|
+
if (result === 'agent-capacity') {
|
|
656
|
+
agentRoom = 0;
|
|
657
|
+
continue;
|
|
658
|
+
}
|
|
425
659
|
pendingCandidates.delete(run);
|
|
426
660
|
if (result !== 'dispatched')
|
|
427
661
|
continue;
|
|
@@ -464,6 +698,7 @@ export function createShiftLoop(opts) {
|
|
|
464
698
|
if (!liveOrReserved.has(run))
|
|
465
699
|
runBrakeKey.delete(run);
|
|
466
700
|
}
|
|
701
|
+
pruneExpiredCapacityCooldown();
|
|
467
702
|
/**
|
|
468
703
|
* PHASE 4, for the work-dir reaper only. `polled` is the set of workflows
|
|
469
704
|
* whose `whats_next` SUCCEEDED this sweep — a workflow whose call threw is
|
|
@@ -477,13 +712,17 @@ export function createShiftLoop(opts) {
|
|
|
477
712
|
const polled = new Set();
|
|
478
713
|
const openRuns = new Set();
|
|
479
714
|
let complete = true;
|
|
715
|
+
let releasedForCapacity = false;
|
|
480
716
|
let remaining = budget;
|
|
481
717
|
const liveRuns = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
|
|
482
718
|
// The agent lane's SECOND budget (D7). Counted from live records and durable
|
|
483
719
|
// reservations, so a crash between reservation and spawn cannot overbook it.
|
|
484
720
|
// A child or reservation that is reaped frees the slot on the next reconcile.
|
|
485
|
-
const
|
|
486
|
-
|
|
721
|
+
const { ceiling, requested, reserve } = agentLane();
|
|
722
|
+
const agentCapLabel = reserve > 0 && ceiling < requested
|
|
723
|
+
? `${ceiling}, ${requested} requested minus a ${reserve}-slot exec reserve`
|
|
724
|
+
: `${ceiling}`;
|
|
725
|
+
let agentRoom = ceiling -
|
|
487
726
|
live.filter((r) => r.kind === 'agent-run').length -
|
|
488
727
|
reserved.filter((r) => r.childKind === 'agent-run').length;
|
|
489
728
|
/**
|
|
@@ -504,7 +743,7 @@ export function createShiftLoop(opts) {
|
|
|
504
743
|
}
|
|
505
744
|
else {
|
|
506
745
|
try {
|
|
507
|
-
const inbox = await opts.hub.whatsNext();
|
|
746
|
+
const inbox = await opts.hub.whatsNext({ serve_capabilities: [...serveCapabilities] });
|
|
508
747
|
instances = (inbox.instances ?? []).map((i) => i.workflow);
|
|
509
748
|
}
|
|
510
749
|
catch (e) {
|
|
@@ -518,14 +757,15 @@ export function createShiftLoop(opts) {
|
|
|
518
757
|
// that omission is what distinguishes the untargeted inbox call from a
|
|
519
758
|
// per-workflow one, since `HubErrorEvent.workflow` is optional.
|
|
520
759
|
emit({ type: 'hub-error', op: 'whats_next', message: errMsg(e) });
|
|
521
|
-
return { dispatched, polled, openRuns, complete: false };
|
|
760
|
+
return { dispatched, polled, openRuns, complete: false, releasedForCapacity };
|
|
522
761
|
}
|
|
523
762
|
}
|
|
524
763
|
// Collect candidates, metering NEW dispatches and deduping in-flight.
|
|
525
764
|
const claimed = new Set();
|
|
526
765
|
const candidates = [];
|
|
527
766
|
/**
|
|
528
|
-
* Refuse one order
|
|
767
|
+
* Refuse one order. Some reasons intentionally leave the claim for the hub
|
|
768
|
+
* pickup window; capacity reasons use `releaseOrder` below instead.
|
|
529
769
|
*
|
|
530
770
|
* Every refusal below is a DROPPED UNIT OF WORK, not a debug aside: the
|
|
531
771
|
* shift declines an order the hub already handed it, and until now the only
|
|
@@ -539,11 +779,39 @@ export function createShiftLoop(opts) {
|
|
|
539
779
|
emit({ type: 'order-dropped', workflow, run: order.run, step: order.step, reason, message });
|
|
540
780
|
claimed.add(order.run);
|
|
541
781
|
};
|
|
782
|
+
/**
|
|
783
|
+
* Refuse an order because this shift lacks capacity, then hand its claim
|
|
784
|
+
* back so a sibling daemon may dispatch it immediately.
|
|
785
|
+
*/
|
|
786
|
+
const releaseOrder = (workflow, order, reason, message) => {
|
|
787
|
+
dropOrder(workflow, order, reason, message);
|
|
788
|
+
releaseClaim(workflow, order.run);
|
|
789
|
+
// Re-arm the next sweep, but only for the two TRANSIENT reasons — see
|
|
790
|
+
// `SweepResult.releasedForCapacity` for why `agent-lane-closed` is not
|
|
791
|
+
// one of them. The independent cooldown prevents a busy cursor from
|
|
792
|
+
// immediately re-claiming this stable step while the same capacity class
|
|
793
|
+
// remains unavailable; it neither replaces server Retry-After backoff
|
|
794
|
+
// nor the worker-failure brake.
|
|
795
|
+
if (reason === 'dispatch-cap-full' || reason === 'agent-cap-full') {
|
|
796
|
+
armCapacityCooldown(workflow, order, reason);
|
|
797
|
+
releasedForCapacity = true;
|
|
798
|
+
}
|
|
799
|
+
};
|
|
542
800
|
for (const wf of instances) {
|
|
801
|
+
// Skipping means no targeted hub observation happened: leave this
|
|
802
|
+
// workflow out of `polled`/`openRuns` so the work-directory reaper never
|
|
803
|
+
// mistakes damping for an empty response. The inbox call and other
|
|
804
|
+
// workflows remain eligible.
|
|
805
|
+
if (workflowCapacityCooldownActive(wf, remaining, agentRoom))
|
|
806
|
+
continue;
|
|
543
807
|
let res;
|
|
544
808
|
const requestStartedAt = monotonicNow();
|
|
545
809
|
try {
|
|
546
|
-
res = await opts.hub.whatsNext({
|
|
810
|
+
res = await opts.hub.whatsNext({
|
|
811
|
+
workflow: wf,
|
|
812
|
+
serve_crews: serveCrews,
|
|
813
|
+
serve_capabilities: [...serveCapabilities],
|
|
814
|
+
});
|
|
547
815
|
}
|
|
548
816
|
catch (e) {
|
|
549
817
|
if (isNonServableRace(e)) {
|
|
@@ -666,8 +934,13 @@ export function createShiftLoop(opts) {
|
|
|
666
934
|
continue;
|
|
667
935
|
}
|
|
668
936
|
if (remaining <= 0) {
|
|
669
|
-
|
|
670
|
-
|
|
937
|
+
if (localQueueHoldMs() > 0) {
|
|
938
|
+
pendingCandidates.set(order.run, candidate);
|
|
939
|
+
opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
|
|
940
|
+
}
|
|
941
|
+
else {
|
|
942
|
+
releaseOrder(wf, order, 'dispatch-cap-full', `at the dispatch cap (${cap}) — handing the claim back to the hub`);
|
|
943
|
+
}
|
|
671
944
|
}
|
|
672
945
|
else {
|
|
673
946
|
remaining--;
|
|
@@ -678,6 +951,7 @@ export function createShiftLoop(opts) {
|
|
|
678
951
|
}
|
|
679
952
|
// Agent wire routing is authoritative. The detached agent-run resolves its
|
|
680
953
|
// own exact step and harness from the order digest, never from this cache.
|
|
954
|
+
// A re-offer already has a live child record; never release it.
|
|
681
955
|
if (workerRuns.has(order.run))
|
|
682
956
|
continue;
|
|
683
957
|
const candidate = {
|
|
@@ -694,13 +968,27 @@ export function createShiftLoop(opts) {
|
|
|
694
968
|
claimed.add(order.run);
|
|
695
969
|
continue;
|
|
696
970
|
}
|
|
971
|
+
if (ceiling === 0) {
|
|
972
|
+
releaseOrder(wf, order, 'agent-lane-closed', 'this shift runs no agent-run children (agent ceiling 0) — handing the claim back to the hub');
|
|
973
|
+
continue;
|
|
974
|
+
}
|
|
697
975
|
if (!reoffer && remaining <= 0) {
|
|
698
|
-
|
|
699
|
-
|
|
976
|
+
if (localQueueHoldMs() > 0) {
|
|
977
|
+
pendingCandidates.set(order.run, candidate);
|
|
978
|
+
opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
|
|
979
|
+
}
|
|
980
|
+
else {
|
|
981
|
+
releaseOrder(wf, order, 'dispatch-cap-full', `at the dispatch cap (${cap}) — handing the claim back to the hub`);
|
|
982
|
+
}
|
|
700
983
|
}
|
|
701
984
|
else if (!reoffer && agentRoom <= 0) {
|
|
702
|
-
|
|
703
|
-
|
|
985
|
+
if (localQueueHoldMs() > 0) {
|
|
986
|
+
pendingCandidates.set(order.run, candidate);
|
|
987
|
+
opts.out(`[${wf}/${order.run}] at the agent-run cap (${agentCapLabel}) — queued for local dispatch`);
|
|
988
|
+
}
|
|
989
|
+
else {
|
|
990
|
+
releaseOrder(wf, order, 'agent-cap-full', `at the agent-run cap (${agentCapLabel}) — handing the claim back to the hub`);
|
|
991
|
+
}
|
|
704
992
|
}
|
|
705
993
|
else {
|
|
706
994
|
if (!reoffer) {
|
|
@@ -725,14 +1013,19 @@ export function createShiftLoop(opts) {
|
|
|
725
1013
|
continue;
|
|
726
1014
|
}
|
|
727
1015
|
if (result === 'total-capacity' || result === 'agent-capacity') {
|
|
728
|
-
pendingCandidates.set(candidate.order.run, candidate);
|
|
729
1016
|
const limit = result === 'total-capacity'
|
|
730
1017
|
? `dispatch cap (${cap})`
|
|
731
|
-
: `agent-run cap (${
|
|
732
|
-
|
|
1018
|
+
: `agent-run cap (${agentCapLabel})`;
|
|
1019
|
+
if (localQueueHoldMs() > 0) {
|
|
1020
|
+
pendingCandidates.set(candidate.order.run, candidate);
|
|
1021
|
+
opts.out(`[${candidate.workflow}/${candidate.order.run}] lost a shared-capacity race at the ${limit} — queued for local dispatch`);
|
|
1022
|
+
}
|
|
1023
|
+
else {
|
|
1024
|
+
releaseOrder(candidate.workflow, candidate.order, result === 'total-capacity' ? 'dispatch-cap-full' : 'agent-cap-full', `lost a shared-capacity race at the ${limit} — handing the claim back to the hub`);
|
|
1025
|
+
}
|
|
733
1026
|
}
|
|
734
1027
|
}
|
|
735
|
-
return { dispatched, polled, openRuns, complete };
|
|
1028
|
+
return { dispatched, polled, openRuns, complete, releasedForCapacity };
|
|
736
1029
|
}
|
|
737
1030
|
/**
|
|
738
1031
|
* PHASE 4 — remove the per-run work directories of runs that are over.
|
|
@@ -788,10 +1081,33 @@ export function createShiftLoop(opts) {
|
|
|
788
1081
|
async function iteration() {
|
|
789
1082
|
const backoffActiveAtStart = monotonicNow() < backoffUntil;
|
|
790
1083
|
let rateLimitedThisIteration = false;
|
|
1084
|
+
// Hub roster cache refresh is low-priority operational work. Its absence is
|
|
1085
|
+
// always a degraded read for children, never a reason to stop dispatch.
|
|
1086
|
+
if (!backoffActiveAtStart &&
|
|
1087
|
+
opts.syncRosters !== undefined &&
|
|
1088
|
+
opts.rosterSyncIntervalMs !== undefined &&
|
|
1089
|
+
monotonicNow() - lastRosterSync >= opts.rosterSyncIntervalMs) {
|
|
1090
|
+
try {
|
|
1091
|
+
await withHubRosterSyncTimeout((signal) => opts.syncRosters(signal), opts.rosterSyncTimeoutMs);
|
|
1092
|
+
}
|
|
1093
|
+
catch (e) {
|
|
1094
|
+
rateLimitedThisIteration = noteServerBackoff(e);
|
|
1095
|
+
opts.err(`roster sync failed: ${errMsg(e)} (continuing)`);
|
|
1096
|
+
emit({ type: 'hub-error', op: 'roster_sync', message: `roster sync failed: ${errMsg(e)} (continuing)` });
|
|
1097
|
+
}
|
|
1098
|
+
finally {
|
|
1099
|
+
// This is a periodic attempt cadence, not a success-only retry loop. A
|
|
1100
|
+
// persistent non-429 failure must not make every 5s poll issue two more
|
|
1101
|
+
// hub calls and append another durable failure record.
|
|
1102
|
+
lastRosterSync = monotonicNow();
|
|
1103
|
+
refreshServeCapabilities();
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
791
1106
|
// Presence when due (starts immediately — this shift exists to conduct).
|
|
792
1107
|
// A server backoff suppresses every hub poll, including presence, while local
|
|
793
1108
|
// reconciliation and queued dispatch continue below.
|
|
794
1109
|
if (!backoffActiveAtStart &&
|
|
1110
|
+
!rateLimitedThisIteration &&
|
|
795
1111
|
monotonicNow() - lastPresence >= opts.presenceIntervalMs) {
|
|
796
1112
|
// `setShift` and `noteAttended` force the next ping by setting
|
|
797
1113
|
// lastPresence to -Infinity. Either can land WHILE this ping is awaiting
|
|
@@ -804,6 +1120,7 @@ export function createShiftLoop(opts) {
|
|
|
804
1120
|
await opts.hub.presencePing({
|
|
805
1121
|
name: shiftName,
|
|
806
1122
|
serve_crews: serveCrews,
|
|
1123
|
+
serve_capabilities: [...serveCapabilities],
|
|
807
1124
|
...(opts.shiftId !== undefined ? { shift_id: opts.shiftId } : {}),
|
|
808
1125
|
...(opts.startedAt !== undefined ? { started_at: opts.startedAt } : {}),
|
|
809
1126
|
...(attendedAt !== undefined ? { attended_at: attendedAt } : {}),
|
|
@@ -827,6 +1144,9 @@ export function createShiftLoop(opts) {
|
|
|
827
1144
|
const reserved = inFlight.reserved;
|
|
828
1145
|
const occupied = live.length + reserved.length;
|
|
829
1146
|
const k = cap - occupied;
|
|
1147
|
+
const agentRoom = agentLane().ceiling -
|
|
1148
|
+
live.filter((record) => record.kind === 'agent-run').length -
|
|
1149
|
+
reserved.filter((record) => record.childKind === 'agent-run').length;
|
|
830
1150
|
// Re-arm the at-capacity report the instant a slot frees, independently of
|
|
831
1151
|
// whether this tick goes on to wake or sweep. The next stretch at capacity
|
|
832
1152
|
// is a NEW episode and gets its own record.
|
|
@@ -859,9 +1179,23 @@ export function createShiftLoop(opts) {
|
|
|
859
1179
|
// had no local capacity, or its sweep failed, the adopted cursor will be
|
|
860
1180
|
// unchanged next time. `sweepOwed` preserves that unconsumed work signal.
|
|
861
1181
|
let swept;
|
|
862
|
-
|
|
1182
|
+
const brakeDue = brakeSweepDueAt !== undefined && monotonicNow() >= brakeSweepDueAt;
|
|
1183
|
+
// Keep a due cooldown armed while total capacity is zero: the outer gate
|
|
1184
|
+
// must still defer hub work, but an unchanged wake needs to retry as soon
|
|
1185
|
+
// as a total slot exists. `sweep()` removes expired entries only when it
|
|
1186
|
+
// can actually make the targeted decision.
|
|
1187
|
+
const capacityCooldownDue = capacityCooldownDueAt !== undefined && monotonicNow() >= capacityCooldownDueAt;
|
|
1188
|
+
const capacityBecameReady = capacityCooldownReady(k, agentRoom);
|
|
1189
|
+
if (wakeSucceeded && (changed || sweepOwed || brakeDue || capacityCooldownDue || capacityBecameReady) && k > 0) {
|
|
1190
|
+
// Clear before awaiting: a new brake armed during sweep must survive.
|
|
1191
|
+
if (brakeDue)
|
|
1192
|
+
brakeSweepDueAt = undefined;
|
|
863
1193
|
swept = await sweep(k, live, reserved);
|
|
864
|
-
|
|
1194
|
+
// A COMPLETE sweep that handed work back for capacity is still unfinished
|
|
1195
|
+
// business. Without this the assignment below would clear the very signal
|
|
1196
|
+
// the release just raised, and the released order would wait for a cursor
|
|
1197
|
+
// change that a single-capability topology never produces.
|
|
1198
|
+
sweepOwed = !swept.complete || swept.releasedForCapacity;
|
|
865
1199
|
}
|
|
866
1200
|
else if (changed && k <= 0) {
|
|
867
1201
|
sweepOwed = true;
|
|
@@ -936,15 +1270,19 @@ export function createShiftLoop(opts) {
|
|
|
936
1270
|
return cap - current.live.length - current.reserved.length;
|
|
937
1271
|
},
|
|
938
1272
|
getCap: () => cap,
|
|
1273
|
+
agentCeiling: () => agentLane().ceiling,
|
|
939
1274
|
setCap: (next) => {
|
|
940
1275
|
cap = next;
|
|
941
1276
|
},
|
|
942
1277
|
getShift: () => ({ name: shiftName, serveCrews: [...serveCrews] }),
|
|
1278
|
+
getServeCapabilities: () => [...serveCapabilities],
|
|
943
1279
|
setShift: (next) => {
|
|
944
1280
|
if (next.name !== undefined)
|
|
945
1281
|
shiftName = next.name;
|
|
946
|
-
if (next.serveCrews !== undefined)
|
|
1282
|
+
if (next.serveCrews !== undefined) {
|
|
947
1283
|
serveCrews = [...next.serveCrews];
|
|
1284
|
+
refreshServeCapabilities();
|
|
1285
|
+
}
|
|
948
1286
|
lastPresence = Number.NEGATIVE_INFINITY; // D6: next iterate() pings immediately
|
|
949
1287
|
presenceGeneration++; // survive an in-flight ping completing after this
|
|
950
1288
|
return { name: shiftName, serveCrews: [...serveCrews] };
|
|
@@ -966,7 +1304,31 @@ export function createShiftLoop(opts) {
|
|
|
966
1304
|
runBrakeKey.delete(run);
|
|
967
1305
|
stepBrake.delete(brakeKey);
|
|
968
1306
|
}
|
|
969
|
-
|
|
1307
|
+
// The hub's closed-run report is authoritative but carries no record identity.
|
|
1308
|
+
removeRecordUnderDispatchLock(run);
|
|
1309
|
+
},
|
|
1310
|
+
noteChildExited: (exit) => {
|
|
1311
|
+
pendingCandidates.delete(exit.run);
|
|
1312
|
+
let removed = false;
|
|
1313
|
+
try {
|
|
1314
|
+
removed = removeRecordUnderDispatchLock(exit.run, { pid: exit.pid }, 1_000, 'owenloop Shift exit reaper');
|
|
1315
|
+
}
|
|
1316
|
+
catch (e) {
|
|
1317
|
+
const retry = e instanceof FileLockTimeoutError
|
|
1318
|
+
? ' (the pid reconciliation will retry)'
|
|
1319
|
+
: '';
|
|
1320
|
+
opts.err(`[${exit.workflow}/${exit.run}] failed to free the dispatch slot after the worker exited: ${errMsg(e)}${retry}`);
|
|
1321
|
+
return;
|
|
1322
|
+
}
|
|
1323
|
+
if (!removed)
|
|
1324
|
+
return;
|
|
1325
|
+
emit({
|
|
1326
|
+
type: 'reaped',
|
|
1327
|
+
workflow: exit.workflow,
|
|
1328
|
+
run: exit.run,
|
|
1329
|
+
kind: exit.kind,
|
|
1330
|
+
pid: exit.pid,
|
|
1331
|
+
});
|
|
970
1332
|
},
|
|
971
1333
|
noteWorkerFailure: (failure) => {
|
|
972
1334
|
const brakeKey = runBrakeKey.get(failure.run);
|