owenloop 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/README.md +137 -35
- package/bin/owenloop.mjs +49 -1
- package/dist/packages/work/src/agent/brief.d.ts +8 -4
- package/dist/packages/work/src/agent/brief.js +7 -3
- package/dist/packages/work/src/agent/loop.d.ts +13 -7
- package/dist/packages/work/src/agent/loop.js +136 -9
- package/dist/packages/work/src/agent/model-policy.d.ts +49 -0
- package/dist/packages/work/src/agent/model-policy.js +138 -0
- package/dist/packages/work/src/bundle/cache.d.ts +7 -5
- package/dist/packages/work/src/bundle/cache.js +20 -6
- package/dist/packages/work/src/bundle/fetch.d.ts +7 -5
- package/dist/packages/work/src/bundle/fetch.js +27 -7
- package/dist/packages/work/src/bundle/types.d.ts +8 -4
- package/dist/packages/work/src/consumed-verifier.d.ts +45 -0
- package/dist/packages/work/src/consumed-verifier.js +205 -0
- package/dist/packages/work/src/exec/instructions.d.ts +99 -0
- package/dist/packages/work/src/exec/instructions.js +323 -0
- package/dist/packages/work/src/exec/loop.d.ts +12 -1
- package/dist/packages/work/src/exec/loop.js +137 -20
- package/dist/packages/work/src/exec/payload.d.ts +29 -0
- package/dist/packages/work/src/exec/payload.js +54 -0
- package/dist/packages/work/src/exec/receipt.d.ts +9 -6
- package/dist/packages/work/src/exec/receipt.js +12 -1
- package/dist/packages/work/src/exec/runner.d.ts +7 -1
- package/dist/packages/work/src/exec/runner.js +74 -3
- package/dist/packages/work/src/harness/child-env.js +7 -0
- package/dist/packages/work/src/harness/claude.d.ts +20 -8
- package/dist/packages/work/src/harness/claude.js +260 -58
- package/dist/packages/work/src/harness/codex.d.ts +5 -4
- package/dist/packages/work/src/harness/codex.js +169 -41
- package/dist/packages/work/src/harness/contract.d.ts +36 -11
- package/dist/packages/work/src/harness/fake.js +1 -0
- package/dist/packages/work/src/harness/jsonrpc-stdio.js +7 -0
- package/dist/packages/work/src/harness/permissions.d.ts +16 -3
- package/dist/packages/work/src/harness/permissions.js +102 -13
- package/dist/packages/work/src/harness/session-store.d.ts +27 -9
- package/dist/packages/work/src/harness/session-store.js +271 -67
- package/dist/packages/work/src/hold/mcp.d.ts +25 -5
- package/dist/packages/work/src/hold/mcp.js +126 -11
- package/dist/packages/work/src/hub/client.d.ts +2 -1
- package/dist/packages/work/src/hub/client.js +14 -1
- package/dist/packages/work/src/hub/types.d.ts +69 -33
- package/dist/packages/work/src/hub/types.js +8 -10
- package/dist/packages/work/src/mcp/server.js +1 -1
- package/dist/packages/work/src/owenloop-bin.d.ts +1 -0
- package/dist/packages/work/src/owenloop-bin.js +18 -0
- package/dist/packages/work/src/roles/agent-run.d.ts +12 -6
- package/dist/packages/work/src/roles/agent-run.js +116 -34
- package/dist/packages/work/src/roles/exec.d.ts +8 -0
- package/dist/packages/work/src/roles/exec.js +27 -2
- package/dist/packages/work/src/roles/hold.d.ts +4 -0
- package/dist/packages/work/src/roles/hold.js +44 -5
- package/dist/packages/work/src/roles/lint.d.ts +3 -4
- package/dist/packages/work/src/roles/lint.js +41 -7
- package/dist/packages/work/src/roles/settings.js +5 -0
- package/dist/packages/work/src/roles/shift.d.ts +1 -0
- package/dist/packages/work/src/roles/shift.js +23 -1
- package/dist/packages/work/src/settings/settings.d.ts +17 -1
- package/dist/packages/work/src/settings/settings.js +39 -0
- package/dist/packages/work/src/shift/loop.d.ts +14 -3
- package/dist/packages/work/src/shift/loop.js +447 -164
- package/dist/packages/work/src/shift/protocol.d.ts +4 -0
- package/dist/packages/work/src/shift/runtime.d.ts +2 -0
- package/dist/packages/work/src/shift/runtime.js +66 -3
- package/dist/packages/work/src/shift/spawn.d.ts +41 -18
- package/dist/packages/work/src/shift/spawn.js +89 -31
- package/dist/packages/work/src/shift/state.d.ts +53 -51
- package/dist/packages/work/src/shift/state.js +325 -102
- package/dist/packages/work/src/submit-proof.d.ts +36 -0
- package/dist/packages/work/src/submit-proof.js +93 -0
- package/dist/src/add.d.ts +64 -259
- package/dist/src/add.js +81 -615
- package/dist/src/archive.d.ts +97 -0
- package/dist/src/archive.js +591 -0
- package/dist/src/bundle/index.d.ts +64 -0
- package/dist/src/bundle/index.js +508 -0
- package/dist/src/bundle/manifest.d.ts +50 -0
- package/dist/src/bundle/manifest.js +435 -0
- package/dist/src/bundle/runtime.d.ts +26 -0
- package/dist/src/bundle/runtime.js +64 -0
- package/dist/src/bundle/tar.d.ts +62 -0
- package/dist/src/bundle/tar.js +238 -0
- package/dist/src/bundle/types.d.ts +105 -0
- package/dist/src/bundle/types.js +21 -0
- package/dist/src/cli.d.ts +90 -20
- package/dist/src/cli.js +2407 -365
- package/dist/src/credentials.d.ts +4 -3
- package/dist/src/credentials.js +21 -19
- package/dist/src/crypto/allowed-signers.d.ts +68 -0
- package/dist/src/crypto/allowed-signers.js +307 -0
- package/dist/src/crypto/canonical.d.ts +8 -0
- package/dist/src/crypto/canonical.js +92 -0
- package/dist/src/crypto/chain.d.ts +65 -0
- package/dist/src/crypto/chain.js +377 -0
- package/dist/src/crypto/dsse.d.ts +209 -0
- package/dist/src/crypto/dsse.js +282 -0
- package/dist/src/crypto/enrollment.d.ts +78 -0
- package/dist/src/crypto/enrollment.js +186 -0
- package/dist/src/crypto/index.d.ts +48 -0
- package/dist/src/crypto/index.js +44 -0
- package/dist/src/crypto/keys.d.ts +258 -0
- package/dist/src/crypto/keys.js +908 -0
- package/dist/src/crypto/org-root.d.ts +33 -0
- package/dist/src/crypto/org-root.js +78 -0
- package/dist/src/crypto/origin-rules.d.ts +56 -0
- package/dist/src/crypto/origin-rules.js +143 -0
- package/dist/src/crypto/policy-floor.d.ts +98 -0
- package/dist/src/crypto/policy-floor.js +353 -0
- package/dist/src/crypto/records.d.ts +316 -0
- package/dist/src/crypto/records.js +157 -0
- package/dist/src/crypto/scope.d.ts +49 -0
- package/dist/src/crypto/scope.js +83 -0
- package/dist/src/crypto/ssh.d.ts +171 -0
- package/dist/src/crypto/ssh.js +396 -0
- package/dist/src/crypto/submission.d.ts +33 -0
- package/dist/src/crypto/submission.js +47 -0
- package/dist/src/crypto/trust-roots.d.ts +29 -0
- package/dist/src/crypto/trust-roots.js +39 -0
- package/dist/src/crypto/verify-consumed.d.ts +67 -0
- package/dist/src/crypto/verify-consumed.js +259 -0
- package/dist/src/crypto/verify-origin.d.ts +50 -0
- package/dist/src/crypto/verify-origin.js +151 -0
- package/dist/src/crypto/verify-publication.d.ts +60 -0
- package/dist/src/crypto/verify-publication.js +150 -0
- package/dist/src/defs.d.ts +24 -1
- package/dist/src/defs.js +145 -25
- package/dist/src/engine.d.ts +99 -3
- package/dist/src/engine.js +238 -34
- package/dist/src/factory.d.ts +16 -0
- package/dist/src/factory.js +16 -4
- package/dist/src/global-config.d.ts +59 -0
- package/dist/src/global-config.js +84 -0
- package/dist/src/hub.d.ts +75 -21
- package/dist/src/hub.js +123 -40
- package/dist/src/index.d.ts +17 -2
- package/dist/src/index.js +20 -2
- package/dist/src/install.d.ts +538 -0
- package/dist/src/install.js +1981 -0
- package/dist/src/lock.d.ts +15 -57
- package/dist/src/lock.js +251 -143
- package/dist/src/mcp/serve.d.ts +46 -22
- package/dist/src/mcp/serve.js +167 -44
- package/dist/src/mcp/server.js +12 -1
- package/dist/src/model.d.ts +15 -0
- package/dist/src/model.js +111 -1
- package/dist/src/order-resolver.d.ts +184 -0
- package/dist/src/order-resolver.js +252 -0
- package/dist/src/package-version.d.ts +10 -0
- package/dist/src/package-version.js +42 -0
- package/dist/src/paths.d.ts +14 -1
- package/dist/src/paths.js +18 -0
- package/dist/src/schemas/enrollment-grant.v1.schema.json +90 -0
- package/dist/src/schemas/index.d.ts +1416 -0
- package/dist/src/schemas/index.js +31 -0
- package/dist/src/schemas/order.v1.schema.json +106 -0
- package/dist/src/schemas/origin.v1.schema.json +68 -0
- package/dist/src/schemas/policy-floor.v1.schema.json +29 -0
- package/dist/src/schemas/publication.v1.schema.json +31 -0
- package/dist/src/schemas/revocation.v1.schema.json +39 -0
- package/dist/src/schemas/submission.v1.schema.json +61 -0
- package/dist/src/store/def-source.d.ts +52 -0
- package/dist/src/store/def-source.js +335 -0
- package/dist/src/store/index-file.d.ts +69 -0
- package/dist/src/store/index-file.js +151 -0
- package/dist/src/store/index.d.ts +27 -0
- package/dist/src/store/index.js +22 -0
- package/dist/src/store/ingestor.d.ts +43 -0
- package/dist/src/store/ingestor.js +211 -0
- package/dist/src/store/install.d.ts +228 -0
- package/dist/src/store/install.js +555 -0
- package/dist/src/store/instruction-source.d.ts +49 -0
- package/dist/src/store/instruction-source.js +298 -0
- package/dist/src/store/origin-evidence.d.ts +21 -0
- package/dist/src/store/origin-evidence.js +66 -0
- package/dist/src/store/pre-commit-verifier.d.ts +68 -0
- package/dist/src/store/pre-commit-verifier.js +418 -0
- package/dist/src/store/resolve.d.ts +138 -0
- package/dist/src/store/resolve.js +363 -0
- package/dist/src/store/types.d.ts +288 -0
- package/dist/src/store/types.js +356 -0
- package/dist/src/types.d.ts +45 -11
- package/dist/src/untar.d.ts +8 -23
- package/dist/src/untar.js +10 -136
- package/docs/agent-runner.md +86 -0
- package/docs/authoring.md +57 -6
- package/docs/bundles.md +250 -0
- package/docs/cli.md +1171 -97
- package/docs/crypto.md +816 -0
- package/docs/design.md +342 -20
- package/docs/embedding.md +132 -25
- package/docs/wire-contracts.md +380 -0
- package/package.json +6 -1
- package/plugins/claude-code/.claude-plugin/marketplace.json +14 -0
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +8 -0
- package/plugins/claude-code/plugin/.mcp.json +14 -0
- package/plugins/claude-code/plugin/hooks/hooks.json +26 -0
- package/plugins/claude-code/plugin/hooks/session-end.sh +40 -0
- package/plugins/claude-code/plugin/hooks/session-start.sh +52 -0
- package/plugins/claude-code/plugin/skills/author/SKILL.md +181 -0
- package/plugins/claude-code/plugin/skills/conduct/SKILL.md +106 -0
- package/plugins/claude-code/plugin/skills/shift/SKILL.md +105 -0
- package/plugins/codex/.agents/plugins/marketplace.json +21 -0
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +7 -0
- package/plugins/codex/plugins/owenloop/.mcp.json +20 -0
- package/plugins/codex/plugins/owenloop/hooks/hooks.json +26 -0
- package/plugins/codex/plugins/owenloop/hooks/session-end.sh +40 -0
- package/plugins/codex/plugins/owenloop/hooks/session-start.sh +52 -0
- package/plugins/codex/plugins/owenloop/skills/author/SKILL.md +181 -0
- package/plugins/codex/plugins/owenloop/skills/conduct/SKILL.md +106 -0
- package/plugins/codex/plugins/owenloop/skills/shift/SKILL.md +105 -0
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
* `createShiftLoop` is a standing Shift loop with every side-effecting
|
|
5
5
|
* dependency injected (hub client, spawner, sleep/clock, output streams, dirs),
|
|
6
6
|
* so the same core can run behind the local shift daemon. Per iteration it: pings
|
|
7
|
-
* presence when due →
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* the poll interval.
|
|
7
|
+
* presence when due → reconciles local children → dispatches any already-claimed
|
|
8
|
+
* orders waiting for local capacity → checks `wake(cursor)` → sweeps `whats_next`
|
|
9
|
+
* when the hub changed or a prior sweep was skipped/failed → classifies/meters
|
|
10
|
+
* new orders → reaps work dirs → adopts the cursor → sleeps the poll interval.
|
|
11
11
|
*
|
|
12
12
|
* DISPATCH SPLIT (D2 decision 3 — the D12 routing contract). BOTH kinds spawn a
|
|
13
13
|
* detached child; there is exactly ONE way each kind runs, and no flag selects
|
|
@@ -15,9 +15,9 @@
|
|
|
15
15
|
* - COMMAND orders spawn a detached `owenloop work exec <wf>/<run> --origin <url>`
|
|
16
16
|
* child that self-leases (unchanged from C3).
|
|
17
17
|
* - AGENT orders spawn a detached `owenloop work agent-run <wf>/<run>` child that
|
|
18
|
-
* hosts the step agent itself through a harness adapter.
|
|
19
|
-
*
|
|
20
|
-
*
|
|
18
|
+
* hosts the step agent itself through a harness adapter. The child resolves the
|
|
19
|
+
* order-pinned instruction digest and selects the verified step and harness;
|
|
20
|
+
* Shift never substitutes definition-name cache metadata for modern orders.
|
|
21
21
|
* - The shift makes NO first-contact `get_order` for any kind. It holds no
|
|
22
22
|
* leases; the spawned child makes first contact and closes the B2 pickup
|
|
23
23
|
* window. A failed hand-off lapses back via that window instead of sitting
|
|
@@ -25,24 +25,24 @@
|
|
|
25
25
|
*
|
|
26
26
|
* METERING (decision 3): free capacity `k = cap − liveInFlight`, where
|
|
27
27
|
* liveInFlight counts every live child record, each one pid-probed (see
|
|
28
|
-
* state.ts). When `k <= 0` the loop skips `whats_next`
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
28
|
+
* state.ts). When `k <= 0` the loop skips `whats_next` but remembers that the
|
|
29
|
+
* adopted wake still needs a sweep. Orders already returned by `whats_next`
|
|
30
|
+
* are already claimed, so over-cap orders enter an in-memory queue and dispatch
|
|
31
|
+
* as soon as local child capacity frees; waiting for another hub wake would
|
|
32
|
+
* strand those claims until pickup expiry.
|
|
32
33
|
*
|
|
33
34
|
* RESILIENCE: presence/wake/whats_next failures are logged and never kill the
|
|
34
35
|
* loop. `stop()` flips a flag checked between awaits; the in-flight sweep
|
|
35
36
|
* finishes, `run()` resolves 0, and no hub call is made afterward. Detached
|
|
36
37
|
* children are never killed on shutdown — that is the drain semantic.
|
|
37
38
|
*
|
|
38
|
-
* PINNED-HASH DISPATCH (E, DD-4):
|
|
39
|
-
*
|
|
40
|
-
* `readDispatchBundle(cacheDir, defName)
|
|
41
|
-
* →
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
* flows through `ChildRecord.hash` untouched.
|
|
39
|
+
* LEGACY PINNED-HASH DISPATCH (E, DD-4): the old wire shape omits both `worker`
|
|
40
|
+
* and `defDigest` and serves only a def NAME. Only that explicitly legacy shape
|
|
41
|
+
* uses `readDispatchBundle(cacheDir, defName)`: 0 pins → latest; exactly 1 pinned
|
|
42
|
+
* hash → that frozen version; >1 conflicting pins → refuse and leave the orders
|
|
43
|
+
* for the pickup window. Modern orders route from authoritative `worker` plus
|
|
44
|
+
* exact `defDigest`; modern command metadata is verified through
|
|
45
|
+
* `resolveOrderStep`, and modern agent metadata is resolved by `agent-run`.
|
|
46
46
|
*
|
|
47
47
|
* LIVE SHIFT IDENTITY: `name`/`serveCrews` on `ShiftLoopOptions` are INITIAL
|
|
48
48
|
* values only. The loop holds them as live closure state (`getShift`/`setShift`
|
|
@@ -51,20 +51,39 @@
|
|
|
51
51
|
* `setShift` also resets the presence timer so the new identity reaches the hub
|
|
52
52
|
* on the very next `iterate()`.
|
|
53
53
|
*/
|
|
54
|
+
import { join } from 'node:path';
|
|
55
|
+
import { performance } from 'node:perf_hooks';
|
|
56
|
+
import { acquireFileLockSync, releaseFileLock } from "../../../../src/lock.js";
|
|
54
57
|
import { readDispatchBundle } from "../bundle/cache.js";
|
|
58
|
+
import { HubError } from "../hub/types.js";
|
|
55
59
|
import { isCommandStep, resolveCommandRouting } from "./routing.js";
|
|
56
|
-
import { reconcileInFlight,
|
|
60
|
+
import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reserveChild, startReservedChild, removeChildRecord, ensureStateDir, } from "./state.js";
|
|
57
61
|
import { DEFAULT_WORK_DIR_TTL_MS, sweepWorkDirs as sweepWorkDirsImpl } from "../agent/workdir.js";
|
|
58
62
|
import { sessionsPath } from "../harness/session-store.js";
|
|
59
63
|
/** The `maxConcurrentAgents` fallback when the option is absent. */
|
|
60
64
|
const DEFAULT_MAX_AGENTS = 4;
|
|
65
|
+
/**
|
|
66
|
+
* The hub reaps a never-contacted claim after 120 seconds. Stop local queueing
|
|
67
|
+
* after 90 seconds so a detached child retains 30 seconds to make first contact.
|
|
68
|
+
*/
|
|
69
|
+
export const MAX_PENDING_CANDIDATE_AGE_MS = 90_000;
|
|
61
70
|
function errMsg(e) {
|
|
62
71
|
return e instanceof Error ? e.message : String(e);
|
|
63
72
|
}
|
|
73
|
+
/** The explicit hub refusal produced when an inbox row becomes terminal after
|
|
74
|
+
* the inbox snapshot but before its targeted serving call. This is a normal
|
|
75
|
+
* stale-candidate race, not a Shift failure worth retrying forever. */
|
|
76
|
+
function isNonServableRace(e) {
|
|
77
|
+
return e instanceof HubError
|
|
78
|
+
&& e.status === 403
|
|
79
|
+
&& e.code === 'forbidden'
|
|
80
|
+
&& e.message.includes('non-running instance is not servable');
|
|
81
|
+
}
|
|
64
82
|
export function createShiftLoop(opts) {
|
|
65
83
|
let stopped = false;
|
|
66
84
|
let cap = opts.cap;
|
|
67
85
|
const isAlive = opts.isAlive;
|
|
86
|
+
const monotonicNow = opts.monotonicNow ?? performance.now.bind(performance);
|
|
68
87
|
// Live shift identity (MCP `clock_in`, D3-D7 of the plan). Seeded from opts,
|
|
69
88
|
// which are now INITIAL values only. Arrays are copied so neither the loop
|
|
70
89
|
// nor a caller of getShift/setShift can mutate the other's state afterward.
|
|
@@ -74,6 +93,29 @@ export function createShiftLoop(opts) {
|
|
|
74
93
|
let cursor;
|
|
75
94
|
let lastPresence = Number.NEGATIVE_INFINITY;
|
|
76
95
|
let attendedAt;
|
|
96
|
+
/**
|
|
97
|
+
* Orders returned by `whats_next` are already claimed. If the hub returns more
|
|
98
|
+
* orders than this process can spawn, retain those claims locally until a child
|
|
99
|
+
* exits and capacity opens. Waiting for another wake cannot work: no hub event
|
|
100
|
+
* is required when a local child exits, and `tick` will not re-return a claim
|
|
101
|
+
* that is already in flight.
|
|
102
|
+
*/
|
|
103
|
+
const pendingCandidates = new Map();
|
|
104
|
+
/** A changed wake whose sweep was skipped or failed must be retried after the
|
|
105
|
+
* cursor is adopted; otherwise the next unchanged wake hides that work. */
|
|
106
|
+
let sweepOwed = false;
|
|
107
|
+
/** Earliest server-approved time for the next polling iteration. */
|
|
108
|
+
let backoffUntil = Number.NEGATIVE_INFINITY;
|
|
109
|
+
/** Bumped whenever a ping starts or is forced due, so a ping that completes
|
|
110
|
+
* after a force does not stamp the cadence over that force. */
|
|
111
|
+
let presenceGeneration = 0;
|
|
112
|
+
function noteServerBackoff(error) {
|
|
113
|
+
if (!(error instanceof HubError) || error.status !== 429)
|
|
114
|
+
return false;
|
|
115
|
+
const delay = error.retryAfterMs ?? opts.pollIntervalMs;
|
|
116
|
+
backoffUntil = Math.max(backoffUntil, monotonicNow() + delay);
|
|
117
|
+
return true;
|
|
118
|
+
}
|
|
77
119
|
function emit(event) {
|
|
78
120
|
if (opts.onEvent === undefined)
|
|
79
121
|
return;
|
|
@@ -85,9 +127,10 @@ export function createShiftLoop(opts) {
|
|
|
85
127
|
}
|
|
86
128
|
}
|
|
87
129
|
function reconcile() {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
130
|
+
const result = reconcileInFlight(opts.stateDir, {
|
|
131
|
+
...(isAlive !== undefined ? { isAlive } : {}),
|
|
132
|
+
now: opts.now(),
|
|
133
|
+
});
|
|
91
134
|
for (const rec of result.reaped) {
|
|
92
135
|
emit({
|
|
93
136
|
type: 'reaped',
|
|
@@ -97,8 +140,163 @@ export function createShiftLoop(opts) {
|
|
|
97
140
|
pid: rec.pid,
|
|
98
141
|
});
|
|
99
142
|
}
|
|
143
|
+
for (const reservation of result.abandoned) {
|
|
144
|
+
opts.err(`[${reservation.workflow}/${reservation.run}] abandoned ${reservation.childKind} dispatch reservation expired and was cancelled`);
|
|
145
|
+
}
|
|
100
146
|
return result;
|
|
101
147
|
}
|
|
148
|
+
/**
|
|
149
|
+
* Recheck shared capacity and create the per-run reservation while holding one
|
|
150
|
+
* state-directory-wide lock. The lock ends before spawn: the durable
|
|
151
|
+
* reservation, not a long critical section, protects capacity during spawn.
|
|
152
|
+
*/
|
|
153
|
+
function reserveCandidate(c) {
|
|
154
|
+
const childKind = c.kind === 'command' ? 'exec' : 'agent-run';
|
|
155
|
+
const dispatchLock = acquireFileLockSync(join(opts.stateDir, '.dispatch.lock'), {
|
|
156
|
+
waitMs: 30_000,
|
|
157
|
+
label: 'owenloop Shift dispatch',
|
|
158
|
+
});
|
|
159
|
+
try {
|
|
160
|
+
const fresh = reconcileInFlight(opts.stateDir, {
|
|
161
|
+
...(isAlive === undefined ? {} : { isAlive }),
|
|
162
|
+
now: opts.now(),
|
|
163
|
+
});
|
|
164
|
+
if (fresh.live.some((record) => record.run === c.order.run) ||
|
|
165
|
+
fresh.reserved.some((record) => record.run === c.order.run))
|
|
166
|
+
return 'duplicate';
|
|
167
|
+
if (fresh.live.length + fresh.reserved.length >= cap)
|
|
168
|
+
return 'total-capacity';
|
|
169
|
+
if (childKind === 'agent-run') {
|
|
170
|
+
const agents = fresh.live.filter((record) => record.kind === 'agent-run').length +
|
|
171
|
+
fresh.reserved.filter((record) => record.childKind === 'agent-run').length;
|
|
172
|
+
if (agents >= (opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS))
|
|
173
|
+
return 'agent-capacity';
|
|
174
|
+
}
|
|
175
|
+
return reserveChild(opts.stateDir, {
|
|
176
|
+
workflow: c.workflow,
|
|
177
|
+
run: c.order.run,
|
|
178
|
+
reservedAt: opts.now(),
|
|
179
|
+
childKind,
|
|
180
|
+
step: c.order.step,
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
finally {
|
|
184
|
+
releaseFileLock(dispatchLock);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
function dispatchCandidate(c) {
|
|
188
|
+
const childKind = c.kind === 'command' ? 'exec' : 'agent-run';
|
|
189
|
+
let reservation;
|
|
190
|
+
let cancel;
|
|
191
|
+
try {
|
|
192
|
+
const reserved = reserveCandidate(c);
|
|
193
|
+
if (typeof reserved === 'string')
|
|
194
|
+
return reserved;
|
|
195
|
+
reservation = reserved.reservation;
|
|
196
|
+
const spawned = opts.spawner({
|
|
197
|
+
workflow: c.workflow,
|
|
198
|
+
run: c.order.run,
|
|
199
|
+
step: c.order.step,
|
|
200
|
+
...(childKind === 'agent-run' ? { kind: 'agent-run' } : {}),
|
|
201
|
+
startGate: reserved.gatePath,
|
|
202
|
+
});
|
|
203
|
+
cancel = spawned.cancel ?? spawned.terminate;
|
|
204
|
+
const rec = finalizeChildReservation(opts.stateDir, reservation, {
|
|
205
|
+
pid: spawned.pid,
|
|
206
|
+
spawnedAt: opts.now(),
|
|
207
|
+
kind: childKind,
|
|
208
|
+
...(childKind === 'agent-run' && c.defName !== undefined ? { def: c.defName } : {}),
|
|
209
|
+
...(childKind === 'agent-run' && c.defHash !== undefined ? { hash: c.defHash } : {}),
|
|
210
|
+
...(childKind === 'agent-run' ? { step: c.order.step } : {}),
|
|
211
|
+
});
|
|
212
|
+
startReservedChild(opts.stateDir, rec);
|
|
213
|
+
emit({
|
|
214
|
+
type: 'dispatched',
|
|
215
|
+
workflow: c.workflow,
|
|
216
|
+
run: c.order.run,
|
|
217
|
+
step: c.order.step,
|
|
218
|
+
kind: childKind,
|
|
219
|
+
pid: spawned.pid,
|
|
220
|
+
});
|
|
221
|
+
opts.out(`dispatched ${c.kind === 'command' ? 'command' : 'agent-run'} ${c.workflow}/${c.order.run} ` +
|
|
222
|
+
`(step '${c.order.step}', pid ${spawned.pid})`);
|
|
223
|
+
return 'dispatched';
|
|
224
|
+
}
|
|
225
|
+
catch (e) {
|
|
226
|
+
cancel?.();
|
|
227
|
+
if (reservation !== undefined) {
|
|
228
|
+
try {
|
|
229
|
+
cancelReservedChild(opts.stateDir, reservation);
|
|
230
|
+
}
|
|
231
|
+
catch (cleanupError) {
|
|
232
|
+
opts.err(`[${c.workflow}/${c.order.run}] failed to cancel dispatch reservation: ${errMsg(cleanupError)}`);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
const message = errMsg(e);
|
|
236
|
+
emit({
|
|
237
|
+
type: 'failed',
|
|
238
|
+
workflow: c.workflow,
|
|
239
|
+
run: c.order.run,
|
|
240
|
+
step: c.order.step,
|
|
241
|
+
kind: childKind,
|
|
242
|
+
message,
|
|
243
|
+
});
|
|
244
|
+
const prefix = c.kind === 'command' ? 'spawn' : 'agent-run spawn';
|
|
245
|
+
opts.err(`${prefix} for ${c.workflow}/${c.order.run} failed: ${message}`);
|
|
246
|
+
return 'failed';
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
function discardExpiredCandidate(candidate, queued) {
|
|
250
|
+
if (monotonicNow() - candidate.requestStartedAt < MAX_PENDING_CANDIDATE_AGE_MS)
|
|
251
|
+
return false;
|
|
252
|
+
const prefix = queued ? 'queued claim' : 'claim';
|
|
253
|
+
opts.err(`[${candidate.workflow}/${candidate.order.run}] ${prefix} expired before local dispatch — leaving the hub pickup window to re-offer it`);
|
|
254
|
+
return true;
|
|
255
|
+
}
|
|
256
|
+
/** Dispatch already-claimed orders when local child capacity becomes free. */
|
|
257
|
+
function drainPending(live, reserved) {
|
|
258
|
+
let remaining = cap - live.length - reserved.length;
|
|
259
|
+
if (remaining <= 0 || pendingCandidates.size === 0)
|
|
260
|
+
return 0;
|
|
261
|
+
const maxAgents = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
|
|
262
|
+
let agentRoom = maxAgents -
|
|
263
|
+
live.filter((r) => r.kind === 'agent-run').length -
|
|
264
|
+
reserved.filter((r) => r.childKind === 'agent-run').length;
|
|
265
|
+
const liveRuns = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
|
|
266
|
+
let dispatched = 0;
|
|
267
|
+
for (const [run, candidate] of pendingCandidates) {
|
|
268
|
+
if (remaining <= 0)
|
|
269
|
+
break;
|
|
270
|
+
if (discardExpiredCandidate(candidate, true)) {
|
|
271
|
+
pendingCandidates.delete(run);
|
|
272
|
+
continue;
|
|
273
|
+
}
|
|
274
|
+
if (liveRuns.has(run)) {
|
|
275
|
+
pendingCandidates.delete(run);
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
278
|
+
// The agent lane's second budget can be full while the dispatch cap still
|
|
279
|
+
// has room. Skip this entry and KEEP it queued (a command entry further
|
|
280
|
+
// down the map is still dispatchable) — `continue`, never `break`, so one
|
|
281
|
+
// blocked agent does not head-of-line-block the whole queue.
|
|
282
|
+
if (candidate.kind === 'agent' && agentRoom <= 0)
|
|
283
|
+
continue;
|
|
284
|
+
const result = dispatchCandidate(candidate);
|
|
285
|
+
if (result === 'total-capacity')
|
|
286
|
+
break;
|
|
287
|
+
if (result === 'agent-capacity')
|
|
288
|
+
continue;
|
|
289
|
+
pendingCandidates.delete(run);
|
|
290
|
+
if (result !== 'dispatched')
|
|
291
|
+
continue;
|
|
292
|
+
dispatched++;
|
|
293
|
+
remaining--;
|
|
294
|
+
liveRuns.add(run);
|
|
295
|
+
if (candidate.kind === 'agent')
|
|
296
|
+
agentRoom--;
|
|
297
|
+
}
|
|
298
|
+
return dispatched;
|
|
299
|
+
}
|
|
102
300
|
/**
|
|
103
301
|
* One sweep. Collects dispatch candidates across the target instance(s)
|
|
104
302
|
* within the free-capacity `budget`, then dispatches by kind: a command order
|
|
@@ -106,7 +304,7 @@ export function createShiftLoop(opts) {
|
|
|
106
304
|
* `owenloop work agent-run` child. Returns how many children were spawned, plus the
|
|
107
305
|
* polled/open-run sets the work-dir reaper needs.
|
|
108
306
|
*/
|
|
109
|
-
async function sweep(budget, live) {
|
|
307
|
+
async function sweep(budget, live, reserved) {
|
|
110
308
|
let dispatched = 0;
|
|
111
309
|
/**
|
|
112
310
|
* PHASE 4, for the work-dir reaper only. `polled` is the set of workflows
|
|
@@ -120,13 +318,16 @@ export function createShiftLoop(opts) {
|
|
|
120
318
|
*/
|
|
121
319
|
const polled = new Set();
|
|
122
320
|
const openRuns = new Set();
|
|
321
|
+
let complete = true;
|
|
123
322
|
let remaining = budget;
|
|
124
|
-
const liveRuns = new Set(live.map((r) => r.run));
|
|
125
|
-
// The agent lane's SECOND budget (D7). Counted from
|
|
126
|
-
//
|
|
127
|
-
//
|
|
323
|
+
const liveRuns = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
|
|
324
|
+
// The agent lane's SECOND budget (D7). Counted from live records and durable
|
|
325
|
+
// reservations, so a crash between reservation and spawn cannot overbook it.
|
|
326
|
+
// A child or reservation that is reaped frees the slot on the next reconcile.
|
|
128
327
|
const maxAgents = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
|
|
129
|
-
let agentRoom = maxAgents -
|
|
328
|
+
let agentRoom = maxAgents -
|
|
329
|
+
live.filter((r) => r.kind === 'agent-run').length -
|
|
330
|
+
reserved.filter((r) => r.childKind === 'agent-run').length;
|
|
130
331
|
/**
|
|
131
332
|
* Runs that ALREADY have a live `agent-run` child. An agent-run child is a
|
|
132
333
|
* PROCESS: re-dispatching one would put a second harness session on a single
|
|
@@ -134,7 +335,10 @@ export function createShiftLoop(opts) {
|
|
|
134
335
|
* re-offer for a run a worker already holds is skipped outright, the way the
|
|
135
336
|
* command lane does it.
|
|
136
337
|
*/
|
|
137
|
-
const workerRuns = new Set(
|
|
338
|
+
const workerRuns = new Set([
|
|
339
|
+
...live.filter((r) => r.kind === 'agent-run').map((r) => r.run),
|
|
340
|
+
...reserved.filter((r) => r.childKind === 'agent-run').map((r) => r.run),
|
|
341
|
+
]);
|
|
138
342
|
// Resolve which instances to poll.
|
|
139
343
|
let instances;
|
|
140
344
|
if (opts.workflow !== undefined) {
|
|
@@ -146,8 +350,9 @@ export function createShiftLoop(opts) {
|
|
|
146
350
|
instances = (inbox.instances ?? []).map((i) => i.workflow);
|
|
147
351
|
}
|
|
148
352
|
catch (e) {
|
|
353
|
+
noteServerBackoff(e);
|
|
149
354
|
opts.err(`inbox whats_next failed: ${errMsg(e)}`);
|
|
150
|
-
return { dispatched, polled, openRuns };
|
|
355
|
+
return { dispatched, polled, openRuns, complete: false };
|
|
151
356
|
}
|
|
152
357
|
}
|
|
153
358
|
// Collect candidates, metering NEW dispatches and deduping in-flight.
|
|
@@ -155,11 +360,25 @@ export function createShiftLoop(opts) {
|
|
|
155
360
|
const candidates = [];
|
|
156
361
|
for (const wf of instances) {
|
|
157
362
|
let res;
|
|
363
|
+
const requestStartedAt = monotonicNow();
|
|
158
364
|
try {
|
|
159
365
|
res = await opts.hub.whatsNext({ workflow: wf, serve_crews: serveCrews });
|
|
160
366
|
}
|
|
161
367
|
catch (e) {
|
|
368
|
+
if (isNonServableRace(e)) {
|
|
369
|
+
// Treat the targeted call as a successful empty observation for the
|
|
370
|
+
// work-dir reaper and consume this wake. A later hub event may list
|
|
371
|
+
// new work, but this stale terminal row must not force every poll to
|
|
372
|
+
// repeat the same ForbiddenError.
|
|
373
|
+
polled.add(wf);
|
|
374
|
+
opts.out(`[${wf}] skipped stale terminal inbox candidate`);
|
|
375
|
+
continue;
|
|
376
|
+
}
|
|
377
|
+
const rateLimited = noteServerBackoff(e);
|
|
378
|
+
complete = false;
|
|
162
379
|
opts.err(`whats_next for ${wf} failed: ${errMsg(e)}`);
|
|
380
|
+
if (rateLimited)
|
|
381
|
+
break;
|
|
163
382
|
continue;
|
|
164
383
|
}
|
|
165
384
|
const orders = res.orders ?? [];
|
|
@@ -167,153 +386,171 @@ export function createShiftLoop(opts) {
|
|
|
167
386
|
for (const o of orders)
|
|
168
387
|
openRuns.add(o.run);
|
|
169
388
|
const defName = res.def;
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
389
|
+
const legacyOrders = orders.filter((order) => !Object.prototype.hasOwnProperty.call(order, 'worker') &&
|
|
390
|
+
!Object.prototype.hasOwnProperty.call(order, 'defDigest'));
|
|
391
|
+
// The definition-name cache exists only for the explicitly identified old
|
|
392
|
+
// wire shape. Modern orders route from `worker` and exact `defDigest`.
|
|
393
|
+
const dispatch = legacyOrders.length > 0 && defName !== undefined
|
|
394
|
+
? readDispatchBundle(opts.cacheDir, defName)
|
|
395
|
+
: { bundle: null };
|
|
173
396
|
const bundle = dispatch.bundle;
|
|
174
|
-
if (
|
|
397
|
+
if (legacyOrders.length > 0 && dispatch.warning !== undefined) {
|
|
175
398
|
opts.err(`[${wf}] ${dispatch.warning}`);
|
|
176
399
|
}
|
|
177
|
-
else if (
|
|
178
|
-
opts.err(`no cached bundle for def '${defName}' — run \`owenloop work prepare ${defName}\` (
|
|
400
|
+
else if (legacyOrders.length > 0 && bundle === null && defName !== undefined) {
|
|
401
|
+
opts.err(`no cached bundle for def '${defName}' — run \`owenloop work prepare ${defName}\` (legacy orders left for pickup)`);
|
|
179
402
|
}
|
|
180
403
|
for (const order of orders) {
|
|
181
404
|
if (claimed.has(order.run))
|
|
182
405
|
continue; // seen this sweep
|
|
183
|
-
|
|
184
|
-
|
|
406
|
+
if (pendingCandidates.has(order.run))
|
|
407
|
+
continue; // already claimed and queued locally
|
|
185
408
|
const reoffer = liveRuns.has(order.run);
|
|
186
|
-
|
|
187
|
-
|
|
409
|
+
const workerPresent = Object.prototype.hasOwnProperty.call(order, 'worker');
|
|
410
|
+
const digestPresent = Object.prototype.hasOwnProperty.call(order, 'defDigest');
|
|
411
|
+
const legacy = !workerPresent && !digestPresent;
|
|
412
|
+
let step;
|
|
413
|
+
let kind;
|
|
414
|
+
let candidateDefName;
|
|
415
|
+
let candidateDefHash;
|
|
416
|
+
if (legacy) {
|
|
417
|
+
step = bundle?.def.steps.find((candidate) => candidate.name === order.step);
|
|
418
|
+
if (step === undefined || bundle === null)
|
|
419
|
+
continue;
|
|
420
|
+
kind = isCommandStep(step) ? 'command' : 'agent';
|
|
421
|
+
candidateDefName = defName;
|
|
422
|
+
candidateDefHash = bundle.def.hash;
|
|
423
|
+
}
|
|
424
|
+
else {
|
|
425
|
+
if (typeof order.defDigest !== 'string' || order.defDigest.trim() === '') {
|
|
426
|
+
opts.err(`[${wf}/${order.run}] malformed modern work order: non-empty defDigest is required whenever modern routing metadata is present — leaving for manual pickup`);
|
|
427
|
+
claimed.add(order.run);
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
// The deployed projection omits `worker` for the default executor. That
|
|
431
|
+
// shape is a modern AGENT order because `defDigest` pins the exact step;
|
|
432
|
+
// only an explicitly authored executor produces `worker` on the wire.
|
|
433
|
+
if (!workerPresent || order.worker === 'agent') {
|
|
434
|
+
kind = 'agent';
|
|
435
|
+
}
|
|
436
|
+
else if (order.worker === 'command') {
|
|
437
|
+
kind = 'command';
|
|
438
|
+
try {
|
|
439
|
+
step = await opts.resolveOrderStep?.(order);
|
|
440
|
+
}
|
|
441
|
+
catch (error) {
|
|
442
|
+
opts.err(`[${wf}/${order.run}] exact command instructions for digest '${order.defDigest}' failed verification: ${errMsg(error)} — leaving for manual pickup`);
|
|
443
|
+
claimed.add(order.run);
|
|
444
|
+
continue;
|
|
445
|
+
}
|
|
446
|
+
if (step === undefined) {
|
|
447
|
+
opts.err(`[${wf}/${order.run}] exact command routing metadata for digest '${order.defDigest}' is unavailable — leaving for manual pickup`);
|
|
448
|
+
claimed.add(order.run);
|
|
449
|
+
continue;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
else if (typeof order.worker !== 'string' || order.worker.trim() === '') {
|
|
453
|
+
opts.err(`[${wf}/${order.run}] malformed modern work order: worker must be absent, 'agent', or 'command' — leaving for manual pickup`);
|
|
454
|
+
claimed.add(order.run);
|
|
455
|
+
continue;
|
|
456
|
+
}
|
|
457
|
+
else {
|
|
458
|
+
opts.err(`[${wf}/${order.run}] unsupported worker '${order.worker}' — leaving for manual pickup`);
|
|
459
|
+
claimed.add(order.run);
|
|
460
|
+
continue;
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
if (kind === 'command') {
|
|
464
|
+
// A command already in flight (or durably reserved) is never respawned.
|
|
188
465
|
if (reoffer)
|
|
189
466
|
continue;
|
|
190
|
-
|
|
191
|
-
for (const w of r.warnings)
|
|
192
|
-
opts.err(`[${wf}/${order.run}] ${w}`);
|
|
193
|
-
if (!r.autoDispatch) {
|
|
194
|
-
opts.out(`[${wf}/${order.run}] command step '${order.step}' routed to Shift — leaving for pickup window`);
|
|
467
|
+
if (step === undefined)
|
|
195
468
|
continue;
|
|
469
|
+
const routing = resolveCommandRouting(opts.commandRouting, step);
|
|
470
|
+
for (const warning of routing.warnings)
|
|
471
|
+
opts.err(`[${wf}/${order.run}] ${warning}`);
|
|
472
|
+
if (!routing.autoDispatch) {
|
|
473
|
+
opts.out(`[${wf}/${order.run}] command step '${order.step}' routed to manual — leaving for pickup window`);
|
|
474
|
+
continue;
|
|
475
|
+
}
|
|
476
|
+
const candidate = {
|
|
477
|
+
order,
|
|
478
|
+
workflow: wf,
|
|
479
|
+
step,
|
|
480
|
+
defName: candidateDefName,
|
|
481
|
+
defHash: candidateDefHash,
|
|
482
|
+
kind,
|
|
483
|
+
requestStartedAt,
|
|
484
|
+
reoffer,
|
|
485
|
+
};
|
|
486
|
+
if (discardExpiredCandidate(candidate, false)) {
|
|
487
|
+
claimed.add(order.run);
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
if (remaining <= 0) {
|
|
491
|
+
pendingCandidates.set(order.run, candidate);
|
|
492
|
+
opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
|
|
493
|
+
}
|
|
494
|
+
else {
|
|
495
|
+
remaining--;
|
|
496
|
+
candidates.push(candidate);
|
|
196
497
|
}
|
|
197
|
-
if (remaining <= 0)
|
|
198
|
-
continue; // out of free capacity for new work
|
|
199
|
-
remaining--;
|
|
200
|
-
candidates.push({ order, workflow: wf, step, defName, defHash: bundle?.def.hash, kind: 'command', reoffer });
|
|
201
498
|
claimed.add(order.run);
|
|
202
499
|
continue;
|
|
203
500
|
}
|
|
204
|
-
// Agent
|
|
205
|
-
//
|
|
501
|
+
// Agent wire routing is authoritative. The detached agent-run resolves its
|
|
502
|
+
// own exact step and harness from the order digest, never from this cache.
|
|
206
503
|
if (workerRuns.has(order.run))
|
|
207
504
|
continue;
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
505
|
+
const candidate = {
|
|
506
|
+
order,
|
|
507
|
+
workflow: wf,
|
|
508
|
+
step: legacy ? step : undefined,
|
|
509
|
+
defName: candidateDefName,
|
|
510
|
+
defHash: candidateDefHash,
|
|
511
|
+
kind,
|
|
512
|
+
requestStartedAt,
|
|
513
|
+
reoffer,
|
|
514
|
+
};
|
|
515
|
+
if (discardExpiredCandidate(candidate, false)) {
|
|
516
|
+
claimed.add(order.run);
|
|
218
517
|
continue;
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
518
|
+
}
|
|
519
|
+
if (!reoffer && remaining <= 0) {
|
|
520
|
+
pendingCandidates.set(order.run, candidate);
|
|
521
|
+
opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
|
|
522
|
+
}
|
|
523
|
+
else if (!reoffer && agentRoom <= 0) {
|
|
524
|
+
pendingCandidates.set(order.run, candidate);
|
|
525
|
+
opts.out(`[${wf}/${order.run}] at the agent-run cap (${maxAgents}) — queued for local dispatch`);
|
|
526
|
+
}
|
|
527
|
+
else {
|
|
528
|
+
if (!reoffer) {
|
|
529
|
+
remaining--;
|
|
530
|
+
agentRoom--;
|
|
230
531
|
}
|
|
231
|
-
|
|
232
|
-
agentRoom--;
|
|
532
|
+
candidates.push(candidate);
|
|
233
533
|
}
|
|
234
|
-
candidates.push({ order, workflow: wf, step, defName, defHash: bundle?.def.hash, kind: 'agent', reoffer });
|
|
235
534
|
claimed.add(order.run);
|
|
236
535
|
}
|
|
237
536
|
}
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
// Dispatch by kind.
|
|
241
|
-
for (const c of candidates) {
|
|
242
|
-
if (c.kind === 'command') {
|
|
243
|
-
try {
|
|
244
|
-
const { pid } = opts.spawner({ workflow: c.workflow, run: c.order.run });
|
|
245
|
-
const rec = { workflow: c.workflow, run: c.order.run, pid, spawnedAt: opts.now(), kind: 'exec' };
|
|
246
|
-
writeChildRecord(opts.stateDir, rec);
|
|
247
|
-
dispatched++;
|
|
248
|
-
emit({
|
|
249
|
-
type: 'dispatched',
|
|
250
|
-
workflow: c.workflow,
|
|
251
|
-
run: c.order.run,
|
|
252
|
-
step: c.order.step,
|
|
253
|
-
kind: 'exec',
|
|
254
|
-
pid,
|
|
255
|
-
});
|
|
256
|
-
opts.out(`dispatched command ${c.workflow}/${c.order.run} (step '${c.order.step}', pid ${pid})`);
|
|
257
|
-
}
|
|
258
|
-
catch (e) {
|
|
259
|
-
const message = errMsg(e);
|
|
260
|
-
emit({
|
|
261
|
-
type: 'failed',
|
|
262
|
-
workflow: c.workflow,
|
|
263
|
-
run: c.order.run,
|
|
264
|
-
step: c.order.step,
|
|
265
|
-
kind: 'exec',
|
|
266
|
-
message,
|
|
267
|
-
});
|
|
268
|
-
opts.err(`spawn for ${c.workflow}/${c.order.run} failed: ${message}`);
|
|
269
|
-
}
|
|
537
|
+
for (const candidate of candidates) {
|
|
538
|
+
if (discardExpiredCandidate(candidate, false))
|
|
270
539
|
continue;
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
// This is the ONLY agent path — no flag, no alternative.
|
|
274
|
-
try {
|
|
275
|
-
const { pid } = opts.spawner({
|
|
276
|
-
workflow: c.workflow,
|
|
277
|
-
run: c.order.run,
|
|
278
|
-
kind: 'agent-run',
|
|
279
|
-
...(c.step?.harness !== undefined && c.step.harness !== '' ? { harness: c.step.harness } : {}),
|
|
280
|
-
});
|
|
281
|
-
const rec = {
|
|
282
|
-
workflow: c.workflow,
|
|
283
|
-
run: c.order.run,
|
|
284
|
-
pid,
|
|
285
|
-
spawnedAt: opts.now(),
|
|
286
|
-
kind: 'agent-run',
|
|
287
|
-
...(c.defName !== undefined ? { def: c.defName } : {}),
|
|
288
|
-
...(c.defHash !== undefined ? { hash: c.defHash } : {}),
|
|
289
|
-
step: c.order.step,
|
|
290
|
-
};
|
|
291
|
-
writeChildRecord(opts.stateDir, rec);
|
|
540
|
+
const result = dispatchCandidate(candidate);
|
|
541
|
+
if (result === 'dispatched') {
|
|
292
542
|
dispatched++;
|
|
293
|
-
|
|
294
|
-
type: 'dispatched',
|
|
295
|
-
workflow: c.workflow,
|
|
296
|
-
run: c.order.run,
|
|
297
|
-
step: c.order.step,
|
|
298
|
-
kind: 'agent-run',
|
|
299
|
-
pid,
|
|
300
|
-
});
|
|
301
|
-
opts.out(`dispatched agent-run ${c.workflow}/${c.order.run} (step '${c.order.step}', pid ${pid})`);
|
|
543
|
+
continue;
|
|
302
544
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
step: c.order.step,
|
|
310
|
-
kind: 'agent-run',
|
|
311
|
-
message,
|
|
312
|
-
});
|
|
313
|
-
opts.err(`agent-run spawn for ${c.workflow}/${c.order.run} failed: ${message}`);
|
|
545
|
+
if (result === 'total-capacity' || result === 'agent-capacity') {
|
|
546
|
+
pendingCandidates.set(candidate.order.run, candidate);
|
|
547
|
+
const limit = result === 'total-capacity'
|
|
548
|
+
? `dispatch cap (${cap})`
|
|
549
|
+
: `agent-run cap (${opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS})`;
|
|
550
|
+
opts.out(`[${candidate.workflow}/${candidate.order.run}] lost a shared-capacity race at the ${limit} — queued for local dispatch`);
|
|
314
551
|
}
|
|
315
552
|
}
|
|
316
|
-
return { dispatched, polled, openRuns };
|
|
553
|
+
return { dispatched, polled, openRuns, complete };
|
|
317
554
|
}
|
|
318
555
|
/**
|
|
319
556
|
* PHASE 4 — remove the per-run work directories of runs that are over.
|
|
@@ -367,8 +604,20 @@ export function createShiftLoop(opts) {
|
|
|
367
604
|
opts.out(`reaped work directory ${dir}`);
|
|
368
605
|
}
|
|
369
606
|
async function iteration() {
|
|
607
|
+
const backoffActiveAtStart = monotonicNow() < backoffUntil;
|
|
608
|
+
let rateLimitedThisIteration = false;
|
|
370
609
|
// Presence when due (starts immediately — this shift exists to conduct).
|
|
371
|
-
|
|
610
|
+
// A server backoff suppresses every hub poll, including presence, while local
|
|
611
|
+
// reconciliation and queued dispatch continue below.
|
|
612
|
+
if (!backoffActiveAtStart &&
|
|
613
|
+
monotonicNow() - lastPresence >= opts.presenceIntervalMs) {
|
|
614
|
+
// `setShift` and `noteAttended` force the next ping by setting
|
|
615
|
+
// lastPresence to -Infinity. Either can land WHILE this ping is awaiting
|
|
616
|
+
// the hub, so record the generation first: on completion, only stamp the
|
|
617
|
+
// cadence if no force arrived in the meantime. Stamping unconditionally
|
|
618
|
+
// would swallow that sentinel and defer the newly recorded attendance or
|
|
619
|
+
// identity by a full presenceIntervalMs.
|
|
620
|
+
const generation = ++presenceGeneration;
|
|
372
621
|
try {
|
|
373
622
|
await opts.hub.presencePing({
|
|
374
623
|
name: shiftName,
|
|
@@ -377,32 +626,58 @@ export function createShiftLoop(opts) {
|
|
|
377
626
|
...(opts.startedAt !== undefined ? { started_at: opts.startedAt } : {}),
|
|
378
627
|
...(attendedAt !== undefined ? { attended_at: attendedAt } : {}),
|
|
379
628
|
});
|
|
380
|
-
|
|
629
|
+
if (generation === presenceGeneration)
|
|
630
|
+
lastPresence = monotonicNow();
|
|
381
631
|
}
|
|
382
632
|
catch (e) {
|
|
633
|
+
rateLimitedThisIteration = noteServerBackoff(e);
|
|
383
634
|
opts.err(`presence ping failed: ${errMsg(e)} (continuing)`);
|
|
384
635
|
}
|
|
385
636
|
}
|
|
386
|
-
// Reconcile in-flight
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
637
|
+
// Reconcile in-flight first. A child exit is local state, not a hub event,
|
|
638
|
+
// so use the freed slot to dispatch claims retained from an earlier sweep
|
|
639
|
+
// before consulting the wake cursor.
|
|
640
|
+
let inFlight = reconcile();
|
|
641
|
+
let dispatched = drainPending(inFlight.live, inFlight.reserved);
|
|
642
|
+
if (dispatched > 0)
|
|
643
|
+
inFlight = reconcile();
|
|
644
|
+
const live = inFlight.live;
|
|
645
|
+
const reserved = inFlight.reserved;
|
|
646
|
+
const occupied = live.length + reserved.length;
|
|
647
|
+
const k = cap - occupied;
|
|
648
|
+
// Retry-After suppresses every remaining hub call in the iteration that
|
|
649
|
+
// received it, and every hub call in later iterations before the deadline.
|
|
650
|
+
// Local child reconciliation and queued-claim dispatch above still run.
|
|
651
|
+
if (backoffActiveAtStart ||
|
|
652
|
+
rateLimitedThisIteration ||
|
|
653
|
+
monotonicNow() < backoffUntil) {
|
|
654
|
+
return dispatched;
|
|
655
|
+
}
|
|
656
|
+
// Cheap wake pre-check. A failed call also suppresses a forced retry for this
|
|
657
|
+
// tick: a rate-limited wake should not immediately be followed by whats_next.
|
|
390
658
|
let changed = false;
|
|
659
|
+
let wakeSucceeded = false;
|
|
391
660
|
try {
|
|
392
661
|
const w = await opts.hub.wake(cursor);
|
|
393
662
|
changed = w.changed;
|
|
394
663
|
cursor = w.cursor;
|
|
664
|
+
wakeSucceeded = true;
|
|
395
665
|
}
|
|
396
666
|
catch (e) {
|
|
667
|
+
noteServerBackoff(e);
|
|
397
668
|
opts.err(`wake failed: ${errMsg(e)} (retrying next tick)`);
|
|
398
669
|
}
|
|
399
|
-
//
|
|
670
|
+
// A changed cursor is not sufficient by itself: if the prior changed tick
|
|
671
|
+
// had no local capacity, or its sweep failed, the adopted cursor will be
|
|
672
|
+
// unchanged next time. `sweepOwed` preserves that unconsumed work signal.
|
|
400
673
|
let swept;
|
|
401
|
-
if (changed && k > 0) {
|
|
402
|
-
swept = await sweep(k, live);
|
|
674
|
+
if (wakeSucceeded && (changed || sweepOwed) && k > 0) {
|
|
675
|
+
swept = await sweep(k, live, reserved);
|
|
676
|
+
sweepOwed = !swept.complete;
|
|
403
677
|
}
|
|
404
|
-
else if (changed) {
|
|
405
|
-
|
|
678
|
+
else if (changed && k <= 0) {
|
|
679
|
+
sweepOwed = true;
|
|
680
|
+
opts.out(`at capacity (${occupied}/${cap} in flight) — deferring whats_next until capacity is free`);
|
|
406
681
|
}
|
|
407
682
|
const liveAfter = reconcile().live;
|
|
408
683
|
// PHASE 4 — the work-directory reaper.
|
|
@@ -415,7 +690,8 @@ export function createShiftLoop(opts) {
|
|
|
415
690
|
// orders" and delete live work. Skipping costs nothing: the next sweeping
|
|
416
691
|
// tick reaps whatever is still reapable.
|
|
417
692
|
reapWorkDirs(swept, liveAfter);
|
|
418
|
-
|
|
693
|
+
dispatched += swept?.dispatched ?? 0;
|
|
694
|
+
return dispatched;
|
|
419
695
|
}
|
|
420
696
|
async function run() {
|
|
421
697
|
ensureStateDir(opts.stateDir);
|
|
@@ -429,7 +705,8 @@ export function createShiftLoop(opts) {
|
|
|
429
705
|
await iteration();
|
|
430
706
|
if (stopped)
|
|
431
707
|
return 0;
|
|
432
|
-
|
|
708
|
+
const delay = Math.max(opts.pollIntervalMs, backoffUntil - monotonicNow());
|
|
709
|
+
await opts.sleep(delay);
|
|
433
710
|
// Re-check after the park so a stop during sleep makes no further hub call.
|
|
434
711
|
if (stopped)
|
|
435
712
|
return 0;
|
|
@@ -442,7 +719,10 @@ export function createShiftLoop(opts) {
|
|
|
442
719
|
run,
|
|
443
720
|
stop,
|
|
444
721
|
iterate: iteration,
|
|
445
|
-
freeCapacity: () =>
|
|
722
|
+
freeCapacity: () => {
|
|
723
|
+
const current = reconcile();
|
|
724
|
+
return cap - current.live.length - current.reserved.length;
|
|
725
|
+
},
|
|
446
726
|
getCap: () => cap,
|
|
447
727
|
setCap: (next) => {
|
|
448
728
|
cap = next;
|
|
@@ -454,14 +734,17 @@ export function createShiftLoop(opts) {
|
|
|
454
734
|
if (next.serveCrews !== undefined)
|
|
455
735
|
serveCrews = [...next.serveCrews];
|
|
456
736
|
lastPresence = Number.NEGATIVE_INFINITY; // D6: next iterate() pings immediately
|
|
737
|
+
presenceGeneration++; // survive an in-flight ping completing after this
|
|
457
738
|
return { name: shiftName, serveCrews: [...serveCrews] };
|
|
458
739
|
},
|
|
459
740
|
noteAttended: (at) => {
|
|
460
741
|
attendedAt = at;
|
|
461
742
|
lastPresence = Number.NEGATIVE_INFINITY;
|
|
743
|
+
presenceGeneration++; // survive an in-flight ping completing after this
|
|
462
744
|
},
|
|
463
745
|
getAttendedAt: () => attendedAt,
|
|
464
746
|
noteRunEnded: (run) => {
|
|
747
|
+
pendingCandidates.delete(run);
|
|
465
748
|
removeChildRecord(opts.stateDir, run);
|
|
466
749
|
},
|
|
467
750
|
};
|