owenloop 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +144 -35
- package/bin/owenloop.mjs +49 -1
- package/dist/packages/work/src/agent/brief.d.ts +29 -4
- package/dist/packages/work/src/agent/brief.js +49 -4
- package/dist/packages/work/src/agent/capability-model.d.ts +123 -0
- package/dist/packages/work/src/agent/capability-model.js +148 -0
- package/dist/packages/work/src/agent/loop.d.ts +56 -7
- package/dist/packages/work/src/agent/loop.js +166 -9
- package/dist/packages/work/src/bundle/cache.d.ts +7 -5
- package/dist/packages/work/src/bundle/cache.js +20 -6
- package/dist/packages/work/src/bundle/fetch.d.ts +7 -5
- package/dist/packages/work/src/bundle/fetch.js +27 -7
- package/dist/packages/work/src/bundle/types.d.ts +8 -4
- package/dist/packages/work/src/consumed-verifier.d.ts +45 -0
- package/dist/packages/work/src/consumed-verifier.js +205 -0
- package/dist/packages/work/src/exec/instructions.d.ts +99 -0
- package/dist/packages/work/src/exec/instructions.js +323 -0
- package/dist/packages/work/src/exec/loop.d.ts +29 -1
- package/dist/packages/work/src/exec/loop.js +313 -35
- package/dist/packages/work/src/exec/payload.d.ts +29 -0
- package/dist/packages/work/src/exec/payload.js +54 -0
- package/dist/packages/work/src/exec/receipt.d.ts +9 -6
- package/dist/packages/work/src/exec/receipt.js +12 -1
- package/dist/packages/work/src/exec/runner.d.ts +7 -1
- package/dist/packages/work/src/exec/runner.js +84 -3
- package/dist/packages/work/src/harness/child-env.js +7 -0
- package/dist/packages/work/src/harness/claude.d.ts +20 -8
- package/dist/packages/work/src/harness/claude.js +260 -58
- package/dist/packages/work/src/harness/codex.d.ts +5 -4
- package/dist/packages/work/src/harness/codex.js +169 -41
- package/dist/packages/work/src/harness/contract.d.ts +36 -11
- package/dist/packages/work/src/harness/fake.js +1 -0
- package/dist/packages/work/src/harness/jsonrpc-stdio.js +7 -0
- package/dist/packages/work/src/harness/permissions.d.ts +16 -3
- package/dist/packages/work/src/harness/permissions.js +102 -13
- package/dist/packages/work/src/harness/session-store.d.ts +30 -10
- package/dist/packages/work/src/harness/session-store.js +317 -89
- package/dist/packages/work/src/hold/mcp.d.ts +25 -5
- package/dist/packages/work/src/hold/mcp.js +134 -12
- package/dist/packages/work/src/hub/client.d.ts +8 -1
- package/dist/packages/work/src/hub/client.js +15 -1
- package/dist/packages/work/src/hub/types.d.ts +137 -33
- package/dist/packages/work/src/hub/types.js +8 -10
- package/dist/packages/work/src/mcp/server.js +1 -1
- package/dist/packages/work/src/owenloop-bin.d.ts +1 -0
- package/dist/packages/work/src/owenloop-bin.js +18 -0
- package/dist/packages/work/src/roles/agent-run.d.ts +12 -6
- package/dist/packages/work/src/roles/agent-run.js +115 -34
- package/dist/packages/work/src/roles/exec.d.ts +8 -0
- package/dist/packages/work/src/roles/exec.js +27 -2
- package/dist/packages/work/src/roles/hold.d.ts +6 -0
- package/dist/packages/work/src/roles/hold.js +83 -13
- package/dist/packages/work/src/roles/lint.d.ts +3 -4
- package/dist/packages/work/src/roles/lint.js +41 -7
- package/dist/packages/work/src/roles/settings.js +19 -2
- package/dist/packages/work/src/roles/shift.d.ts +1 -0
- package/dist/packages/work/src/roles/shift.js +36 -2
- package/dist/packages/work/src/settings/settings.d.ts +70 -1
- package/dist/packages/work/src/settings/settings.js +90 -4
- package/dist/packages/work/src/shift/logretention.d.ts +253 -0
- package/dist/packages/work/src/shift/logretention.js +407 -0
- package/dist/packages/work/src/shift/logsink.d.ts +14 -0
- package/dist/packages/work/src/shift/logsink.js +98 -0
- package/dist/packages/work/src/shift/loop.d.ts +32 -4
- package/dist/packages/work/src/shift/loop.js +687 -166
- package/dist/packages/work/src/shift/protocol.d.ts +128 -1
- package/dist/packages/work/src/shift/protocol.js +14 -0
- package/dist/packages/work/src/shift/runtime.d.ts +16 -0
- package/dist/packages/work/src/shift/runtime.js +291 -5
- package/dist/packages/work/src/shift/server.d.ts +13 -0
- package/dist/packages/work/src/shift/server.js +61 -36
- package/dist/packages/work/src/shift/spawn.d.ts +115 -19
- package/dist/packages/work/src/shift/spawn.js +234 -30
- package/dist/packages/work/src/shift/state.d.ts +76 -49
- package/dist/packages/work/src/shift/state.js +350 -103
- package/dist/packages/work/src/shift/truncate.d.ts +35 -0
- package/dist/packages/work/src/shift/truncate.js +73 -0
- package/dist/packages/work/src/submit-proof.d.ts +61 -0
- package/dist/packages/work/src/submit-proof.js +119 -0
- package/dist/packages/work/src/submit-value.d.ts +48 -0
- package/dist/packages/work/src/submit-value.js +118 -0
- package/dist/packages/work/src/usage.d.ts +1 -1
- package/dist/packages/work/src/usage.js +11 -0
- package/dist/src/add.d.ts +64 -259
- package/dist/src/add.js +81 -615
- package/dist/src/archive.d.ts +97 -0
- package/dist/src/archive.js +591 -0
- package/dist/src/bundle/index.d.ts +64 -0
- package/dist/src/bundle/index.js +508 -0
- package/dist/src/bundle/manifest.d.ts +50 -0
- package/dist/src/bundle/manifest.js +435 -0
- package/dist/src/bundle/runtime.d.ts +26 -0
- package/dist/src/bundle/runtime.js +64 -0
- package/dist/src/bundle/tar.d.ts +62 -0
- package/dist/src/bundle/tar.js +238 -0
- package/dist/src/bundle/types.d.ts +105 -0
- package/dist/src/bundle/types.js +21 -0
- package/dist/src/capabilities.d.ts +93 -0
- package/dist/src/capabilities.js +101 -0
- package/dist/src/cli.d.ts +90 -20
- package/dist/src/cli.js +2421 -374
- package/dist/src/credentials.d.ts +4 -3
- package/dist/src/credentials.js +21 -19
- package/dist/src/crypto/allowed-signers.d.ts +68 -0
- package/dist/src/crypto/allowed-signers.js +307 -0
- package/dist/src/crypto/canonical.d.ts +8 -0
- package/dist/src/crypto/canonical.js +92 -0
- package/dist/src/crypto/chain.d.ts +65 -0
- package/dist/src/crypto/chain.js +377 -0
- package/dist/src/crypto/dsse.d.ts +209 -0
- package/dist/src/crypto/dsse.js +282 -0
- package/dist/src/crypto/enrollment.d.ts +78 -0
- package/dist/src/crypto/enrollment.js +186 -0
- package/dist/src/crypto/index.d.ts +48 -0
- package/dist/src/crypto/index.js +44 -0
- package/dist/src/crypto/keys.d.ts +258 -0
- package/dist/src/crypto/keys.js +908 -0
- package/dist/src/crypto/org-root.d.ts +33 -0
- package/dist/src/crypto/org-root.js +78 -0
- package/dist/src/crypto/origin-rules.d.ts +56 -0
- package/dist/src/crypto/origin-rules.js +143 -0
- package/dist/src/crypto/policy-floor.d.ts +98 -0
- package/dist/src/crypto/policy-floor.js +353 -0
- package/dist/src/crypto/records.d.ts +319 -0
- package/dist/src/crypto/records.js +168 -0
- package/dist/src/crypto/scope.d.ts +49 -0
- package/dist/src/crypto/scope.js +83 -0
- package/dist/src/crypto/ssh.d.ts +171 -0
- package/dist/src/crypto/ssh.js +396 -0
- package/dist/src/crypto/submission.d.ts +33 -0
- package/dist/src/crypto/submission.js +47 -0
- package/dist/src/crypto/trust-roots.d.ts +29 -0
- package/dist/src/crypto/trust-roots.js +39 -0
- package/dist/src/crypto/verify-consumed.d.ts +67 -0
- package/dist/src/crypto/verify-consumed.js +259 -0
- package/dist/src/crypto/verify-origin.d.ts +50 -0
- package/dist/src/crypto/verify-origin.js +151 -0
- package/dist/src/crypto/verify-publication.d.ts +60 -0
- package/dist/src/crypto/verify-publication.js +150 -0
- package/dist/src/defs.d.ts +24 -1
- package/dist/src/defs.js +335 -27
- package/dist/src/engine.d.ts +211 -6
- package/dist/src/engine.js +459 -59
- package/dist/src/factory.d.ts +16 -0
- package/dist/src/factory.js +16 -4
- package/dist/src/global-config.d.ts +59 -0
- package/dist/src/global-config.js +84 -0
- package/dist/src/hub.d.ts +102 -32
- package/dist/src/hub.js +147 -58
- package/dist/src/index.d.ts +19 -2
- package/dist/src/index.js +23 -2
- package/dist/src/install.d.ts +538 -0
- package/dist/src/install.js +1981 -0
- package/dist/src/lock.d.ts +15 -57
- package/dist/src/lock.js +251 -143
- package/dist/src/mcp/serve.d.ts +46 -22
- package/dist/src/mcp/serve.js +174 -44
- package/dist/src/mcp/server.js +12 -1
- package/dist/src/model.d.ts +51 -0
- package/dist/src/model.js +157 -1
- package/dist/src/order-resolver.d.ts +184 -0
- package/dist/src/order-resolver.js +252 -0
- package/dist/src/package-version.d.ts +10 -0
- package/dist/src/package-version.js +42 -0
- package/dist/src/paths.d.ts +14 -1
- package/dist/src/paths.js +18 -0
- package/dist/src/schemas/enrollment-grant.v1.schema.json +90 -0
- package/dist/src/schemas/index.d.ts +1433 -0
- package/dist/src/schemas/index.js +31 -0
- package/dist/src/schemas/order.v1.schema.json +120 -0
- package/dist/src/schemas/origin.v1.schema.json +68 -0
- package/dist/src/schemas/policy-floor.v1.schema.json +29 -0
- package/dist/src/schemas/publication.v1.schema.json +31 -0
- package/dist/src/schemas/revocation.v1.schema.json +39 -0
- package/dist/src/schemas/submission.v1.schema.json +61 -0
- package/dist/src/store/def-source.d.ts +52 -0
- package/dist/src/store/def-source.js +335 -0
- package/dist/src/store/index-file.d.ts +69 -0
- package/dist/src/store/index-file.js +151 -0
- package/dist/src/store/index.d.ts +27 -0
- package/dist/src/store/index.js +22 -0
- package/dist/src/store/ingestor.d.ts +43 -0
- package/dist/src/store/ingestor.js +211 -0
- package/dist/src/store/install.d.ts +228 -0
- package/dist/src/store/install.js +555 -0
- package/dist/src/store/instruction-source.d.ts +49 -0
- package/dist/src/store/instruction-source.js +298 -0
- package/dist/src/store/origin-evidence.d.ts +21 -0
- package/dist/src/store/origin-evidence.js +66 -0
- package/dist/src/store/pre-commit-verifier.d.ts +68 -0
- package/dist/src/store/pre-commit-verifier.js +418 -0
- package/dist/src/store/resolve.d.ts +138 -0
- package/dist/src/store/resolve.js +363 -0
- package/dist/src/store/types.d.ts +288 -0
- package/dist/src/store/types.js +356 -0
- package/dist/src/store.d.ts +33 -1
- package/dist/src/store.js +57 -4
- package/dist/src/types.d.ts +146 -12
- package/dist/src/untar.d.ts +8 -23
- package/dist/src/untar.js +10 -136
- package/docs/agent-runner.md +86 -0
- package/docs/authoring.md +64 -6
- package/docs/bundles.md +350 -0
- package/docs/cli.md +1298 -106
- package/docs/crypto.md +832 -0
- package/docs/design.md +342 -20
- package/docs/embedding.md +132 -25
- package/docs/shift-logs.md +642 -0
- package/docs/wire-contracts.md +395 -0
- package/package.json +6 -1
- package/plugins/claude-code/.claude-plugin/marketplace.json +14 -0
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +8 -0
- package/plugins/claude-code/plugin/.mcp.json +14 -0
- package/plugins/claude-code/plugin/hooks/hooks.json +26 -0
- package/plugins/claude-code/plugin/hooks/session-end.sh +40 -0
- package/plugins/claude-code/plugin/hooks/session-start.sh +52 -0
- package/plugins/claude-code/plugin/skills/author/SKILL.md +181 -0
- package/plugins/claude-code/plugin/skills/conduct/SKILL.md +106 -0
- package/plugins/claude-code/plugin/skills/shift/SKILL.md +105 -0
- package/plugins/codex/.agents/plugins/marketplace.json +21 -0
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +7 -0
- package/plugins/codex/plugins/owenloop/.mcp.json +20 -0
- package/plugins/codex/plugins/owenloop/hooks/hooks.json +26 -0
- package/plugins/codex/plugins/owenloop/hooks/session-end.sh +40 -0
- package/plugins/codex/plugins/owenloop/hooks/session-start.sh +52 -0
- package/plugins/codex/plugins/owenloop/skills/author/SKILL.md +181 -0
- package/plugins/codex/plugins/owenloop/skills/conduct/SKILL.md +106 -0
- package/plugins/codex/plugins/owenloop/skills/shift/SKILL.md +105 -0
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
* `createShiftLoop` is a standing Shift loop with every side-effecting
|
|
5
5
|
* dependency injected (hub client, spawner, sleep/clock, output streams, dirs),
|
|
6
6
|
* so the same core can run behind the local shift daemon. Per iteration it: pings
|
|
7
|
-
* presence when due →
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* the poll interval.
|
|
7
|
+
* presence when due → reconciles local children → dispatches any already-claimed
|
|
8
|
+
* orders waiting for local capacity → checks `wake(cursor)` → sweeps `whats_next`
|
|
9
|
+
* when the hub changed or a prior sweep was skipped/failed → classifies/meters
|
|
10
|
+
* new orders → reaps work dirs → adopts the cursor → sleeps the poll interval.
|
|
11
11
|
*
|
|
12
12
|
* DISPATCH SPLIT (D2 decision 3 — the D12 routing contract). BOTH kinds spawn a
|
|
13
13
|
* detached child; there is exactly ONE way each kind runs, and no flag selects
|
|
@@ -15,9 +15,9 @@
|
|
|
15
15
|
* - COMMAND orders spawn a detached `owenloop work exec <wf>/<run> --origin <url>`
|
|
16
16
|
* child that self-leases (unchanged from C3).
|
|
17
17
|
* - AGENT orders spawn a detached `owenloop work agent-run <wf>/<run>` child that
|
|
18
|
-
* hosts the step agent itself through a harness adapter.
|
|
19
|
-
*
|
|
20
|
-
*
|
|
18
|
+
* hosts the step agent itself through a harness adapter. The child resolves the
|
|
19
|
+
* order-pinned instruction digest and selects the verified step and harness;
|
|
20
|
+
* Shift never substitutes definition-name cache metadata for modern orders.
|
|
21
21
|
* - The shift makes NO first-contact `get_order` for any kind. It holds no
|
|
22
22
|
* leases; the spawned child makes first contact and closes the B2 pickup
|
|
23
23
|
* window. A failed hand-off lapses back via that window instead of sitting
|
|
@@ -25,24 +25,24 @@
|
|
|
25
25
|
*
|
|
26
26
|
* METERING (decision 3): free capacity `k = cap − liveInFlight`, where
|
|
27
27
|
* liveInFlight counts every live child record, each one pid-probed (see
|
|
28
|
-
* state.ts). When `k <= 0` the loop skips `whats_next`
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
28
|
+
* state.ts). When `k <= 0` the loop skips `whats_next` but remembers that the
|
|
29
|
+
* adopted wake still needs a sweep. Orders already returned by `whats_next`
|
|
30
|
+
* are already claimed, so over-cap orders enter an in-memory queue and dispatch
|
|
31
|
+
* as soon as local child capacity frees; waiting for another hub wake would
|
|
32
|
+
* strand those claims until pickup expiry.
|
|
32
33
|
*
|
|
33
34
|
* RESILIENCE: presence/wake/whats_next failures are logged and never kill the
|
|
34
35
|
* loop. `stop()` flips a flag checked between awaits; the in-flight sweep
|
|
35
36
|
* finishes, `run()` resolves 0, and no hub call is made afterward. Detached
|
|
36
37
|
* children are never killed on shutdown — that is the drain semantic.
|
|
37
38
|
*
|
|
38
|
-
* PINNED-HASH DISPATCH (E, DD-4):
|
|
39
|
-
*
|
|
40
|
-
* `readDispatchBundle(cacheDir, defName)
|
|
41
|
-
* →
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
* flows through `ChildRecord.hash` untouched.
|
|
39
|
+
* LEGACY PINNED-HASH DISPATCH (E, DD-4): the old wire shape omits both `worker`
|
|
40
|
+
* and `defDigest` and serves only a def NAME. Only that explicitly legacy shape
|
|
41
|
+
* uses `readDispatchBundle(cacheDir, defName)`: 0 pins → latest; exactly 1 pinned
|
|
42
|
+
* hash → that frozen version; >1 conflicting pins → refuse and leave the orders
|
|
43
|
+
* for the pickup window. Modern orders route from authoritative `worker` plus
|
|
44
|
+
* exact `defDigest`; modern command metadata is verified through
|
|
45
|
+
* `resolveOrderStep`, and modern agent metadata is resolved by `agent-run`.
|
|
46
46
|
*
|
|
47
47
|
* LIVE SHIFT IDENTITY: `name`/`serveCrews` on `ShiftLoopOptions` are INITIAL
|
|
48
48
|
* values only. The loop holds them as live closure state (`getShift`/`setShift`
|
|
@@ -51,20 +51,103 @@
|
|
|
51
51
|
* `setShift` also resets the presence timer so the new identity reaches the hub
|
|
52
52
|
* on the very next `iterate()`.
|
|
53
53
|
*/
|
|
54
|
+
import { join } from 'node:path';
|
|
55
|
+
import { performance } from 'node:perf_hooks';
|
|
56
|
+
import { acquireFileLockSync, releaseFileLock } from "../../../../src/lock.js";
|
|
54
57
|
import { readDispatchBundle } from "../bundle/cache.js";
|
|
58
|
+
import { HubError } from "../hub/types.js";
|
|
55
59
|
import { isCommandStep, resolveCommandRouting } from "./routing.js";
|
|
56
|
-
import { reconcileInFlight,
|
|
60
|
+
import { cancelReservedChild, finalizeChildReservation, reconcileInFlight, reserveChild, startReservedChild, removeChildRecord, ensureStateDir, } from "./state.js";
|
|
57
61
|
import { DEFAULT_WORK_DIR_TTL_MS, sweepWorkDirs as sweepWorkDirsImpl } from "../agent/workdir.js";
|
|
58
62
|
import { sessionsPath } from "../harness/session-store.js";
|
|
63
|
+
import { stampShiftEvent, } from "./protocol.js";
|
|
59
64
|
/** The `maxConcurrentAgents` fallback when the option is absent. */
|
|
60
65
|
const DEFAULT_MAX_AGENTS = 4;
|
|
66
|
+
/**
|
|
67
|
+
* The hub reaps a never-contacted claim after 120 seconds. Stop local queueing
|
|
68
|
+
* after 90 seconds so a detached child retains 30 seconds to make first contact.
|
|
69
|
+
*/
|
|
70
|
+
export const MAX_PENDING_CANDIDATE_AGE_MS = 90_000;
|
|
71
|
+
/**
|
|
72
|
+
* THE PER-STEP FAILURE BRAKE.
|
|
73
|
+
*
|
|
74
|
+
* Every in-flight guard in this file keys on the RUN id — `liveRuns`,
|
|
75
|
+
* `workerRuns`, `claimed`, `pendingCandidates`. That is correct for the case
|
|
76
|
+
* they were written for: one claim, one child. It is blind to the case where a
|
|
77
|
+
* step fails the same way forever.
|
|
78
|
+
*
|
|
79
|
+
* Observed 2026-08-12: `local-check` of one instance was dispatched five-plus
|
|
80
|
+
* times in seconds under a DIFFERENT run id each time
|
|
81
|
+
* (run_fd94d88a…, run_5ec4d807…, run_d8cdf03f…, run_67c8e2db…, run_072755ba…).
|
|
82
|
+
* The child refused its consumed artifact, exited non-zero WITHOUT submitting,
|
|
83
|
+
* the hub released the claim, and the next poll handed the same step back under
|
|
84
|
+
* a fresh run id. A fresh run id matches none of the run-keyed guards, so
|
|
85
|
+
* nothing bounded the respawns.
|
|
86
|
+
*
|
|
87
|
+
* So the brake is keyed on the identity that actually repeats — the STEP, as
|
|
88
|
+
* `(workflow, step, key)`. `key` is part of the identity because a fanned-out
|
|
89
|
+
* step has one legitimately concurrent order per key, and those must not brake
|
|
90
|
+
* each other.
|
|
91
|
+
*
|
|
92
|
+
* WHAT IS COUNTED, AND WHY IT IS NOT DISPATCHES. The event counted against that
|
|
93
|
+
* key is a WORKER FAILURE — a child that exited non-zero, reported into the
|
|
94
|
+
* loop through `noteWorkerFailure` — never a dispatch. Counting dispatches
|
|
95
|
+
* conflates a storm with legitimate concurrency: the hub can offer several
|
|
96
|
+
* DISTINCT runs of one step in a single sweep, and metering the second through
|
|
97
|
+
* fifth of those would throttle healthy work. A failure is direct evidence that
|
|
98
|
+
* running this step again right now is unlikely to help; a dispatch is evidence
|
|
99
|
+
* of nothing.
|
|
100
|
+
*
|
|
101
|
+
* It is a rate limit, never a ban. A step that has not failed is dispatched
|
|
102
|
+
* with zero delay, however many orders arrive for it at once.
|
|
103
|
+
*/
|
|
104
|
+
const STEP_BRAKE_DELAYS_MS = [2_000, 8_000, 30_000, 120_000, 300_000];
|
|
105
|
+
/**
|
|
106
|
+
* Treat a failure this long after the previous one as the start of a NEW
|
|
107
|
+
* streak, resetting the count to zero first. Comfortably longer than the
|
|
108
|
+
* longest delay in the table (300s), so a step failing at maximum backoff never
|
|
109
|
+
* decays out of it, while an isolated transient long after an old streak starts
|
|
110
|
+
* again at the shortest delay instead of inheriting a stale penalty.
|
|
111
|
+
*/
|
|
112
|
+
const STEP_BRAKE_DECAY_MS = 600_000;
|
|
113
|
+
/**
|
|
114
|
+
* Forget a step's brake entry once this long has passed since its window
|
|
115
|
+
* closed, so the map cannot grow for the life of the shift. Pure housekeeping —
|
|
116
|
+
* `STEP_BRAKE_DECAY_MS` is what governs behaviour.
|
|
117
|
+
*/
|
|
118
|
+
const STEP_BRAKE_FORGET_MS = 1_800_000;
|
|
119
|
+
/**
|
|
120
|
+
* How long to refuse the next dispatch of a step after `count` consecutive
|
|
121
|
+
* worker failures. `count` is at least 1 at every call site (this is reached
|
|
122
|
+
* only from the failure path), so the FIRST failure already costs 2s; past the
|
|
123
|
+
* end of the table the last entry repeats, so a permanently-failing step
|
|
124
|
+
* settles at one dispatch per five minutes instead of one per poll.
|
|
125
|
+
*/
|
|
126
|
+
function stepBrakeDelayMs(count) {
|
|
127
|
+
const index = Math.min(count, STEP_BRAKE_DELAYS_MS.length) - 1;
|
|
128
|
+
return STEP_BRAKE_DELAYS_MS[index] ?? 0;
|
|
129
|
+
}
|
|
130
|
+
/** The identity a storm repeats on. `key` is '' for an unfanned step. */
|
|
131
|
+
function stepBrakeKey(workflow, step, key) {
|
|
132
|
+
return `${workflow}\u0000${step}\u0000${key ?? ''}`;
|
|
133
|
+
}
|
|
61
134
|
function errMsg(e) {
|
|
62
135
|
return e instanceof Error ? e.message : String(e);
|
|
63
136
|
}
|
|
137
|
+
/** The explicit hub refusal produced when an inbox row becomes terminal after
|
|
138
|
+
* the inbox snapshot but before its targeted serving call. This is a normal
|
|
139
|
+
* stale-candidate race, not a Shift failure worth retrying forever. */
|
|
140
|
+
function isNonServableRace(e) {
|
|
141
|
+
return e instanceof HubError
|
|
142
|
+
&& e.status === 403
|
|
143
|
+
&& e.code === 'forbidden'
|
|
144
|
+
&& e.message.includes('non-running instance is not servable');
|
|
145
|
+
}
|
|
64
146
|
export function createShiftLoop(opts) {
|
|
65
147
|
let stopped = false;
|
|
66
148
|
let cap = opts.cap;
|
|
67
149
|
const isAlive = opts.isAlive;
|
|
150
|
+
const monotonicNow = opts.monotonicNow ?? performance.now.bind(performance);
|
|
68
151
|
// Live shift identity (MCP `clock_in`, D3-D7 of the plan). Seeded from opts,
|
|
69
152
|
// which are now INITIAL values only. Arrays are copied so neither the loop
|
|
70
153
|
// nor a caller of getShift/setShift can mutate the other's state afterward.
|
|
@@ -74,20 +157,67 @@ export function createShiftLoop(opts) {
|
|
|
74
157
|
let cursor;
|
|
75
158
|
let lastPresence = Number.NEGATIVE_INFINITY;
|
|
76
159
|
let attendedAt;
|
|
77
|
-
|
|
160
|
+
/**
|
|
161
|
+
* Orders returned by `whats_next` are already claimed. If the hub returns more
|
|
162
|
+
* orders than this process can spawn, retain those claims locally until a child
|
|
163
|
+
* exits and capacity opens. Waiting for another wake cannot work: no hub event
|
|
164
|
+
* is required when a local child exits, and `tick` will not re-return a claim
|
|
165
|
+
* that is already in flight.
|
|
166
|
+
*/
|
|
167
|
+
const pendingCandidates = new Map();
|
|
168
|
+
/** A changed wake whose sweep was skipped or failed must be retried after the
|
|
169
|
+
* cursor is adopted; otherwise the next unchanged wake hides that work. */
|
|
170
|
+
let sweepOwed = false;
|
|
171
|
+
/**
|
|
172
|
+
* Has the current at-capacity EPISODE already produced a `capacity` event?
|
|
173
|
+
*
|
|
174
|
+
* Set when the event fires, cleared the moment a slot is free again, so one
|
|
175
|
+
* unbroken stretch at capacity yields exactly one record no matter how many
|
|
176
|
+
* ticks it spans. See the emit site for why the console line next to it is
|
|
177
|
+
* deliberately NOT edge-triggered.
|
|
178
|
+
*/
|
|
179
|
+
let atCapReported = false;
|
|
180
|
+
/** Earliest server-approved time for the next polling iteration. */
|
|
181
|
+
let backoffUntil = Number.NEGATIVE_INFINITY;
|
|
182
|
+
/** Bumped whenever a ping starts or is forced due, so a ping that completes
|
|
183
|
+
* after a force does not stamp the cadence over that force. */
|
|
184
|
+
let presenceGeneration = 0;
|
|
185
|
+
function noteServerBackoff(error) {
|
|
186
|
+
if (!(error instanceof HubError) || error.status !== 429)
|
|
187
|
+
return false;
|
|
188
|
+
const delay = error.retryAfterMs ?? opts.pollIntervalMs;
|
|
189
|
+
backoffUntil = Math.max(backoffUntil, monotonicNow() + delay);
|
|
190
|
+
return true;
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Fan one observation to every consumer, stamped.
|
|
194
|
+
*
|
|
195
|
+
* THE SINGLE ENVELOPE-STAMPING POINT for everything the loop observes. Call
|
|
196
|
+
* sites build a `ShiftEventBody` — payload only — and this function adds the
|
|
197
|
+
* timestamp and the shift identity, so no construction site can forget them
|
|
198
|
+
* and no consumer can receive a half-stamped record. `shiftName` is read live
|
|
199
|
+
* rather than captured, because `clock_in` can rename a shift mid-run and the
|
|
200
|
+
* record should say what the shift was called when it was written.
|
|
201
|
+
*
|
|
202
|
+
* A THROWING CONSUMER MUST NOT KILL THE SHIFT. The report goes to `opts.err`
|
|
203
|
+
* as free text and stays free text: it is the sink's own failure report, and
|
|
204
|
+
* turning it into an event would write it to the sink that just failed.
|
|
205
|
+
*/
|
|
206
|
+
function emit(body) {
|
|
78
207
|
if (opts.onEvent === undefined)
|
|
79
208
|
return;
|
|
80
209
|
try {
|
|
81
|
-
opts.onEvent(
|
|
210
|
+
opts.onEvent(stampShiftEvent(body, { name: shiftName, id: opts.shiftId ?? '' }, opts.now()));
|
|
82
211
|
}
|
|
83
212
|
catch (e) {
|
|
84
213
|
opts.err(`shift event sink failed: ${errMsg(e)} (continuing)`);
|
|
85
214
|
}
|
|
86
215
|
}
|
|
87
216
|
function reconcile() {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
217
|
+
const result = reconcileInFlight(opts.stateDir, {
|
|
218
|
+
...(isAlive !== undefined ? { isAlive } : {}),
|
|
219
|
+
now: opts.now(),
|
|
220
|
+
});
|
|
91
221
|
for (const rec of result.reaped) {
|
|
92
222
|
emit({
|
|
93
223
|
type: 'reaped',
|
|
@@ -97,8 +227,210 @@ export function createShiftLoop(opts) {
|
|
|
97
227
|
pid: rec.pid,
|
|
98
228
|
});
|
|
99
229
|
}
|
|
230
|
+
for (const reservation of result.abandoned) {
|
|
231
|
+
opts.err(`[${reservation.workflow}/${reservation.run}] abandoned ${reservation.childKind} dispatch reservation expired and was cancelled`);
|
|
232
|
+
}
|
|
100
233
|
return result;
|
|
101
234
|
}
|
|
235
|
+
/**
|
|
236
|
+
* Per-step failure history for the brake. Keyed by `(workflow, step, key)` —
|
|
237
|
+
* see `stepBrakeKey`. `count` is the current consecutive-failure streak for
|
|
238
|
+
* that step, `failedAt` the monotonic instant of the most recent failure (the
|
|
239
|
+
* input to the `STEP_BRAKE_DECAY_MS` streak reset), and `nextAllowedAt` the
|
|
240
|
+
* monotonic instant the next spawn of that step becomes permitted.
|
|
241
|
+
*/
|
|
242
|
+
const stepBrake = new Map();
|
|
243
|
+
/**
|
|
244
|
+
* Brake key of the step each dispatched run belongs to.
|
|
245
|
+
*
|
|
246
|
+
* A `WorkerFailure` carries the run id and the step name but NOT the fan-out
|
|
247
|
+
* key, and the brake identity includes the key — so the mapping is recorded
|
|
248
|
+
* here at dispatch, when the whole order is in hand, rather than
|
|
249
|
+
* reconstructed from the failure. Written synchronously in
|
|
250
|
+
* `dispatchCandidate` immediately after the spawn returns, which always
|
|
251
|
+
* precedes the child's `exit` event (that fires on a later tick), so a
|
|
252
|
+
* failure can never arrive before its own entry exists.
|
|
253
|
+
*
|
|
254
|
+
* Entries are removed when the run's failure is noted, when the run ends
|
|
255
|
+
* (`noteRunEnded`), and for any run no longer live at sweep time — so this
|
|
256
|
+
* map tracks live dispatches only and cannot grow without bound.
|
|
257
|
+
*/
|
|
258
|
+
const runBrakeKey = new Map();
|
|
259
|
+
/**
|
|
260
|
+
* Recheck shared capacity and create the per-run reservation while holding one
|
|
261
|
+
* state-directory-wide lock. The lock ends before spawn: the durable
|
|
262
|
+
* reservation, not a long critical section, protects capacity during spawn.
|
|
263
|
+
*/
|
|
264
|
+
function reserveCandidate(c) {
|
|
265
|
+
const childKind = c.kind === 'command' ? 'exec' : 'agent-run';
|
|
266
|
+
const dispatchLock = acquireFileLockSync(join(opts.stateDir, '.dispatch.lock'), {
|
|
267
|
+
waitMs: 30_000,
|
|
268
|
+
label: 'owenloop Shift dispatch',
|
|
269
|
+
});
|
|
270
|
+
try {
|
|
271
|
+
const fresh = reconcileInFlight(opts.stateDir, {
|
|
272
|
+
...(isAlive === undefined ? {} : { isAlive }),
|
|
273
|
+
now: opts.now(),
|
|
274
|
+
});
|
|
275
|
+
if (fresh.live.some((record) => record.run === c.order.run) ||
|
|
276
|
+
fresh.reserved.some((record) => record.run === c.order.run))
|
|
277
|
+
return 'duplicate';
|
|
278
|
+
if (fresh.live.length + fresh.reserved.length >= cap)
|
|
279
|
+
return 'total-capacity';
|
|
280
|
+
if (childKind === 'agent-run') {
|
|
281
|
+
const agents = fresh.live.filter((record) => record.kind === 'agent-run').length +
|
|
282
|
+
fresh.reserved.filter((record) => record.childKind === 'agent-run').length;
|
|
283
|
+
if (agents >= (opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS))
|
|
284
|
+
return 'agent-capacity';
|
|
285
|
+
}
|
|
286
|
+
return reserveChild(opts.stateDir, {
|
|
287
|
+
workflow: c.workflow,
|
|
288
|
+
run: c.order.run,
|
|
289
|
+
reservedAt: opts.now(),
|
|
290
|
+
childKind,
|
|
291
|
+
step: c.order.step,
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
finally {
|
|
295
|
+
releaseFileLock(dispatchLock);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
function dispatchCandidate(c) {
|
|
299
|
+
const childKind = c.kind === 'command' ? 'exec' : 'agent-run';
|
|
300
|
+
// THE BRAKE. This function is the single choke point every spawn passes
|
|
301
|
+
// through — both the sweep's dispatch loop and `drainPending` reach a child
|
|
302
|
+
// only through here — so gating it here cannot be bypassed by either path.
|
|
303
|
+
// Braking BEFORE `reserveCandidate` means no reservation, no state-dir
|
|
304
|
+
// write and no lock contention for an order this shift will not run. The
|
|
305
|
+
// claim already taken by `whats_next` is deliberately left to lapse: the
|
|
306
|
+
// hub reaps a never-contacted claim after 120s, which throttles the
|
|
307
|
+
// re-offer rate on the hub side too.
|
|
308
|
+
const brakeKey = stepBrakeKey(c.workflow, c.order.step, c.order.key);
|
|
309
|
+
const brake = stepBrake.get(brakeKey);
|
|
310
|
+
if (brake !== undefined && monotonicNow() < brake.nextAllowedAt) {
|
|
311
|
+
const waitMs = Math.round(brake.nextAllowedAt - monotonicNow());
|
|
312
|
+
opts.err(`[${c.workflow}/${c.order.run}] step '${c.order.step}' has failed ` +
|
|
313
|
+
`${brake.count} time(s) in a row — braking for ${waitMs}ms and ` +
|
|
314
|
+
`leaving this claim to lapse`);
|
|
315
|
+
return 'braked';
|
|
316
|
+
}
|
|
317
|
+
let reservation;
|
|
318
|
+
let cancel;
|
|
319
|
+
try {
|
|
320
|
+
const reserved = reserveCandidate(c);
|
|
321
|
+
if (typeof reserved === 'string')
|
|
322
|
+
return reserved;
|
|
323
|
+
reservation = reserved.reservation;
|
|
324
|
+
const spawned = opts.spawner({
|
|
325
|
+
workflow: c.workflow,
|
|
326
|
+
run: c.order.run,
|
|
327
|
+
step: c.order.step,
|
|
328
|
+
...(childKind === 'agent-run' ? { kind: 'agent-run' } : {}),
|
|
329
|
+
startGate: reserved.gatePath,
|
|
330
|
+
});
|
|
331
|
+
cancel = spawned.cancel ?? spawned.terminate;
|
|
332
|
+
const rec = finalizeChildReservation(opts.stateDir, reservation, {
|
|
333
|
+
pid: spawned.pid,
|
|
334
|
+
spawnedAt: opts.now(),
|
|
335
|
+
kind: childKind,
|
|
336
|
+
...(childKind === 'agent-run' && c.defName !== undefined ? { def: c.defName } : {}),
|
|
337
|
+
...(childKind === 'agent-run' && c.defHash !== undefined ? { hash: c.defHash } : {}),
|
|
338
|
+
...(childKind === 'agent-run' ? { step: c.order.step } : {}),
|
|
339
|
+
});
|
|
340
|
+
startReservedChild(opts.stateDir, rec);
|
|
341
|
+
// Remember which step this run belongs to, so a later worker failure —
|
|
342
|
+
// which names the run but not the fan-out key — can be charged to the
|
|
343
|
+
// right brake key. NOTHING is counted here: a dispatch is not evidence of
|
|
344
|
+
// trouble, and counting it would meter the legitimately concurrent runs
|
|
345
|
+
// of one step that a single sweep can offer.
|
|
346
|
+
runBrakeKey.set(c.order.run, brakeKey);
|
|
347
|
+
emit({
|
|
348
|
+
type: 'dispatched',
|
|
349
|
+
workflow: c.workflow,
|
|
350
|
+
run: c.order.run,
|
|
351
|
+
step: c.order.step,
|
|
352
|
+
kind: childKind,
|
|
353
|
+
pid: spawned.pid,
|
|
354
|
+
});
|
|
355
|
+
opts.out(`dispatched ${c.kind === 'command' ? 'command' : 'agent-run'} ${c.workflow}/${c.order.run} ` +
|
|
356
|
+
`(step '${c.order.step}', pid ${spawned.pid})`);
|
|
357
|
+
return 'dispatched';
|
|
358
|
+
}
|
|
359
|
+
catch (e) {
|
|
360
|
+
cancel?.();
|
|
361
|
+
if (reservation !== undefined) {
|
|
362
|
+
try {
|
|
363
|
+
cancelReservedChild(opts.stateDir, reservation);
|
|
364
|
+
}
|
|
365
|
+
catch (cleanupError) {
|
|
366
|
+
opts.err(`[${c.workflow}/${c.order.run}] failed to cancel dispatch reservation: ${errMsg(cleanupError)}`);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
const message = errMsg(e);
|
|
370
|
+
emit({
|
|
371
|
+
type: 'failed',
|
|
372
|
+
workflow: c.workflow,
|
|
373
|
+
run: c.order.run,
|
|
374
|
+
step: c.order.step,
|
|
375
|
+
kind: childKind,
|
|
376
|
+
message,
|
|
377
|
+
});
|
|
378
|
+
const prefix = c.kind === 'command' ? 'spawn' : 'agent-run spawn';
|
|
379
|
+
opts.err(`${prefix} for ${c.workflow}/${c.order.run} failed: ${message}`);
|
|
380
|
+
return 'failed';
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
function discardExpiredCandidate(candidate, queued) {
|
|
384
|
+
if (monotonicNow() - candidate.requestStartedAt < MAX_PENDING_CANDIDATE_AGE_MS)
|
|
385
|
+
return false;
|
|
386
|
+
const prefix = queued ? 'queued claim' : 'claim';
|
|
387
|
+
opts.err(`[${candidate.workflow}/${candidate.order.run}] ${prefix} expired before local dispatch — leaving the hub pickup window to re-offer it`);
|
|
388
|
+
return true;
|
|
389
|
+
}
|
|
390
|
+
/** Dispatch already-claimed orders when local child capacity becomes free. */
|
|
391
|
+
function drainPending(live, reserved) {
|
|
392
|
+
let remaining = cap - live.length - reserved.length;
|
|
393
|
+
if (remaining <= 0 || pendingCandidates.size === 0)
|
|
394
|
+
return 0;
|
|
395
|
+
const maxAgents = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
|
|
396
|
+
let agentRoom = maxAgents -
|
|
397
|
+
live.filter((r) => r.kind === 'agent-run').length -
|
|
398
|
+
reserved.filter((r) => r.childKind === 'agent-run').length;
|
|
399
|
+
const liveRuns = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
|
|
400
|
+
let dispatched = 0;
|
|
401
|
+
for (const [run, candidate] of pendingCandidates) {
|
|
402
|
+
if (remaining <= 0)
|
|
403
|
+
break;
|
|
404
|
+
if (discardExpiredCandidate(candidate, true)) {
|
|
405
|
+
pendingCandidates.delete(run);
|
|
406
|
+
continue;
|
|
407
|
+
}
|
|
408
|
+
if (liveRuns.has(run)) {
|
|
409
|
+
pendingCandidates.delete(run);
|
|
410
|
+
continue;
|
|
411
|
+
}
|
|
412
|
+
// The agent lane's second budget can be full while the dispatch cap still
|
|
413
|
+
// has room. Skip this entry and KEEP it queued (a command entry further
|
|
414
|
+
// down the map is still dispatchable) — `continue`, never `break`, so one
|
|
415
|
+
// blocked agent does not head-of-line-block the whole queue.
|
|
416
|
+
if (candidate.kind === 'agent' && agentRoom <= 0)
|
|
417
|
+
continue;
|
|
418
|
+
const result = dispatchCandidate(candidate);
|
|
419
|
+
if (result === 'total-capacity')
|
|
420
|
+
break;
|
|
421
|
+
if (result === 'agent-capacity')
|
|
422
|
+
continue;
|
|
423
|
+
pendingCandidates.delete(run);
|
|
424
|
+
if (result !== 'dispatched')
|
|
425
|
+
continue;
|
|
426
|
+
dispatched++;
|
|
427
|
+
remaining--;
|
|
428
|
+
liveRuns.add(run);
|
|
429
|
+
if (candidate.kind === 'agent')
|
|
430
|
+
agentRoom--;
|
|
431
|
+
}
|
|
432
|
+
return dispatched;
|
|
433
|
+
}
|
|
102
434
|
/**
|
|
103
435
|
* One sweep. Collects dispatch candidates across the target instance(s)
|
|
104
436
|
* within the free-capacity `budget`, then dispatches by kind: a command order
|
|
@@ -106,8 +438,30 @@ export function createShiftLoop(opts) {
|
|
|
106
438
|
* `owenloop work agent-run` child. Returns how many children were spawned, plus the
|
|
107
439
|
* polled/open-run sets the work-dir reaper needs.
|
|
108
440
|
*/
|
|
109
|
-
async function sweep(budget, live) {
|
|
441
|
+
async function sweep(budget, live, reserved) {
|
|
110
442
|
let dispatched = 0;
|
|
443
|
+
// Housekeeping for both brake maps, once per sweep rather than on write, so
|
|
444
|
+
// neither can grow for the life of a long-lived shift.
|
|
445
|
+
//
|
|
446
|
+
// `stepBrake`: forget an entry whose window closed STEP_BRAKE_FORGET_MS
|
|
447
|
+
// ago. Behaviour is already governed by STEP_BRAKE_DECAY_MS, which resets a
|
|
448
|
+
// stale streak; this only reclaims the memory.
|
|
449
|
+
//
|
|
450
|
+
// `runBrakeKey`: drop every run that is no longer live. A run whose child
|
|
451
|
+
// exited CLEANLY reports no failure, so nothing else would ever remove its
|
|
452
|
+
// entry. A non-zero exit is charged by `noteWorkerFailure`, which the
|
|
453
|
+
// child's own `exit` event drives on the next event-loop tick — always well
|
|
454
|
+
// before the next sweep, which waits a whole poll interval. If that order
|
|
455
|
+
// ever did invert, the cost is one uncharged failure, never a wrong charge.
|
|
456
|
+
for (const [key, entry] of stepBrake) {
|
|
457
|
+
if (monotonicNow() - entry.nextAllowedAt > STEP_BRAKE_FORGET_MS)
|
|
458
|
+
stepBrake.delete(key);
|
|
459
|
+
}
|
|
460
|
+
const liveOrReserved = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
|
|
461
|
+
for (const run of runBrakeKey.keys()) {
|
|
462
|
+
if (!liveOrReserved.has(run))
|
|
463
|
+
runBrakeKey.delete(run);
|
|
464
|
+
}
|
|
111
465
|
/**
|
|
112
466
|
* PHASE 4, for the work-dir reaper only. `polled` is the set of workflows
|
|
113
467
|
* whose `whats_next` SUCCEEDED this sweep — a workflow whose call threw is
|
|
@@ -120,13 +474,16 @@ export function createShiftLoop(opts) {
|
|
|
120
474
|
*/
|
|
121
475
|
const polled = new Set();
|
|
122
476
|
const openRuns = new Set();
|
|
477
|
+
let complete = true;
|
|
123
478
|
let remaining = budget;
|
|
124
|
-
const liveRuns = new Set(live.map((r) => r.run));
|
|
125
|
-
// The agent lane's SECOND budget (D7). Counted from
|
|
126
|
-
//
|
|
127
|
-
//
|
|
479
|
+
const liveRuns = new Set([...live.map((r) => r.run), ...reserved.map((r) => r.run)]);
|
|
480
|
+
// The agent lane's SECOND budget (D7). Counted from live records and durable
|
|
481
|
+
// reservations, so a crash between reservation and spawn cannot overbook it.
|
|
482
|
+
// A child or reservation that is reaped frees the slot on the next reconcile.
|
|
128
483
|
const maxAgents = opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS;
|
|
129
|
-
let agentRoom = maxAgents -
|
|
484
|
+
let agentRoom = maxAgents -
|
|
485
|
+
live.filter((r) => r.kind === 'agent-run').length -
|
|
486
|
+
reserved.filter((r) => r.childKind === 'agent-run').length;
|
|
130
487
|
/**
|
|
131
488
|
* Runs that ALREADY have a live `agent-run` child. An agent-run child is a
|
|
132
489
|
* PROCESS: re-dispatching one would put a second harness session on a single
|
|
@@ -134,7 +491,10 @@ export function createShiftLoop(opts) {
|
|
|
134
491
|
* re-offer for a run a worker already holds is skipped outright, the way the
|
|
135
492
|
* command lane does it.
|
|
136
493
|
*/
|
|
137
|
-
const workerRuns = new Set(
|
|
494
|
+
const workerRuns = new Set([
|
|
495
|
+
...live.filter((r) => r.kind === 'agent-run').map((r) => r.run),
|
|
496
|
+
...reserved.filter((r) => r.childKind === 'agent-run').map((r) => r.run),
|
|
497
|
+
]);
|
|
138
498
|
// Resolve which instances to poll.
|
|
139
499
|
let instances;
|
|
140
500
|
if (opts.workflow !== undefined) {
|
|
@@ -146,20 +506,59 @@ export function createShiftLoop(opts) {
|
|
|
146
506
|
instances = (inbox.instances ?? []).map((i) => i.workflow);
|
|
147
507
|
}
|
|
148
508
|
catch (e) {
|
|
509
|
+
noteServerBackoff(e);
|
|
149
510
|
opts.err(`inbox whats_next failed: ${errMsg(e)}`);
|
|
150
|
-
|
|
511
|
+
// THE THIRD HUB-CALL FAILURE PATH, and the most consequential: this one
|
|
512
|
+
// aborts the WHOLE sweep, because the shift never learned which
|
|
513
|
+
// workflows to poll. The targeted `whats_next` below loses one workflow
|
|
514
|
+
// and the `wake` above loses one tick; this loses everything. It is
|
|
515
|
+
// recorded on the same terms as the other two, with `workflow` absent —
|
|
516
|
+
// that omission is what distinguishes the untargeted inbox call from a
|
|
517
|
+
// per-workflow one, since `HubErrorEvent.workflow` is optional.
|
|
518
|
+
emit({ type: 'hub-error', op: 'whats_next', message: errMsg(e) });
|
|
519
|
+
return { dispatched, polled, openRuns, complete: false };
|
|
151
520
|
}
|
|
152
521
|
}
|
|
153
522
|
// Collect candidates, metering NEW dispatches and deduping in-flight.
|
|
154
523
|
const claimed = new Set();
|
|
155
524
|
const candidates = [];
|
|
525
|
+
/**
|
|
526
|
+
* Refuse one order and leave it for the hub pickup window.
|
|
527
|
+
*
|
|
528
|
+
* Every refusal below is a DROPPED UNIT OF WORK, not a debug aside: the
|
|
529
|
+
* shift declines an order the hub already handed it, and until now the only
|
|
530
|
+
* trace was a stderr line a dispatched shift discards. One helper, so the
|
|
531
|
+
* console text and the record cannot drift apart, and so a future refusal
|
|
532
|
+
* cannot be added as text-only by accident. `reason` is the stable machine
|
|
533
|
+
* discriminator; `message` is the same human text as before, unchanged.
|
|
534
|
+
*/
|
|
535
|
+
const dropOrder = (workflow, order, reason, message) => {
|
|
536
|
+
opts.err(`[${workflow}/${order.run}] ${message}`);
|
|
537
|
+
emit({ type: 'order-dropped', workflow, run: order.run, step: order.step, reason, message });
|
|
538
|
+
claimed.add(order.run);
|
|
539
|
+
};
|
|
156
540
|
for (const wf of instances) {
|
|
157
541
|
let res;
|
|
542
|
+
const requestStartedAt = monotonicNow();
|
|
158
543
|
try {
|
|
159
544
|
res = await opts.hub.whatsNext({ workflow: wf, serve_crews: serveCrews });
|
|
160
545
|
}
|
|
161
546
|
catch (e) {
|
|
547
|
+
if (isNonServableRace(e)) {
|
|
548
|
+
// Treat the targeted call as a successful empty observation for the
|
|
549
|
+
// work-dir reaper and consume this wake. A later hub event may list
|
|
550
|
+
// new work, but this stale terminal row must not force every poll to
|
|
551
|
+
// repeat the same ForbiddenError.
|
|
552
|
+
polled.add(wf);
|
|
553
|
+
opts.out(`[${wf}] skipped stale terminal inbox candidate`);
|
|
554
|
+
continue;
|
|
555
|
+
}
|
|
556
|
+
const rateLimited = noteServerBackoff(e);
|
|
557
|
+
complete = false;
|
|
162
558
|
opts.err(`whats_next for ${wf} failed: ${errMsg(e)}`);
|
|
559
|
+
emit({ type: 'hub-error', op: 'whats_next', workflow: wf, message: errMsg(e) });
|
|
560
|
+
if (rateLimited)
|
|
561
|
+
break;
|
|
163
562
|
continue;
|
|
164
563
|
}
|
|
165
564
|
const orders = res.orders ?? [];
|
|
@@ -167,153 +566,171 @@ export function createShiftLoop(opts) {
|
|
|
167
566
|
for (const o of orders)
|
|
168
567
|
openRuns.add(o.run);
|
|
169
568
|
const defName = res.def;
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
569
|
+
const legacyOrders = orders.filter((order) => !Object.prototype.hasOwnProperty.call(order, 'worker') &&
|
|
570
|
+
!Object.prototype.hasOwnProperty.call(order, 'defDigest'));
|
|
571
|
+
// The definition-name cache exists only for the explicitly identified old
|
|
572
|
+
// wire shape. Modern orders route from `worker` and exact `defDigest`.
|
|
573
|
+
const dispatch = legacyOrders.length > 0 && defName !== undefined
|
|
574
|
+
? readDispatchBundle(opts.cacheDir, defName)
|
|
575
|
+
: { bundle: null };
|
|
173
576
|
const bundle = dispatch.bundle;
|
|
174
|
-
if (
|
|
577
|
+
if (legacyOrders.length > 0 && dispatch.warning !== undefined) {
|
|
175
578
|
opts.err(`[${wf}] ${dispatch.warning}`);
|
|
176
579
|
}
|
|
177
|
-
else if (
|
|
178
|
-
opts.err(`no cached bundle for def '${defName}' — run \`owenloop work prepare ${defName}\` (
|
|
580
|
+
else if (legacyOrders.length > 0 && bundle === null && defName !== undefined) {
|
|
581
|
+
opts.err(`no cached bundle for def '${defName}' — run \`owenloop work prepare ${defName}\` (legacy orders left for pickup)`);
|
|
582
|
+
emit({ type: 'bundle-miss', workflow: wf, def: defName });
|
|
179
583
|
}
|
|
180
584
|
for (const order of orders) {
|
|
181
585
|
if (claimed.has(order.run))
|
|
182
586
|
continue; // seen this sweep
|
|
183
|
-
|
|
184
|
-
|
|
587
|
+
if (pendingCandidates.has(order.run))
|
|
588
|
+
continue; // already claimed and queued locally
|
|
185
589
|
const reoffer = liveRuns.has(order.run);
|
|
186
|
-
|
|
187
|
-
|
|
590
|
+
const workerPresent = Object.prototype.hasOwnProperty.call(order, 'worker');
|
|
591
|
+
const digestPresent = Object.prototype.hasOwnProperty.call(order, 'defDigest');
|
|
592
|
+
const legacy = !workerPresent && !digestPresent;
|
|
593
|
+
let step;
|
|
594
|
+
let kind;
|
|
595
|
+
let candidateDefName;
|
|
596
|
+
let candidateDefHash;
|
|
597
|
+
if (legacy) {
|
|
598
|
+
step = bundle?.def.steps.find((candidate) => candidate.name === order.step);
|
|
599
|
+
if (step === undefined || bundle === null)
|
|
600
|
+
continue;
|
|
601
|
+
kind = isCommandStep(step) ? 'command' : 'agent';
|
|
602
|
+
candidateDefName = defName;
|
|
603
|
+
candidateDefHash = bundle.def.hash;
|
|
604
|
+
}
|
|
605
|
+
else {
|
|
606
|
+
if (typeof order.defDigest !== 'string' || order.defDigest.trim() === '') {
|
|
607
|
+
dropOrder(wf, order, 'malformed-digest', 'malformed modern work order: non-empty defDigest is required whenever modern routing metadata is present — leaving for manual pickup');
|
|
608
|
+
continue;
|
|
609
|
+
}
|
|
610
|
+
// The deployed projection omits `worker` for the default executor. That
|
|
611
|
+
// shape is a modern AGENT order because `defDigest` pins the exact step;
|
|
612
|
+
// only an explicitly authored executor produces `worker` on the wire.
|
|
613
|
+
if (!workerPresent || order.worker === 'agent') {
|
|
614
|
+
kind = 'agent';
|
|
615
|
+
}
|
|
616
|
+
else if (order.worker === 'command') {
|
|
617
|
+
kind = 'command';
|
|
618
|
+
try {
|
|
619
|
+
step = await opts.resolveOrderStep?.(order);
|
|
620
|
+
}
|
|
621
|
+
catch (error) {
|
|
622
|
+
dropOrder(wf, order, 'verification-failed', `exact command instructions for digest '${order.defDigest}' failed verification: ${errMsg(error)} — leaving for manual pickup`);
|
|
623
|
+
continue;
|
|
624
|
+
}
|
|
625
|
+
if (step === undefined) {
|
|
626
|
+
dropOrder(wf, order, 'metadata-unavailable', `exact command routing metadata for digest '${order.defDigest}' is unavailable — leaving for manual pickup`);
|
|
627
|
+
continue;
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
else if (typeof order.worker !== 'string' || order.worker.trim() === '') {
|
|
631
|
+
dropOrder(wf, order, 'malformed-worker', "malformed modern work order: worker must be absent, 'agent', or 'command' — leaving for manual pickup");
|
|
632
|
+
continue;
|
|
633
|
+
}
|
|
634
|
+
else {
|
|
635
|
+
dropOrder(wf, order, 'unsupported-worker', `unsupported worker '${order.worker}' — leaving for manual pickup`);
|
|
636
|
+
continue;
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
if (kind === 'command') {
|
|
640
|
+
// A command already in flight (or durably reserved) is never respawned.
|
|
188
641
|
if (reoffer)
|
|
189
642
|
continue;
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
opts.
|
|
643
|
+
if (step === undefined)
|
|
644
|
+
continue;
|
|
645
|
+
const routing = resolveCommandRouting(opts.commandRouting, step);
|
|
646
|
+
for (const warning of routing.warnings)
|
|
647
|
+
opts.err(`[${wf}/${order.run}] ${warning}`);
|
|
648
|
+
if (!routing.autoDispatch) {
|
|
649
|
+
opts.out(`[${wf}/${order.run}] command step '${order.step}' routed to manual — leaving for pickup window`);
|
|
650
|
+
continue;
|
|
651
|
+
}
|
|
652
|
+
const candidate = {
|
|
653
|
+
order,
|
|
654
|
+
workflow: wf,
|
|
655
|
+
step,
|
|
656
|
+
defName: candidateDefName,
|
|
657
|
+
defHash: candidateDefHash,
|
|
658
|
+
kind,
|
|
659
|
+
requestStartedAt,
|
|
660
|
+
reoffer,
|
|
661
|
+
};
|
|
662
|
+
if (discardExpiredCandidate(candidate, false)) {
|
|
663
|
+
claimed.add(order.run);
|
|
195
664
|
continue;
|
|
196
665
|
}
|
|
197
|
-
if (remaining <= 0)
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
666
|
+
if (remaining <= 0) {
|
|
667
|
+
pendingCandidates.set(order.run, candidate);
|
|
668
|
+
opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
|
|
669
|
+
}
|
|
670
|
+
else {
|
|
671
|
+
remaining--;
|
|
672
|
+
candidates.push(candidate);
|
|
673
|
+
}
|
|
201
674
|
claimed.add(order.run);
|
|
202
675
|
continue;
|
|
203
676
|
}
|
|
204
|
-
// Agent
|
|
205
|
-
//
|
|
677
|
+
// Agent wire routing is authoritative. The detached agent-run resolves its
|
|
678
|
+
// own exact step and harness from the order digest, never from this cache.
|
|
206
679
|
if (workerRuns.has(order.run))
|
|
207
680
|
continue;
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
681
|
+
const candidate = {
|
|
682
|
+
order,
|
|
683
|
+
workflow: wf,
|
|
684
|
+
step: legacy ? step : undefined,
|
|
685
|
+
defName: candidateDefName,
|
|
686
|
+
defHash: candidateDefHash,
|
|
687
|
+
kind,
|
|
688
|
+
requestStartedAt,
|
|
689
|
+
reoffer,
|
|
690
|
+
};
|
|
691
|
+
if (discardExpiredCandidate(candidate, false)) {
|
|
692
|
+
claimed.add(order.run);
|
|
218
693
|
continue;
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
694
|
+
}
|
|
695
|
+
if (!reoffer && remaining <= 0) {
|
|
696
|
+
pendingCandidates.set(order.run, candidate);
|
|
697
|
+
opts.out(`[${wf}/${order.run}] at the dispatch cap (${cap}) — queued for local dispatch`);
|
|
698
|
+
}
|
|
699
|
+
else if (!reoffer && agentRoom <= 0) {
|
|
700
|
+
pendingCandidates.set(order.run, candidate);
|
|
701
|
+
opts.out(`[${wf}/${order.run}] at the agent-run cap (${maxAgents}) — queued for local dispatch`);
|
|
702
|
+
}
|
|
703
|
+
else {
|
|
704
|
+
if (!reoffer) {
|
|
705
|
+
remaining--;
|
|
706
|
+
agentRoom--;
|
|
230
707
|
}
|
|
231
|
-
|
|
232
|
-
agentRoom--;
|
|
708
|
+
candidates.push(candidate);
|
|
233
709
|
}
|
|
234
|
-
candidates.push({ order, workflow: wf, step, defName, defHash: bundle?.def.hash, kind: 'agent', reoffer });
|
|
235
710
|
claimed.add(order.run);
|
|
236
711
|
}
|
|
237
712
|
}
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
//
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
const { pid } = opts.spawner({ workflow: c.workflow, run: c.order.run });
|
|
245
|
-
const rec = { workflow: c.workflow, run: c.order.run, pid, spawnedAt: opts.now(), kind: 'exec' };
|
|
246
|
-
writeChildRecord(opts.stateDir, rec);
|
|
247
|
-
dispatched++;
|
|
248
|
-
emit({
|
|
249
|
-
type: 'dispatched',
|
|
250
|
-
workflow: c.workflow,
|
|
251
|
-
run: c.order.run,
|
|
252
|
-
step: c.order.step,
|
|
253
|
-
kind: 'exec',
|
|
254
|
-
pid,
|
|
255
|
-
});
|
|
256
|
-
opts.out(`dispatched command ${c.workflow}/${c.order.run} (step '${c.order.step}', pid ${pid})`);
|
|
257
|
-
}
|
|
258
|
-
catch (e) {
|
|
259
|
-
const message = errMsg(e);
|
|
260
|
-
emit({
|
|
261
|
-
type: 'failed',
|
|
262
|
-
workflow: c.workflow,
|
|
263
|
-
run: c.order.run,
|
|
264
|
-
step: c.order.step,
|
|
265
|
-
kind: 'exec',
|
|
266
|
-
message,
|
|
267
|
-
});
|
|
268
|
-
opts.err(`spawn for ${c.workflow}/${c.order.run} failed: ${message}`);
|
|
269
|
-
}
|
|
713
|
+
// 'braked' and 'failed' are deliberately NOT queued into pendingCandidates:
|
|
714
|
+
// both mean this shift is not going to run that order now, and re-queuing a
|
|
715
|
+
// braked candidate would just re-dispatch it on the next drain, defeating
|
|
716
|
+
// the brake. The claim lapses through the hub's pickup window instead.
|
|
717
|
+
for (const candidate of candidates) {
|
|
718
|
+
if (discardExpiredCandidate(candidate, false))
|
|
270
719
|
continue;
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
// This is the ONLY agent path — no flag, no alternative.
|
|
274
|
-
try {
|
|
275
|
-
const { pid } = opts.spawner({
|
|
276
|
-
workflow: c.workflow,
|
|
277
|
-
run: c.order.run,
|
|
278
|
-
kind: 'agent-run',
|
|
279
|
-
...(c.step?.harness !== undefined && c.step.harness !== '' ? { harness: c.step.harness } : {}),
|
|
280
|
-
});
|
|
281
|
-
const rec = {
|
|
282
|
-
workflow: c.workflow,
|
|
283
|
-
run: c.order.run,
|
|
284
|
-
pid,
|
|
285
|
-
spawnedAt: opts.now(),
|
|
286
|
-
kind: 'agent-run',
|
|
287
|
-
...(c.defName !== undefined ? { def: c.defName } : {}),
|
|
288
|
-
...(c.defHash !== undefined ? { hash: c.defHash } : {}),
|
|
289
|
-
step: c.order.step,
|
|
290
|
-
};
|
|
291
|
-
writeChildRecord(opts.stateDir, rec);
|
|
720
|
+
const result = dispatchCandidate(candidate);
|
|
721
|
+
if (result === 'dispatched') {
|
|
292
722
|
dispatched++;
|
|
293
|
-
|
|
294
|
-
type: 'dispatched',
|
|
295
|
-
workflow: c.workflow,
|
|
296
|
-
run: c.order.run,
|
|
297
|
-
step: c.order.step,
|
|
298
|
-
kind: 'agent-run',
|
|
299
|
-
pid,
|
|
300
|
-
});
|
|
301
|
-
opts.out(`dispatched agent-run ${c.workflow}/${c.order.run} (step '${c.order.step}', pid ${pid})`);
|
|
723
|
+
continue;
|
|
302
724
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
step: c.order.step,
|
|
310
|
-
kind: 'agent-run',
|
|
311
|
-
message,
|
|
312
|
-
});
|
|
313
|
-
opts.err(`agent-run spawn for ${c.workflow}/${c.order.run} failed: ${message}`);
|
|
725
|
+
if (result === 'total-capacity' || result === 'agent-capacity') {
|
|
726
|
+
pendingCandidates.set(candidate.order.run, candidate);
|
|
727
|
+
const limit = result === 'total-capacity'
|
|
728
|
+
? `dispatch cap (${cap})`
|
|
729
|
+
: `agent-run cap (${opts.maxConcurrentAgents ?? DEFAULT_MAX_AGENTS})`;
|
|
730
|
+
opts.out(`[${candidate.workflow}/${candidate.order.run}] lost a shared-capacity race at the ${limit} — queued for local dispatch`);
|
|
314
731
|
}
|
|
315
732
|
}
|
|
316
|
-
return { dispatched, polled, openRuns };
|
|
733
|
+
return { dispatched, polled, openRuns, complete };
|
|
317
734
|
}
|
|
318
735
|
/**
|
|
319
736
|
* PHASE 4 — remove the per-run work directories of runs that are over.
|
|
@@ -367,8 +784,20 @@ export function createShiftLoop(opts) {
|
|
|
367
784
|
opts.out(`reaped work directory ${dir}`);
|
|
368
785
|
}
|
|
369
786
|
async function iteration() {
|
|
787
|
+
const backoffActiveAtStart = monotonicNow() < backoffUntil;
|
|
788
|
+
let rateLimitedThisIteration = false;
|
|
370
789
|
// Presence when due (starts immediately — this shift exists to conduct).
|
|
371
|
-
|
|
790
|
+
// A server backoff suppresses every hub poll, including presence, while local
|
|
791
|
+
// reconciliation and queued dispatch continue below.
|
|
792
|
+
if (!backoffActiveAtStart &&
|
|
793
|
+
monotonicNow() - lastPresence >= opts.presenceIntervalMs) {
|
|
794
|
+
// `setShift` and `noteAttended` force the next ping by setting
|
|
795
|
+
// lastPresence to -Infinity. Either can land WHILE this ping is awaiting
|
|
796
|
+
// the hub, so record the generation first: on completion, only stamp the
|
|
797
|
+
// cadence if no force arrived in the meantime. Stamping unconditionally
|
|
798
|
+
// would swallow that sentinel and defer the newly recorded attendance or
|
|
799
|
+
// identity by a full presenceIntervalMs.
|
|
800
|
+
const generation = ++presenceGeneration;
|
|
372
801
|
try {
|
|
373
802
|
await opts.hub.presencePing({
|
|
374
803
|
name: shiftName,
|
|
@@ -377,32 +806,88 @@ export function createShiftLoop(opts) {
|
|
|
377
806
|
...(opts.startedAt !== undefined ? { started_at: opts.startedAt } : {}),
|
|
378
807
|
...(attendedAt !== undefined ? { attended_at: attendedAt } : {}),
|
|
379
808
|
});
|
|
380
|
-
|
|
809
|
+
if (generation === presenceGeneration)
|
|
810
|
+
lastPresence = monotonicNow();
|
|
381
811
|
}
|
|
382
812
|
catch (e) {
|
|
813
|
+
rateLimitedThisIteration = noteServerBackoff(e);
|
|
383
814
|
opts.err(`presence ping failed: ${errMsg(e)} (continuing)`);
|
|
384
815
|
}
|
|
385
816
|
}
|
|
386
|
-
// Reconcile in-flight
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
817
|
+
// Reconcile in-flight first. A child exit is local state, not a hub event,
|
|
818
|
+
// so use the freed slot to dispatch claims retained from an earlier sweep
|
|
819
|
+
// before consulting the wake cursor.
|
|
820
|
+
let inFlight = reconcile();
|
|
821
|
+
let dispatched = drainPending(inFlight.live, inFlight.reserved);
|
|
822
|
+
if (dispatched > 0)
|
|
823
|
+
inFlight = reconcile();
|
|
824
|
+
const live = inFlight.live;
|
|
825
|
+
const reserved = inFlight.reserved;
|
|
826
|
+
const occupied = live.length + reserved.length;
|
|
827
|
+
const k = cap - occupied;
|
|
828
|
+
// Re-arm the at-capacity report the instant a slot frees, independently of
|
|
829
|
+
// whether this tick goes on to wake or sweep. The next stretch at capacity
|
|
830
|
+
// is a NEW episode and gets its own record.
|
|
831
|
+
if (k > 0)
|
|
832
|
+
atCapReported = false;
|
|
833
|
+
// Retry-After suppresses every remaining hub call in the iteration that
|
|
834
|
+
// received it, and every hub call in later iterations before the deadline.
|
|
835
|
+
// Local child reconciliation and queued-claim dispatch above still run.
|
|
836
|
+
if (backoffActiveAtStart ||
|
|
837
|
+
rateLimitedThisIteration ||
|
|
838
|
+
monotonicNow() < backoffUntil) {
|
|
839
|
+
return dispatched;
|
|
840
|
+
}
|
|
841
|
+
// Cheap wake pre-check. A failed call also suppresses a forced retry for this
|
|
842
|
+
// tick: a rate-limited wake should not immediately be followed by whats_next.
|
|
390
843
|
let changed = false;
|
|
844
|
+
let wakeSucceeded = false;
|
|
391
845
|
try {
|
|
392
846
|
const w = await opts.hub.wake(cursor);
|
|
393
847
|
changed = w.changed;
|
|
394
848
|
cursor = w.cursor;
|
|
849
|
+
wakeSucceeded = true;
|
|
395
850
|
}
|
|
396
851
|
catch (e) {
|
|
852
|
+
noteServerBackoff(e);
|
|
397
853
|
opts.err(`wake failed: ${errMsg(e)} (retrying next tick)`);
|
|
854
|
+
emit({ type: 'hub-error', op: 'wake', message: errMsg(e) });
|
|
398
855
|
}
|
|
399
|
-
//
|
|
856
|
+
// A changed cursor is not sufficient by itself: if the prior changed tick
|
|
857
|
+
// had no local capacity, or its sweep failed, the adopted cursor will be
|
|
858
|
+
// unchanged next time. `sweepOwed` preserves that unconsumed work signal.
|
|
400
859
|
let swept;
|
|
401
|
-
if (changed && k > 0) {
|
|
402
|
-
swept = await sweep(k, live);
|
|
860
|
+
if (wakeSucceeded && (changed || sweepOwed) && k > 0) {
|
|
861
|
+
swept = await sweep(k, live, reserved);
|
|
862
|
+
sweepOwed = !swept.complete;
|
|
403
863
|
}
|
|
404
|
-
else if (changed) {
|
|
405
|
-
|
|
864
|
+
else if (changed && k <= 0) {
|
|
865
|
+
sweepOwed = true;
|
|
866
|
+
opts.out(`at capacity (${occupied}/${cap} in flight) — deferring whats_next until capacity is free`);
|
|
867
|
+
// EDGE-TRIGGERED: at most one record per at-capacity EPISODE, not per
|
|
868
|
+
// tick. At MOST, because this branch needs `changed` — an episode during
|
|
869
|
+
// which no wake reports a changed cursor produces no record at all. The
|
|
870
|
+
// absence of one therefore says nothing about occupancy; `dispatched` and
|
|
871
|
+
// `reaped` are the unconditional pair a reader should count.
|
|
872
|
+
//
|
|
873
|
+
// This branch runs on every changed wake for as long as the shift stays
|
|
874
|
+
// full, which on a busy hub is every poll interval. The console line above
|
|
875
|
+
// is a live tail an operator is watching, and repeating it is how it says
|
|
876
|
+
// "still stuck" — so it stays level-triggered and unchanged. The event has
|
|
877
|
+
// one durable consumer that repetition actively harms: `shift.log`, which
|
|
878
|
+
// this change does not rotate. Entering the state is the fact worth
|
|
879
|
+
// recording; staying in it is recoverable from the next record's `ts`.
|
|
880
|
+
//
|
|
881
|
+
// The record's ROUTING is decided elsewhere, and the two are separate
|
|
882
|
+
// concerns. `runtime.ts` withholds `capacity` from the socket entirely
|
|
883
|
+
// (`FILE_ONLY_EVENTS`) because a parked `shift next` must not be woken by
|
|
884
|
+
// news that nothing happened. Edge-triggering keeps the FILE readable;
|
|
885
|
+
// that set is what keeps the SOCKET correct. Neither substitutes for the
|
|
886
|
+
// other — the loop emits, `consumeEvent` decides who hears it.
|
|
887
|
+
if (!atCapReported) {
|
|
888
|
+
atCapReported = true;
|
|
889
|
+
emit({ type: 'capacity', inFlight: occupied, cap });
|
|
890
|
+
}
|
|
406
891
|
}
|
|
407
892
|
const liveAfter = reconcile().live;
|
|
408
893
|
// PHASE 4 — the work-directory reaper.
|
|
@@ -415,7 +900,8 @@ export function createShiftLoop(opts) {
|
|
|
415
900
|
// orders" and delete live work. Skipping costs nothing: the next sweeping
|
|
416
901
|
// tick reaps whatever is still reapable.
|
|
417
902
|
reapWorkDirs(swept, liveAfter);
|
|
418
|
-
|
|
903
|
+
dispatched += swept?.dispatched ?? 0;
|
|
904
|
+
return dispatched;
|
|
419
905
|
}
|
|
420
906
|
async function run() {
|
|
421
907
|
ensureStateDir(opts.stateDir);
|
|
@@ -429,7 +915,8 @@ export function createShiftLoop(opts) {
|
|
|
429
915
|
await iteration();
|
|
430
916
|
if (stopped)
|
|
431
917
|
return 0;
|
|
432
|
-
|
|
918
|
+
const delay = Math.max(opts.pollIntervalMs, backoffUntil - monotonicNow());
|
|
919
|
+
await opts.sleep(delay);
|
|
433
920
|
// Re-check after the park so a stop during sleep makes no further hub call.
|
|
434
921
|
if (stopped)
|
|
435
922
|
return 0;
|
|
@@ -442,7 +929,10 @@ export function createShiftLoop(opts) {
|
|
|
442
929
|
run,
|
|
443
930
|
stop,
|
|
444
931
|
iterate: iteration,
|
|
445
|
-
freeCapacity: () =>
|
|
932
|
+
freeCapacity: () => {
|
|
933
|
+
const current = reconcile();
|
|
934
|
+
return cap - current.live.length - current.reserved.length;
|
|
935
|
+
},
|
|
446
936
|
getCap: () => cap,
|
|
447
937
|
setCap: (next) => {
|
|
448
938
|
cap = next;
|
|
@@ -454,15 +944,46 @@ export function createShiftLoop(opts) {
|
|
|
454
944
|
if (next.serveCrews !== undefined)
|
|
455
945
|
serveCrews = [...next.serveCrews];
|
|
456
946
|
lastPresence = Number.NEGATIVE_INFINITY; // D6: next iterate() pings immediately
|
|
947
|
+
presenceGeneration++; // survive an in-flight ping completing after this
|
|
457
948
|
return { name: shiftName, serveCrews: [...serveCrews] };
|
|
458
949
|
},
|
|
459
950
|
noteAttended: (at) => {
|
|
460
951
|
attendedAt = at;
|
|
461
952
|
lastPresence = Number.NEGATIVE_INFINITY;
|
|
953
|
+
presenceGeneration++; // survive an in-flight ping completing after this
|
|
462
954
|
},
|
|
463
955
|
getAttendedAt: () => attendedAt,
|
|
464
956
|
noteRunEnded: (run) => {
|
|
957
|
+
pendingCandidates.delete(run);
|
|
958
|
+
// A run that ENDED is a run that progressed — the shift's MCP `submit`
|
|
959
|
+
// tool calls this only when the hub reports the submit closed the run.
|
|
960
|
+
// That clears the step's failure streak outright, so one green run wipes
|
|
961
|
+
// out an earlier storm's penalty instead of leaving it to decay.
|
|
962
|
+
const brakeKey = runBrakeKey.get(run);
|
|
963
|
+
if (brakeKey !== undefined) {
|
|
964
|
+
runBrakeKey.delete(run);
|
|
965
|
+
stepBrake.delete(brakeKey);
|
|
966
|
+
}
|
|
465
967
|
removeChildRecord(opts.stateDir, run);
|
|
466
968
|
},
|
|
969
|
+
noteWorkerFailure: (failure) => {
|
|
970
|
+
const brakeKey = runBrakeKey.get(failure.run);
|
|
971
|
+
if (brakeKey === undefined)
|
|
972
|
+
return; // not ours, or already accounted for
|
|
973
|
+
runBrakeKey.delete(failure.run);
|
|
974
|
+
const at = monotonicNow();
|
|
975
|
+
const prior = stepBrake.get(brakeKey);
|
|
976
|
+
// A failure long after the previous one starts a NEW streak rather than
|
|
977
|
+
// extending a stale one — see `STEP_BRAKE_DECAY_MS`.
|
|
978
|
+
const priorCount = prior !== undefined && at - prior.failedAt < STEP_BRAKE_DECAY_MS
|
|
979
|
+
? prior.count
|
|
980
|
+
: 0;
|
|
981
|
+
const count = priorCount + 1;
|
|
982
|
+
stepBrake.set(brakeKey, {
|
|
983
|
+
count,
|
|
984
|
+
failedAt: at,
|
|
985
|
+
nextAllowedAt: at + stepBrakeDelayMs(count),
|
|
986
|
+
});
|
|
987
|
+
},
|
|
467
988
|
};
|
|
468
989
|
}
|