owenloop 0.5.8 → 0.5.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/CHANGELOG.md +69 -0
  2. package/README.md +81 -39
  3. package/bin/owenloop.mjs +16 -1
  4. package/dist/packages/work/src/agent/loop.d.ts +23 -8
  5. package/dist/packages/work/src/agent/loop.js +55 -6
  6. package/dist/packages/work/src/bundle/pull.d.ts +21 -0
  7. package/dist/packages/work/src/bundle/pull.js +118 -0
  8. package/dist/packages/work/src/consumed-verifier.js +6 -17
  9. package/dist/packages/work/src/exec/instructions.d.ts +4 -1
  10. package/dist/packages/work/src/exec/instructions.js +3 -0
  11. package/dist/packages/work/src/exec/loop.d.ts +2 -2
  12. package/dist/packages/work/src/exec/loop.js +273 -46
  13. package/dist/packages/work/src/exec/receipt.d.ts +7 -7
  14. package/dist/packages/work/src/exec/receipt.js +6 -6
  15. package/dist/packages/work/src/harness/claude.js +116 -0
  16. package/dist/packages/work/src/harness/codex.d.ts +8 -0
  17. package/dist/packages/work/src/harness/codex.js +27 -1
  18. package/dist/packages/work/src/harness/contract.d.ts +9 -0
  19. package/dist/packages/work/src/hub/client.d.ts +19 -4
  20. package/dist/packages/work/src/hub/client.js +7 -3
  21. package/dist/packages/work/src/hub/types.d.ts +84 -0
  22. package/dist/packages/work/src/main.js +1 -0
  23. package/dist/packages/work/src/roles/agent-run.js +57 -32
  24. package/dist/packages/work/src/roles/exec.js +27 -8
  25. package/dist/packages/work/src/roles/settings.js +2 -0
  26. package/dist/packages/work/src/roles/shift.js +14 -2
  27. package/dist/packages/work/src/roles/util.d.ts +1 -0
  28. package/dist/packages/work/src/roles/util.js +10 -0
  29. package/dist/packages/work/src/settings/hub-roster-cache.d.ts +68 -0
  30. package/dist/packages/work/src/settings/hub-roster-cache.js +370 -0
  31. package/dist/packages/work/src/settings/roster.d.ts +84 -0
  32. package/dist/packages/work/src/settings/roster.js +364 -5
  33. package/dist/packages/work/src/settings/serving.d.ts +17 -0
  34. package/dist/packages/work/src/settings/serving.js +79 -0
  35. package/dist/packages/work/src/settings/settings.d.ts +14 -1
  36. package/dist/packages/work/src/settings/settings.js +14 -0
  37. package/dist/packages/work/src/shift/loop.d.ts +108 -5
  38. package/dist/packages/work/src/shift/loop.js +416 -54
  39. package/dist/packages/work/src/shift/protocol.d.ts +7 -5
  40. package/dist/packages/work/src/shift/runtime.d.ts +18 -0
  41. package/dist/packages/work/src/shift/runtime.js +104 -5
  42. package/dist/packages/work/src/shift/server.js +2 -0
  43. package/dist/packages/work/src/shift/spawn.d.ts +17 -2
  44. package/dist/packages/work/src/shift/spawn.js +16 -13
  45. package/dist/packages/work/src/shift/state.d.ts +29 -2
  46. package/dist/packages/work/src/shift/state.js +45 -15
  47. package/dist/packages/work/src/submit-proof.d.ts +11 -8
  48. package/dist/packages/work/src/submit-proof.js +11 -8
  49. package/dist/packages/work/src/usage.d.ts +1 -1
  50. package/dist/packages/work/src/usage.js +7 -1
  51. package/dist/packages/work/src/util/modifier-init.d.ts +7 -0
  52. package/dist/packages/work/src/util/modifier-init.js +73 -0
  53. package/dist/src/bundle/tar.d.ts +9 -4
  54. package/dist/src/bundle/tar.js +9 -4
  55. package/dist/src/capabilities.d.ts +76 -6
  56. package/dist/src/capabilities.js +49 -6
  57. package/dist/src/capability-mapping-client.d.ts +29 -0
  58. package/dist/src/capability-mapping-client.js +160 -0
  59. package/dist/src/cli.d.ts +16 -3
  60. package/dist/src/cli.js +1925 -281
  61. package/dist/src/config-dir.d.ts +1 -1
  62. package/dist/src/config-dir.js +1 -1
  63. package/dist/src/crypto/index.d.ts +1 -1
  64. package/dist/src/crypto/index.js +1 -1
  65. package/dist/src/crypto/org-root.d.ts +30 -12
  66. package/dist/src/crypto/org-root.js +162 -14
  67. package/dist/src/crypto/records.d.ts +2 -0
  68. package/dist/src/crypto/records.js +2 -0
  69. package/dist/src/defs.d.ts +38 -2
  70. package/dist/src/defs.js +593 -86
  71. package/dist/src/engine.d.ts +103 -25
  72. package/dist/src/engine.js +331 -79
  73. package/dist/src/hub.d.ts +77 -0
  74. package/dist/src/hub.js +162 -0
  75. package/dist/src/implements.d.ts +56 -0
  76. package/dist/src/implements.js +790 -0
  77. package/dist/src/index.d.ts +11 -9
  78. package/dist/src/index.js +6 -5
  79. package/dist/src/mcp/serve.d.ts +22 -0
  80. package/dist/src/mcp/serve.js +330 -41
  81. package/dist/src/mcp/server.d.ts +10 -7
  82. package/dist/src/mcp/server.js +53 -8
  83. package/dist/src/model.d.ts +48 -10
  84. package/dist/src/model.js +252 -99
  85. package/dist/src/order-resolver.d.ts +1 -0
  86. package/dist/src/order-resolver.js +8 -1
  87. package/dist/src/schemas/index.d.ts +11 -0
  88. package/dist/src/schemas/order.v1.schema.json +7 -1
  89. package/dist/src/store/def-source.d.ts +19 -0
  90. package/dist/src/store/def-source.js +87 -17
  91. package/dist/src/store/gc-recovery.d.ts +46 -0
  92. package/dist/src/store/gc-recovery.js +258 -0
  93. package/dist/src/store/gc.d.ts +94 -0
  94. package/dist/src/store/gc.js +562 -0
  95. package/dist/src/store/index.d.ts +3 -1
  96. package/dist/src/store/index.js +2 -0
  97. package/dist/src/store/install.d.ts +34 -5
  98. package/dist/src/store/install.js +73 -11
  99. package/dist/src/store/pre-commit-verifier.js +42 -2
  100. package/dist/src/store/snapshot-guard.d.ts +18 -0
  101. package/dist/src/store/snapshot-guard.js +94 -0
  102. package/dist/src/store.d.ts +69 -4
  103. package/dist/src/store.js +254 -16
  104. package/dist/src/types.d.ts +101 -20
  105. package/dist/src/util.js +4 -1
  106. package/docs/authoring.md +235 -10
  107. package/docs/bundles.md +45 -2
  108. package/docs/cli.md +437 -55
  109. package/docs/crypto.md +32 -10
  110. package/docs/design.md +177 -47
  111. package/docs/evals/mcp-charter-baseline.json +428 -0
  112. package/docs/mcp-charter-eval.md +124 -0
  113. package/docs/routing.md +189 -34
  114. package/docs/shift-logs.md +38 -9
  115. package/docs/wire-contracts.md +78 -14
  116. package/examples/workflows/compile-dev-playbook.yaml +1 -1
  117. package/examples/workflows/delivery.yaml +31 -14
  118. package/examples/workflows/ship.yaml +38 -4
  119. package/package.json +2 -1
  120. package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
  121. package/plugins/claude-code/plugin/.mcp.json +1 -1
  122. package/plugins/claude-code/plugin/skills/author/SKILL.md +22 -0
  123. package/plugins/claude-code/plugin/skills/ephemeral/SKILL.md +95 -0
  124. package/plugins/claude-code/plugin/skills/graduate/SKILL.md +156 -0
  125. package/plugins/claude-code/plugin/skills/plan/SKILL.md +236 -0
  126. package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
  127. package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
  128. package/plugins/codex/plugins/owenloop/skills/author/SKILL.md +22 -0
  129. package/plugins/codex/plugins/owenloop/skills/ephemeral/SKILL.md +95 -0
  130. package/plugins/codex/plugins/owenloop/skills/graduate/SKILL.md +156 -0
  131. package/plugins/codex/plugins/owenloop/skills/plan/SKILL.md +236 -0
@@ -119,10 +119,10 @@ export interface CapacityEvent {
119
119
  inFlight: number;
120
120
  cap: number;
121
121
  }
122
- /** A hub call failed. `workflow` is present only for per-workflow calls. */
122
+ /** A hub call failed. `workflow` is present for per-workflow whats_next and release calls. */
123
123
  export interface HubErrorEvent {
124
124
  type: 'hub-error';
125
- op: 'wake' | 'whats_next';
125
+ op: 'wake' | 'whats_next' | 'roster_sync' | 'release';
126
126
  workflow?: string;
127
127
  message: string;
128
128
  }
@@ -133,8 +133,9 @@ export interface BundleMissEvent {
133
133
  def: string;
134
134
  }
135
135
  /**
136
- * The shift refused one order and left it for the hub pickup window. A dropped
137
- * unit of work, which is why it is a record and not a debug aside. `reason` is
136
+ * The shift refused one order. Capacity and expiry reasons hand its claim back
137
+ * to the hub; malformed and unsupported reasons leave it for the pickup
138
+ * window. A dropped unit of work is a record, not a debug aside. `reason` is
138
139
  * the stable machine discriminator; `message` is the human text.
139
140
  */
140
141
  export interface OrderDroppedEvent {
@@ -142,7 +143,7 @@ export interface OrderDroppedEvent {
142
143
  workflow: string;
143
144
  run: string;
144
145
  step: string;
145
- reason: 'malformed-digest' | 'malformed-worker' | 'unsupported-worker' | 'verification-failed' | 'metadata-unavailable';
146
+ reason: 'malformed-digest' | 'malformed-worker' | 'unsupported-worker' | 'verification-failed' | 'metadata-unavailable' | 'agent-lane-closed' | 'dispatch-cap-full' | 'agent-cap-full' | 'claim-expired';
146
147
  message: string;
147
148
  }
148
149
  /**
@@ -205,6 +206,7 @@ export interface ShiftStatus {
205
206
  cap: number;
206
207
  free: number;
207
208
  running: number;
209
+ agent_ceiling: number;
208
210
  attended_at: number | null;
209
211
  started_at: number;
210
212
  }
@@ -1,4 +1,7 @@
1
1
  import { type ShiftEventBody } from './protocol.ts';
2
+ import { type Spawner } from './spawn.ts';
3
+ import { type Liveness, type Reconciliation } from './state.ts';
4
+ import { type AcquireFileLockOpts } from '../../../../src/lock.ts';
2
5
  import { installSignalHandlers, type SignalHost } from '../roles/signals.ts';
3
6
  export { installSignalHandlers, type SignalHost };
4
7
  /**
@@ -13,6 +16,8 @@ export declare function reachesSocketConsumer(type: ShiftEventBody['type']): boo
13
16
  export declare function resolveCap(flagCap: number | undefined, settingsCap: number | undefined): number;
14
17
  export declare function resolveStateDirOverride(flag: string | undefined, env: Record<string, string | undefined>, settingsStateDir: string | undefined): string | undefined;
15
18
  export declare function resolveMaxConcurrentAgents(flagMax: number | undefined, settingsMax: number | undefined): number;
19
+ export declare function resolveExecReserve(flagReserve: number | undefined, settingsReserve: number | undefined): number;
20
+ export declare function resolveLocalQueueHoldMs(flagHold: number | undefined, settingsHold: number | undefined): number;
16
21
  export declare function resolveShiftName(flagName: string | undefined, opts?: {
17
22
  shiftId?: string;
18
23
  hostname?: string;
@@ -29,6 +34,8 @@ export interface ParsedArgs {
29
34
  pollIntervalMs?: number;
30
35
  once?: boolean;
31
36
  maxAgents?: number;
37
+ execReserve?: number;
38
+ localQueueHoldMs?: number;
32
39
  cacheDir?: string;
33
40
  stateDir?: string;
34
41
  /** `--log-dir` — where `shift.log` and `<run>.log` are written. */
@@ -46,6 +53,15 @@ export interface ParsedArgs {
46
53
  }
47
54
  /** Parse shift's internal `--flag value` and `--flag=value` forms. */
48
55
  export declare function parseArgs(args: string[]): ParsedArgs;
56
+ /**
57
+ * Reconcile startup state under the same per-removal dispatch lock as the live
58
+ * loop. A busy neighbour is recoverable: the first poll-loop iteration retries
59
+ * shortly, whereas failing the entire Shift startup would strand its work.
60
+ */
61
+ export declare function reconcileStartupState(stateDir: string, report: (line: string) => void, options?: {
62
+ dispatchLockOptions?: Pick<AcquireFileLockOpts, 'beforeOpen'>;
63
+ isAlive?: Liveness;
64
+ }): Reconciliation | undefined;
49
65
  export interface ShiftRuntimeOptions {
50
66
  /** Build the Unix-socket shift daemon around the same loop instead of self-driving directly. */
51
67
  daemon?: boolean;
@@ -53,6 +69,8 @@ export interface ShiftRuntimeOptions {
53
69
  role?: 'shift';
54
70
  /** Socket path selected by the shift command. */
55
71
  socketPath?: string;
72
+ /** Test/embedder spawner; production uses the detached worker spawner. */
73
+ spawner?: Spawner;
56
74
  }
57
75
  /** Public Shift daemon transport is a Unix-domain socket on macOS and Linux. */
58
76
  export declare function assertShiftDaemonPlatform(platform?: NodeJS.Platform): void;
@@ -13,13 +13,17 @@ import { performance } from 'node:perf_hooks';
13
13
  import { createHubClient } from "../hub/client.js";
14
14
  import { resolveBearer } from "../credentials/resolve.js";
15
15
  import { loadSettings } from "../settings/settings.js";
16
+ import { DEFAULT_HUB_ROSTER_SYNC_TIMEOUT_MS, syncHubRosterCache, withHubRosterSyncTimeout } from "../settings/hub-roster-cache.js";
17
+ import { computeServeCapabilities } from "../settings/serving.js";
16
18
  import { resolveCacheDir } from "../bundle/cache.js";
17
- import { createShiftLoop } from "./loop.js";
19
+ import { createLockedRemovalCallbacks, createShiftLoop } from "./loop.js";
20
+ import { createHubBundleRecoveryHandler } from "../bundle/pull.js";
18
21
  import { createShiftLogSink } from "./logsink.js";
19
22
  import { prepareShiftLogDir, shiftLogFile } from "./logretention.js";
20
23
  import { stampShiftEvent } from "./protocol.js";
21
24
  import { createDefaultSpawner } from "./spawn.js";
22
25
  import { resolveStateDir, ensureStateDir, reconcileInFlight } from "./state.js";
26
+ import { FileLockTimeoutError } from "../../../../src/lock.js";
23
27
  import { reconcileActiveSessions, sessionsPath } from "../harness/session-store.js";
24
28
  import { resolveAllowedWorkdirRoots, resolveWorkRepo, resolveWorkRoot, } from "../agent/workdir.js";
25
29
  import { installSignalHandlers } from "../roles/signals.js";
@@ -31,7 +35,9 @@ export { installSignalHandlers };
31
35
  const DEFAULT_CAP = 3;
32
36
  const DEFAULT_POLL_MS = 5_000;
33
37
  const DEFAULT_MAX_AGENTS = 4;
38
+ const DEFAULT_EXEC_RESERVE = 1;
34
39
  const DEFAULT_PRESENCE_MS = 60_000;
40
+ const DEFAULT_ROSTER_SYNC_MS = 15 * 60_000;
35
41
  /**
36
42
  * Record types written to the LOG FILE but never delivered over the socket.
37
43
  *
@@ -108,6 +114,12 @@ export function resolveStateDirOverride(flag, env, settingsStateDir) {
108
114
  export function resolveMaxConcurrentAgents(flagMax, settingsMax) {
109
115
  return flagMax ?? settingsMax ?? DEFAULT_MAX_AGENTS;
110
116
  }
117
+ export function resolveExecReserve(flagReserve, settingsReserve) {
118
+ return flagReserve ?? settingsReserve ?? DEFAULT_EXEC_RESERVE;
119
+ }
120
+ export function resolveLocalQueueHoldMs(flagHold, settingsHold) {
121
+ return flagHold ?? settingsHold ?? 0;
122
+ }
111
123
  export function resolveShiftName(flagName, opts = {}) {
112
124
  if (flagName !== undefined && flagName !== '')
113
125
  return flagName;
@@ -148,6 +160,8 @@ export function parseArgs(args) {
148
160
  case '--serve-crews':
149
161
  case '--cap':
150
162
  case '--max-agents':
163
+ case '--exec-reserve':
164
+ case '--local-queue-hold':
151
165
  case '--workflow':
152
166
  case '--poll-interval':
153
167
  case '--cache-dir':
@@ -197,6 +211,18 @@ export function parseArgs(args) {
197
211
  return { error: n.error };
198
212
  parsed.maxAgents = n;
199
213
  }
214
+ else if (name === '--exec-reserve') {
215
+ const n = intFlag(r.value, '--exec-reserve');
216
+ if (typeof n !== 'number')
217
+ return { error: n.error };
218
+ parsed.execReserve = n;
219
+ }
220
+ else if (name === '--local-queue-hold') {
221
+ const n = intFlag(r.value, '--local-queue-hold');
222
+ if (typeof n !== 'number')
223
+ return { error: n.error };
224
+ parsed.localQueueHoldMs = n;
225
+ }
200
226
  else if (name === '--poll-interval') {
201
227
  const n = intFlag(r.value, '--poll-interval');
202
228
  if (typeof n !== 'number')
@@ -221,12 +247,35 @@ export function parseArgs(args) {
221
247
  function usage() {
222
248
  process.stderr.write('usage: owenloop work shift [--origin <url>] [--as <account>] [--name <n>] [--serve-crews a,b] [--cap <n>]\n' +
223
249
  ' [--workflow <id>] [--poll-interval <ms>] [--once]\n' +
224
- ' [--max-agents <n>] [--cache-dir <p>] [--state-dir <p>]\n' +
250
+ ' [--max-agents <n>] [--exec-reserve <n>] [--local-queue-hold <ms>] [--cache-dir <p>] [--state-dir <p>]\n' +
225
251
  ' [--log-dir <p>] [--log-max-age <ms>] [--work-root <dir>]...\n');
226
252
  }
227
253
  function errMsg(err) {
228
254
  return err instanceof Error ? err.message : String(err);
229
255
  }
256
+ /**
257
+ * Reconcile startup state under the same per-removal dispatch lock as the live
258
+ * loop. A busy neighbour is recoverable: the first poll-loop iteration retries
259
+ * shortly, whereas failing the entire Shift startup would strand its work.
260
+ */
261
+ export function reconcileStartupState(stateDir, report, options = {}) {
262
+ try {
263
+ return reconcileInFlight(stateDir, {
264
+ ...(options.isAlive === undefined ? {} : { isAlive: options.isAlive }),
265
+ ...createLockedRemovalCallbacks(stateDir, {
266
+ dispatchLockOptions: options.dispatchLockOptions,
267
+ waitMs: 1_000,
268
+ label: 'owenloop Shift startup reaper',
269
+ }),
270
+ });
271
+ }
272
+ catch (error) {
273
+ if (!(error instanceof FileLockTimeoutError))
274
+ throw error;
275
+ report(`startup dispatch-state reconciliation deferred: ${errMsg(error)} (the poll-loop reconciliation will retry)`);
276
+ return undefined;
277
+ }
278
+ }
230
279
  /** Public Shift daemon transport is a Unix-domain socket on macOS and Linux. */
231
280
  export function assertShiftDaemonPlatform(platform = process.platform) {
232
281
  if (platform === 'win32') {
@@ -306,7 +355,7 @@ export async function runShiftRuntime(parsed, options = {}) {
306
355
  // Reconcile the Shift's own dispatch records for capacity housekeeping. The
307
356
  // session sweep below must use the session row's PID and ownership fields,
308
357
  // not this run-id set, because every Shift on the machine shares the store.
309
- reconcileInFlight(stateDir);
358
+ reconcileStartupState(stateDir, (line) => process.stderr.write(`${roleLabel}: ${line}\n`));
310
359
  try {
311
360
  const retired = reconcileActiveSessions(sessionsPath(cacheDir), { shiftName: name, shiftOwner }, Date.now());
312
361
  for (const rec of retired) {
@@ -353,6 +402,8 @@ export async function runShiftRuntime(parsed, options = {}) {
353
402
  : undefined;
354
403
  const cap = resolveCap(parsed.cap, settings.dispatchCap);
355
404
  const maxConcurrentAgents = resolveMaxConcurrentAgents(parsed.maxAgents, settings.maxConcurrentAgents);
405
+ const execReserve = resolveExecReserve(parsed.execReserve, settings.execReserve);
406
+ const localQueueHoldMs = resolveLocalQueueHoldMs(parsed.localQueueHoldMs, settings.localQueueHoldMs);
356
407
  const workRoot = resolveWorkRoot(env, settings.workRoot, cacheDir);
357
408
  const workRepo = resolveWorkRepo(env, settings.workRepo);
358
409
  /**
@@ -384,6 +435,14 @@ export async function runShiftRuntime(parsed, options = {}) {
384
435
  projectRoot: join(process.cwd(), 'workflows'),
385
436
  globalRoot: globalStoreRoot(home),
386
437
  verifier: createBundleIngestor(),
438
+ onMissing: createHubBundleRecoveryHandler({
439
+ origin,
440
+ token,
441
+ home,
442
+ projectRoot: join(process.cwd(), 'workflows'),
443
+ env,
444
+ warn: (line) => process.stderr.write(`${roleLabel}: ${line}\n`),
445
+ }),
387
446
  });
388
447
  const resolveOrderStep = async (order) => {
389
448
  if (instructionSource === undefined ||
@@ -447,6 +506,18 @@ export async function runShiftRuntime(parsed, options = {}) {
447
506
  process.stderr.write(`${roleLabel}: shift event sink failed: ${errMsg(err)} (continuing)\n`);
448
507
  }
449
508
  };
509
+ // The agent-run child must stay completely offline. Refresh before entering
510
+ // the park loop, but let an unavailable hub degrade to machine layers. The
511
+ // durable error is buffered until after `parked`: a parked shift's first log
512
+ // record is its self-describing identity, even when this refresh fails.
513
+ let startupRosterSyncFailure;
514
+ try {
515
+ await withHubRosterSyncTimeout((signal) => syncHubRosterCache({ client: hub, env, origin, account, signal }));
516
+ }
517
+ catch (error) {
518
+ startupRosterSyncFailure = `roster sync failed at shift start: ${errMsg(error)} (continuing)`;
519
+ process.stderr.write(`${roleLabel}: ${startupRosterSyncFailure}\n`);
520
+ }
450
521
  const reportWorkerFailure = (failure) => {
451
522
  // A worker failure is detected by the SPAWNER's `exit`/`error` listener, not
452
523
  // inside the loop's sweep, so it never passes through the loop's `emit()`.
@@ -471,13 +542,16 @@ export async function runShiftRuntime(parsed, options = {}) {
471
542
  // way forever is re-dispatched on a backoff instead of once per poll.
472
543
  loopRef.current?.noteWorkerFailure(failure);
473
544
  };
474
- const spawner = createDefaultSpawner(origin, account, undefined, shiftId, reportWorkerFailure,
545
+ const reportWorkerExit = (exit) => {
546
+ loopRef.current?.noteChildExited(exit);
547
+ };
548
+ const spawner = options.spawner ?? createDefaultSpawner(origin, account, undefined, shiftId, reportWorkerFailure,
475
549
  // `undefined` when the log directory could not be created, which is what
476
550
  // makes `buildSpawnPlan` emit no `logFile` and the worker launch with its
477
551
  // output discarded exactly as it did before this change.
478
552
  logDirReady
479
553
  ? { dir: logDir, err: (line) => process.stderr.write(`${line}\n`) }
480
- : undefined, allowedWorkdirRoots, parsed.serveCrews);
554
+ : undefined, allowedWorkdirRoots, reportWorkerExit);
481
555
  const pollIntervalMs = parsed.pollIntervalMs ?? DEFAULT_POLL_MS;
482
556
  const loop = createShiftLoop({
483
557
  hub,
@@ -501,7 +575,18 @@ export async function runShiftRuntime(parsed, options = {}) {
501
575
  resolveOrderStep,
502
576
  pollIntervalMs,
503
577
  presenceIntervalMs: DEFAULT_PRESENCE_MS,
578
+ rosterSyncIntervalMs: DEFAULT_ROSTER_SYNC_MS,
579
+ rosterSyncTimeoutMs: DEFAULT_HUB_ROSTER_SYNC_TIMEOUT_MS,
580
+ syncRosters: (signal) => syncHubRosterCache({ client: hub, env, origin, account, signal }),
581
+ computeServeCapabilities: (crews) => computeServeCapabilities({
582
+ env,
583
+ crews,
584
+ hub: { origin, account },
585
+ warn: (message) => process.stderr.write(`${roleLabel}: shift: ${message}\n`),
586
+ }),
504
587
  maxConcurrentAgents,
588
+ execReserve,
589
+ localQueueHoldMs,
505
590
  workRoot,
506
591
  ...(workRepo !== undefined ? { workRepo } : {}),
507
592
  shiftId,
@@ -542,6 +627,12 @@ export async function runShiftRuntime(parsed, options = {}) {
542
627
  cwd: process.cwd(),
543
628
  }));
544
629
  };
630
+ const emitStartupRosterSyncFailure = () => {
631
+ if (startupRosterSyncFailure === undefined)
632
+ return;
633
+ consumeEvent(stamp({ type: 'hub-error', op: 'roster_sync', message: startupRosterSyncFailure }));
634
+ startupRosterSyncFailure = undefined;
635
+ };
545
636
  if (daemonMode) {
546
637
  daemon = createShiftDaemon({
547
638
  socketPath: options.socketPath ?? join(stateDir, 'shift.sock'),
@@ -567,6 +658,10 @@ export async function runShiftRuntime(parsed, options = {}) {
567
658
  if (parsed.once !== true) {
568
659
  process.stdout.write(`owenloop shift: parked as '${name}' @ ${origin} (cap ${cap})\n`);
569
660
  emitParked();
661
+ emitStartupRosterSyncFailure();
662
+ }
663
+ else {
664
+ emitStartupRosterSyncFailure();
570
665
  }
571
666
  return daemon.run();
572
667
  }
@@ -574,6 +669,10 @@ export async function runShiftRuntime(parsed, options = {}) {
574
669
  if (parsed.once !== true) {
575
670
  process.stdout.write(`owenloop work shift: parked as '${name}' @ ${origin} (cap ${cap})\n`);
576
671
  emitParked();
672
+ emitStartupRosterSyncFailure();
673
+ }
674
+ else {
675
+ emitStartupRosterSyncFailure();
577
676
  }
578
677
  return loop.run();
579
678
  }
@@ -287,6 +287,7 @@ export function createShiftDaemon(opts) {
287
287
  name: shift.name,
288
288
  serve_crews: shift.serveCrews,
289
289
  ...capacity(),
290
+ agent_ceiling: opts.loop.agentCeiling(),
290
291
  attended_at: opts.loop.getAttendedAt() ?? null,
291
292
  started_at: opts.startedAt,
292
293
  };
@@ -430,6 +431,7 @@ export function createShiftDaemon(opts) {
430
431
  await opts.hub.presencePing({
431
432
  name: shift.name,
432
433
  serve_crews: shift.serveCrews,
434
+ serve_capabilities: opts.loop.getServeCapabilities(),
433
435
  ...(opts.shiftId !== undefined ? { shift_id: opts.shiftId } : {}),
434
436
  started_at: opts.startedAt,
435
437
  // Deliberately omit attended_at. The merged hub contract interprets
@@ -68,6 +68,21 @@ export interface WorkerFailure {
68
68
  message: string;
69
69
  }
70
70
  export type WorkerFailureReporter = (failure: WorkerFailure) => void;
71
+ /**
72
+ * A dispatched worker process ENDED. Fired for every terminal outcome —
73
+ * a clean exit (including an agent-run that completed mid-turn and exited 0),
74
+ * a nonzero exit, a signal kill, and a failure to spawn. The shift frees the
75
+ * dispatch slot from this, so it must never be conditional on the exit code.
76
+ */
77
+ export interface WorkerExit {
78
+ workflow: string;
79
+ run: string;
80
+ kind: 'exec' | 'agent-run';
81
+ pid: number;
82
+ exitStatus: number | null;
83
+ signal: NodeJS.Signals | null;
84
+ }
85
+ export type WorkerExitReporter = (exit: WorkerExit) => void;
71
86
  /**
72
87
  * The stdio topology of a detached worker.
73
88
  *
@@ -137,7 +152,7 @@ export interface SpawnPlan {
137
152
  * is pure, and `createDefaultSpawner` substitutes the descriptors. Omitted ⇒ no
138
153
  * `logFile` key at all, and the plan is byte-identical to the pre-logging shape.
139
154
  */
140
- export declare function buildSpawnPlan(spec: SpawnSpec, origin: string, account: string, binPath: string, execPath?: string, shiftId?: string, logDir?: string, allowedWorkdirRoots?: string[], serveCrews?: string[]): SpawnPlan;
155
+ export declare function buildSpawnPlan(spec: SpawnSpec, origin: string, account: string, binPath: string, execPath?: string, shiftId?: string, logDir?: string, allowedWorkdirRoots?: string[]): SpawnPlan;
141
156
  /**
142
157
  * The default detached spawner. Captures the resolved hub `origin`, the
143
158
  * resolved `account`, the packaged bin path, and the dispatching Shift's
@@ -146,4 +161,4 @@ export declare function buildSpawnPlan(spec: SpawnSpec, origin: string, account:
146
161
  * relying on `buildSpawnPlan`'s default) so `shiftId` — trailing after it
147
162
  * — can be supplied positionally.
148
163
  */
149
- export declare function createDefaultSpawner(origin: string, account: string, binPath?: string, shiftId?: string, onFailure?: WorkerFailureReporter, logging?: WorkerLogOptions, allowedWorkdirRoots?: string[], serveCrews?: string[]): Spawner;
164
+ export declare function createDefaultSpawner(origin: string, account: string, binPath?: string, shiftId?: string, onFailure?: WorkerFailureReporter, logging?: WorkerLogOptions, allowedWorkdirRoots?: string[], onExit?: WorkerExitReporter): Spawner;
@@ -107,13 +107,9 @@ function err(e) {
107
107
  * is pure, and `createDefaultSpawner` substitutes the descriptors. Omitted ⇒ no
108
108
  * `logFile` key at all, and the plan is byte-identical to the pre-logging shape.
109
109
  */
110
- export function buildSpawnPlan(spec, origin, account, binPath, execPath = process.execPath, shiftId, logDir, allowedWorkdirRoots, serveCrews) {
110
+ export function buildSpawnPlan(spec, origin, account, binPath, execPath = process.execPath, shiftId, logDir, allowedWorkdirRoots) {
111
111
  const role = spec.kind === 'agent-run' ? 'agent-run' : 'exec';
112
- // `OWENLOOP_SERVE_CREWS` is Shift-owned routing plumbing, never ambient
113
- // configuration. A child for --all must not inherit a stale named-crew list
114
- // from the Shift process that launched it.
115
112
  const env = { ...process.env };
116
- delete env['OWENLOOP_SERVE_CREWS'];
117
113
  env['OWENLOOP_ACCOUNT'] = account;
118
114
  if (spec.startGate !== undefined)
119
115
  env['OWENLOOP_START_GATE'] = spec.startGate;
@@ -124,11 +120,6 @@ export function buildSpawnPlan(spec, origin, account, binPath, execPath = proces
124
120
  if (allowedWorkdirRoots !== undefined && allowedWorkdirRoots.length > 0) {
125
121
  env['OWENLOOP_ALLOWED_WORKDIR_ROOTS'] = allowedWorkdirRoots.join(':');
126
122
  }
127
- // Shift-internal roster handoff only. It is deliberately not passed to the
128
- // harness child and is absent for command workers.
129
- if (role === 'agent-run' && serveCrews !== undefined && serveCrews.length > 0) {
130
- env['OWENLOOP_SERVE_CREWS'] = serveCrews.join(',');
131
- }
132
123
  return {
133
124
  command: execPath,
134
125
  args: [
@@ -156,7 +147,7 @@ export function buildSpawnPlan(spec, origin, account, binPath, execPath = proces
156
147
  * relying on `buildSpawnPlan`'s default) so `shiftId` — trailing after it
157
148
  * — can be supplied positionally.
158
149
  */
159
- export function createDefaultSpawner(origin, account, binPath = resolveOwenloopBin(), shiftId, onFailure, logging, allowedWorkdirRoots, serveCrews) {
150
+ export function createDefaultSpawner(origin, account, binPath = resolveOwenloopBin(), shiftId, onFailure, logging, allowedWorkdirRoots, onExit) {
160
151
  // ONE report per shift, not one per dispatch. Every condition that stops a
161
152
  // worker log from opening — a full disk, a read-only log directory, an
162
153
  // operator deleting it underneath a running shift — PERSISTS across the
@@ -172,7 +163,7 @@ export function createDefaultSpawner(origin, account, binPath = resolveOwenloopB
172
163
  logging?.err?.(line);
173
164
  };
174
165
  return (spec) => {
175
- const plan = buildSpawnPlan(spec, origin, account, binPath, process.execPath, shiftId, logging?.dir, allowedWorkdirRoots, serveCrews);
166
+ const plan = buildSpawnPlan(spec, origin, account, binPath, process.execPath, shiftId, logging?.dir, allowedWorkdirRoots);
176
167
  // Open the log ONCE and hand the SAME descriptor to slots 1 and 2. Opening
177
168
  // it twice would create two independent file offsets, and the two streams
178
169
  // would overwrite each other's bytes — silent corruption no unit test on the
@@ -226,6 +217,7 @@ export function createDefaultSpawner(origin, account, binPath = resolveOwenloopB
226
217
  // discard them before an operator can read them — reading them is the whole
227
218
  // point of `<run>.log`.
228
219
  let failureReported = false;
220
+ let exitReported = false;
229
221
  const report = (exitStatus, signal, message) => {
230
222
  if (failureReported || onFailure === undefined)
231
223
  return;
@@ -241,8 +233,18 @@ export function createDefaultSpawner(origin, account, binPath = resolveOwenloopB
241
233
  message,
242
234
  });
243
235
  };
244
- child.once('error', () => report(null, null, 'worker process failed to start'));
236
+ const reportExit = (exitStatus, signal) => {
237
+ if (exitReported || onExit === undefined || child.pid === undefined)
238
+ return;
239
+ exitReported = true;
240
+ onExit({ workflow: spec.workflow, run: spec.run, kind, pid: child.pid, exitStatus, signal });
241
+ };
242
+ child.once('error', () => {
243
+ report(null, null, 'worker process failed to start');
244
+ reportExit(null, null);
245
+ });
245
246
  child.once('exit', (code, signal) => {
247
+ reportExit(code, signal);
246
248
  if (code === 0)
247
249
  return;
248
250
  report(code, signal, 'worker exited without completing successfully');
@@ -261,6 +263,7 @@ export function createDefaultSpawner(origin, account, binPath = resolveOwenloopB
261
263
  // registered above fires afterwards and emits a SECOND `failed` event —
262
264
  // two daemon events for one dispatch attempt.
263
265
  failureReported = true;
266
+ exitReported = true;
264
267
  throw new Error(`spawn of 'owenloop work ${kind} ${spec.workflow}/${spec.run}' returned no pid`);
265
268
  }
266
269
  const kill = () => {
@@ -74,13 +74,15 @@ export declare function finalizeChildReservation(stateDir: string, reservation:
74
74
  /** Open a child's gate only after the PID-bearing record is durable. */
75
75
  export declare function startReservedChild(stateDir: string, record: ChildRecord): void;
76
76
  /** Cancel a gated child and remove the matching reservation/record. */
77
- export declare function cancelReservedChild(stateDir: string, reservation: ChildReservation): void;
77
+ export declare function cancelReservedChild(stateDir: string, reservation: ChildReservation): boolean;
78
78
  /**
79
79
  * Delete one run's in-flight state. A still-gated reservation or finalized child
80
80
  * is cancelled first so an external run-ended signal cannot leave a worker gate
81
81
  * behind that later opens without a capacity record.
82
82
  */
83
- export declare function removeChildRecord(stateDir: string, run: string): void;
83
+ export declare function removeChildRecord(stateDir: string, run: string, options?: {
84
+ pid?: number;
85
+ }): boolean;
84
86
  export type Liveness = (pid: number) => boolean;
85
87
  export declare const defaultIsAlive: Liveness;
86
88
  export interface Reconciliation {
@@ -93,7 +95,32 @@ export interface ReconcileOptions {
93
95
  isAlive?: Liveness;
94
96
  now?: number;
95
97
  reservationMaxAgeMs?: number;
98
+ /**
99
+ * Called before a stale reservation is removed. The Shift loop uses this to
100
+ * share its dispatch lock with reservation creation; standalone state callers
101
+ * intentionally retain the direct removal default.
102
+ */
103
+ removeAbandonedReservation?: (reservation: ChildReservation) => boolean;
104
+ /**
105
+ * Called before a dead child record is removed. The Shift loop uses this to
106
+ * serialize the read/guard/unlink sequence with a later re-dispatch.
107
+ */
108
+ removeDeadChild?: (record: ChildRecord) => boolean;
109
+ /**
110
+ * Finish a persisted PID record's start-gate handoff. The Shift daemon uses
111
+ * this to serialize the re-read/identity-check/write sequence with a later
112
+ * re-dispatch; standalone state callers intentionally retain the direct
113
+ * settlement default.
114
+ */
115
+ settleLiveChild?: (record: ChildRecord) => ChildRecord | undefined;
96
116
  }
117
+ /**
118
+ * Open and clear a persisted child's start gate only if the record is still
119
+ * the same PID and gate-token pair the reconciler observed. A re-dispatch can
120
+ * replace either identity between an unlocked observation and this settlement;
121
+ * in that case it owns the record and nothing is changed.
122
+ */
123
+ export declare function settleChildGate(stateDir: string, record: ChildRecord): ChildRecord | undefined;
97
124
  /**
98
125
  * Reconcile live children and durable reservations. Fresh reservations count as
99
126
  * occupied capacity. An expired reservation is cancelled before its record is
@@ -297,6 +297,7 @@ export function startReservedChild(stateDir, record) {
297
297
  /** Cancel a gated child and remove the matching reservation/record. */
298
298
  export function cancelReservedChild(stateDir, reservation) {
299
299
  const path = gateFile(stateDir, reservation.token);
300
+ let removed = false;
300
301
  try {
301
302
  signalGate(path, 'cancel', true);
302
303
  }
@@ -309,22 +310,29 @@ export function cancelReservedChild(stateDir, reservation) {
309
310
  const matchesFinalized = current !== undefined &&
310
311
  !isReservation(current) &&
311
312
  current.gateToken === reservation.token;
312
- if (matchesReservation || matchesFinalized)
313
+ if (matchesReservation || matchesFinalized) {
313
314
  durableRemove(recordFile(stateDir, reservation.run));
315
+ removed = true;
316
+ }
314
317
  }
318
+ return removed;
315
319
  }
316
320
  /**
317
321
  * Delete one run's in-flight state. A still-gated reservation or finalized child
318
322
  * is cancelled first so an external run-ended signal cannot leave a worker gate
319
323
  * behind that later opens without a capacity record.
320
324
  */
321
- export function removeChildRecord(stateDir, run) {
325
+ export function removeChildRecord(stateDir, run, options) {
322
326
  const path = recordFile(stateDir, run);
323
327
  const current = readOneStateRecord(path);
324
- if (current !== undefined && isReservation(current)) {
325
- cancelReservedChild(stateDir, current);
326
- return;
327
- }
328
+ if (current === undefined)
329
+ return false;
330
+ // A late exit report must never remove a record written by a re-dispatch of
331
+ // the same run. Reservations have no pid, so they are likewise newer owners.
332
+ if (options?.pid !== undefined && (isReservation(current) || current.pid !== options.pid))
333
+ return false;
334
+ if (current !== undefined && isReservation(current))
335
+ return cancelReservedChild(stateDir, current);
328
336
  if (current?.gateToken !== undefined) {
329
337
  const gatePath = gateFile(stateDir, current.gateToken);
330
338
  try {
@@ -335,6 +343,7 @@ export function removeChildRecord(stateDir, run) {
335
343
  }
336
344
  }
337
345
  durableRemove(path);
346
+ return true;
338
347
  }
339
348
  export const defaultIsAlive = (pid) => {
340
349
  if (!Number.isInteger(pid) || pid <= 0)
@@ -347,6 +356,27 @@ export const defaultIsAlive = (pid) => {
347
356
  return error.code !== 'ESRCH';
348
357
  }
349
358
  };
359
+ /**
360
+ * Open and clear a persisted child's start gate only if the record is still
361
+ * the same PID and gate-token pair the reconciler observed. A re-dispatch can
362
+ * replace either identity between an unlocked observation and this settlement;
363
+ * in that case it owns the record and nothing is changed.
364
+ */
365
+ export function settleChildGate(stateDir, record) {
366
+ if (record.gateToken === undefined)
367
+ return record;
368
+ const current = readOneStateRecord(recordFile(stateDir, record.run));
369
+ if (current === undefined ||
370
+ isReservation(current) ||
371
+ current.pid !== record.pid ||
372
+ current.gateToken !== record.gateToken)
373
+ return undefined;
374
+ signalGate(gateFile(stateDir, current.gateToken), 'start', true);
375
+ const settled = { ...current };
376
+ delete settled.gateToken;
377
+ writeStateRecord(stateDir, settled);
378
+ return settled;
379
+ }
350
380
  /**
351
381
  * Reconcile live children and durable reservations. Fresh reservations count as
352
382
  * occupied capacity. An expired reservation is cancelled before its record is
@@ -377,23 +407,23 @@ export function reconcileInFlight(stateDir, arg) {
377
407
  reserved.push(record);
378
408
  continue;
379
409
  }
380
- cancelReservedChild(stateDir, record);
381
- abandoned.push(record);
410
+ const removed = options.removeAbandonedReservation?.(record) ?? cancelReservedChild(stateDir, record);
411
+ if (removed)
412
+ abandoned.push(record);
382
413
  continue;
383
414
  }
384
415
  if (!isAlive(record.pid)) {
385
- removeChildRecord(stateDir, record.run);
386
- reaped.push(record);
416
+ const removed = options.removeDeadChild?.(record) ?? removeChildRecord(stateDir, record.run, { pid: record.pid });
417
+ if (removed)
418
+ reaped.push(record);
387
419
  continue;
388
420
  }
389
421
  if (record.gateToken !== undefined) {
390
422
  // A restart after PID persistence but before the parent opened the gate can
391
423
  // safely finish the handoff: the durable PID record is the prerequisite.
392
- signalGate(gateFile(stateDir, record.gateToken), 'start', true);
393
- const settled = { ...record };
394
- delete settled.gateToken;
395
- writeStateRecord(stateDir, settled);
396
- live.push(settled);
424
+ const settled = options.settleLiveChild === undefined ? settleChildGate(stateDir, record) : options.settleLiveChild(record);
425
+ if (settled !== undefined)
426
+ live.push(settled);
397
427
  }
398
428
  else {
399
429
  live.push(record);
@@ -38,14 +38,17 @@ export declare function buildSubmitProof(opts: SubmitProofOptions): Promise<stri
38
38
  * the proof against.
39
39
  *
40
40
  * Retry-safety comes from the hub, not from this function. The target is
41
- * computed once inside the claim transaction and persisted with the immutable
42
- * order, so every retry of the same claim — a reconnect, a lost response, a
43
- * restarted holder — reads the same number. The engine's refusal paths that
44
- * leave the run open (schema-reject) do not bump the artifact, so the target
45
- * survives in-claim retries; a refinement is a NEW claim and therefore a newly
46
- * issued target. If a target ever did go stale, the consumer's version check
47
- * refuses the artifact. The failure mode is a refusal, never an admitted
48
- * unverified value.
41
+ * issued inside the claim transaction and persisted with the order, so a
42
+ * reconnect, a lost response, or a restarted holder re-reads the same number
43
+ * rather than guessing one; a refinement is a NEW claim and therefore a newly
44
+ * issued target. It is not, however, frozen for the life of the claim. A
45
+ * refusal that leaves the run open without bumping the artifact (a schema
46
+ * reject) leaves the target correct, but a judgment/human reject FOLLOWS a
47
+ * commit that already bumped it, so the engine re-stamps the target on the
48
+ * still-open claim at the reject site — this function simply re-reads whatever
49
+ * the current order says, which is why it must never cache one. If a target
50
+ * does go stale anyway, the consumer's version check refuses the artifact. The
51
+ * failure mode is a refusal, never an admitted unverified value.
49
52
  *
50
53
  * A hub that projects no `owes[].version` (pre-version-aware) yields
51
54
  * `undefined`, and the caller submits unsigned rather than signing a guess.