owenloop 0.5.30 → 0.5.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/packages/work/src/agent/loop.d.ts +5 -3
- package/dist/packages/work/src/agent/loop.js +24 -3
- package/dist/packages/work/src/agent/workdir.d.ts +17 -0
- package/dist/packages/work/src/agent/workdir.js +24 -0
- package/dist/packages/work/src/exec/loop.d.ts +8 -1
- package/dist/packages/work/src/exec/loop.js +24 -1
- package/dist/packages/work/src/hub/client.d.ts +10 -3
- package/dist/packages/work/src/hub/client.js +5 -4
- package/dist/packages/work/src/hub/deadline.d.ts +20 -0
- package/dist/packages/work/src/hub/deadline.js +49 -0
- package/dist/packages/work/src/roles/agent-run.js +1 -0
- package/dist/packages/work/src/roles/exec.js +1 -0
- package/dist/packages/work/src/settings/hub-roster-cache.js +5 -23
- package/dist/packages/work/src/shift/loop.d.ts +59 -0
- package/dist/packages/work/src/shift/loop.js +177 -19
- package/dist/packages/work/src/shift/protocol.d.ts +53 -1
- package/dist/packages/work/src/shift/runtime.js +10 -1
- package/dist/packages/work/src/shift/server.js +2 -0
- package/dist/src/bundle/manifest.d.ts +6 -1
- package/dist/src/bundle/manifest.js +38 -8
- package/dist/src/bundle/runtime.d.ts +1 -1
- package/dist/src/bundle/runtime.js +1 -0
- package/dist/src/bundle/types.d.ts +3 -2
- package/dist/src/cli.js +17 -3
- package/dist/src/defs.js +17 -4
- package/dist/src/engine.d.ts +3 -0
- package/dist/src/engine.js +42 -9
- package/dist/src/factory.d.ts +2 -0
- package/dist/src/factory.js +2 -0
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.js +1 -1
- package/dist/src/store/def-source.d.ts +1 -1
- package/dist/src/store/def-source.js +25 -0
- package/dist/src/store/gc.js +9 -7
- package/dist/src/store/index.d.ts +1 -0
- package/dist/src/store/index.js +1 -0
- package/dist/src/store/install.js +9 -3
- package/dist/src/store/instruction-source.js +9 -5
- package/dist/src/store/verified-lock.d.ts +4 -0
- package/dist/src/store/verified-lock.js +38 -0
- package/dist/src/store.js +4 -2
- package/docs/authoring.md +12 -0
- package/docs/bundles.md +3 -2
- package/docs/cli.md +39 -11
- package/docs/shift-logs.md +53 -4
- package/package.json +1 -1
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/plugin/.mcp.json +1 -1
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
- package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,20 @@ All notable changes to this project are documented here. The format is based on
|
|
|
4
4
|
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
|
|
5
5
|
adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
6
|
|
|
7
|
+
## [0.5.31](https://github.com/typicalday/owenloop/compare/v0.5.30...v0.5.31) (2026-10-01)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
### Features
|
|
11
|
+
|
|
12
|
+
* **store:** support exact calls to named bundle workflows ([#327](https://github.com/typicalday/owenloop/issues/327)) ([6336841](https://github.com/typicalday/owenloop/commit/633684192b68d005e1e65e72b8848d4bcd02f91f))
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
### Bug Fixes
|
|
16
|
+
|
|
17
|
+
* **deps:** refresh vulnerable transitive lock entries ([#331](https://github.com/typicalday/owenloop/issues/331)) ([02c3bcd](https://github.com/typicalday/owenloop/commit/02c3bcdc8751e40b7f11f746528ab30f0302df2a))
|
|
18
|
+
* **shift:** deadline every poll-loop hub call and add a stall watchdog ([#315](https://github.com/typicalday/owenloop/issues/315)) ([7e3a2a7](https://github.com/typicalday/owenloop/commit/7e3a2a7c3d9be77803ae6a618d01124a49c90997))
|
|
19
|
+
* **work:** release an order whose workdir no longer exists ([#316](https://github.com/typicalday/owenloop/issues/316)) ([d4c6adb](https://github.com/typicalday/owenloop/commit/d4c6adb4da512f9b5579c68881447eb320142293)), closes [#301](https://github.com/typicalday/owenloop/issues/301)
|
|
20
|
+
|
|
7
21
|
## [0.5.30](https://github.com/typicalday/owenloop/compare/v0.5.29...v0.5.30) (2026-09-05)
|
|
8
22
|
|
|
9
23
|
|
|
@@ -8,7 +8,7 @@ import { type SessionRecord, type SessionTaskRef } from '../harness/session-stor
|
|
|
8
8
|
import { type CapabilityMatch } from './capability-model.ts';
|
|
9
9
|
import type { MergedRoster } from '../settings/roster.ts';
|
|
10
10
|
/** Discriminated result of one worker life; the role maps these to exit codes. */
|
|
11
|
-
export type AgentRunOutcome = 'submitted' | 'held' | 'completed' | 'misroute' | 'workdir-denied' | 'no-template' | 'no-harness' | 'incompatible-harness-policy' | 'unstamped-order' | 'unresolvable-crew' | 'unresolvable-capability' | 'unverified-consumed' | 'session-store-failed' | 'no-submit' | 'killed' | 'lease-lost' | 'ownership-error' | 'hub-unreachable' | 'stopped';
|
|
11
|
+
export type AgentRunOutcome = 'submitted' | 'held' | 'completed' | 'misroute' | 'workdir-denied' | 'workdir-missing' | 'no-template' | 'no-harness' | 'incompatible-harness-policy' | 'unstamped-order' | 'unresolvable-crew' | 'unresolvable-capability' | 'unverified-consumed' | 'session-store-failed' | 'no-submit' | 'killed' | 'lease-lost' | 'ownership-error' | 'hub-unreachable' | 'stopped';
|
|
12
12
|
/**
|
|
13
13
|
* Locate the normalized step spec for an order. Injected because the lookup is
|
|
14
14
|
* I/O (the verified workflow store) and because resolving the local definition
|
|
@@ -123,8 +123,10 @@ export interface AgentRunLoopOptions {
|
|
|
123
123
|
/**
|
|
124
124
|
* Does this directory still exist? Injected for the same reason every other
|
|
125
125
|
* side effect here is: a unit test drives the "the work dir was reaped, so the
|
|
126
|
-
* session cannot resume" branch
|
|
127
|
-
*
|
|
126
|
+
* session cannot resume" branch — and the "the order-named workdir was
|
|
127
|
+
* reclaimed, so the order cannot be served" branch — without touching a
|
|
128
|
+
* filesystem. Defaults to `isExistingDirectory` (`./workdir.ts`): an
|
|
129
|
+
* existing DIRECTORY, so a path that became a plain file answers false.
|
|
128
130
|
*/
|
|
129
131
|
dirExists?: (path: string) => boolean;
|
|
130
132
|
sleep: (ms: number) => Promise<void>;
|
|
@@ -56,8 +56,8 @@
|
|
|
56
56
|
* cold replay WITHIN THE SAME FIRING — the order is still leased, so handing it
|
|
57
57
|
* back would waste a whole re-offer cycle to learn something already known.
|
|
58
58
|
*/
|
|
59
|
-
import {
|
|
60
|
-
import { isWorkdirAllowed } from "./workdir.js";
|
|
59
|
+
import { resolve } from 'node:path';
|
|
60
|
+
import { isExistingDirectory, isWorkdirAllowed } from "./workdir.js";
|
|
61
61
|
import { createApprovalRequester } from "./approvals.js";
|
|
62
62
|
import { createLeaseLoop } from "../lease/loop.js";
|
|
63
63
|
import { HarnessTurnError, isHarnessTurnError, isResumeUnavailable } from "../harness/contract.js";
|
|
@@ -656,6 +656,27 @@ export function createAgentRunLoop(opts) {
|
|
|
656
656
|
`which is outside every work root this machine declared (${roots}) — releasing for the pickup window`);
|
|
657
657
|
return releaseWith('workdir-denied', 'workdir-denied');
|
|
658
658
|
}
|
|
659
|
+
// EXISTENCE. The hub resolved `workdirFrom:` from an artifact VALUE, and a
|
|
660
|
+
// value can outlive the directory it names: a cleanup step reclaims the
|
|
661
|
+
// worktree, then a rejection re-arms an earlier step whose workdir was that
|
|
662
|
+
// worktree (owenloop #301). Opening a session there fails inside the
|
|
663
|
+
// harness with an error that never names the cwd, burns an attempt, and
|
|
664
|
+
// leaves nothing in `routing alerts` naming the cause.
|
|
665
|
+
//
|
|
666
|
+
// RELEASED, not failed, BEFORE the step spec is loaded or any provider
|
|
667
|
+
// session opens, through the same `releaseWith` the roots check above
|
|
668
|
+
// uses — but with a reason that NAMES THE PATH. `releaseWith` hands the
|
|
669
|
+
// reason to `lease.stop`, the lease loop forwards it as the release
|
|
670
|
+
// request's `reason`, and the hub records it as an `unservable-release`
|
|
671
|
+
// routing alert, so `owenloop routing alerts --workflow <wf>` shows the
|
|
672
|
+
// vanished directory. Only an order-NAMED workdir is checked: the fallback
|
|
673
|
+
// `<workRoot>/<workflow>/<run>/` is created by this worker on first use.
|
|
674
|
+
if (packet.workdir !== undefined && !(opts.dirExists ?? isExistingDirectory)(packet.workdir)) {
|
|
675
|
+
const missing = resolve(packet.workdir);
|
|
676
|
+
opts.err(`owenloop work agent-run: step '${packet.step}' (${order}) names workdir '${missing}', ` +
|
|
677
|
+
'which no longer exists on this machine — releasing for the pickup window');
|
|
678
|
+
return releaseWith(`step workdir no longer exists: ${missing}`, 'workdir-missing');
|
|
679
|
+
}
|
|
659
680
|
// The step spec. No bundle / no spec ⇒ we cannot brief anybody; release so
|
|
660
681
|
// the order lapses back through the hub's pickup window.
|
|
661
682
|
let material;
|
|
@@ -804,7 +825,7 @@ export function createAgentRunLoop(opts) {
|
|
|
804
825
|
// filtered to `at > prev.deliveredReasonAt` — the packet's `owes` IS the set
|
|
805
826
|
// of paths being re-armed, so no extra path filter is needed here.
|
|
806
827
|
const prev = opts.latestSession?.(task) ?? null;
|
|
807
|
-
const dirExists = opts.dirExists ??
|
|
828
|
+
const dirExists = opts.dirExists ?? isExistingDirectory;
|
|
808
829
|
const delta = renderRejection({
|
|
809
830
|
packet,
|
|
810
831
|
...(prev?.deliveredReasonAt !== undefined ? { deliveredReasonAt: prev.deliveredReasonAt } : {}),
|
|
@@ -44,6 +44,23 @@ export declare function resolveAllowedWorkdirRoots(env: Record<string, string |
|
|
|
44
44
|
* unconditionally instead of branching on whether the operator declared one.
|
|
45
45
|
*/
|
|
46
46
|
export declare function isWorkdirAllowed(workdir: string, roots: string[]): boolean;
|
|
47
|
+
/**
|
|
48
|
+
* Is `path` an existing DIRECTORY right now?
|
|
49
|
+
*
|
|
50
|
+
* The default behind the `dirExists` seam of both worker loops. `statSync`
|
|
51
|
+
* follows symlinks, so a link to a directory counts; a plain file does not, and
|
|
52
|
+
* neither does a path whose stat fails for any reason (ENOENT, ENOTDIR on a
|
|
53
|
+
* component that became a file, EACCES) — a directory this process cannot stat
|
|
54
|
+
* is one it cannot spawn in either.
|
|
55
|
+
*
|
|
56
|
+
* Why the loops check at all: the hub resolves `workdirFrom:` from an artifact
|
|
57
|
+
* VALUE, and that value can outlive the directory it names — a cleanup step
|
|
58
|
+
* reclaims a worktree, then a rejection re-arms an earlier step whose workdir
|
|
59
|
+
* was that worktree (owenloop #301). Spawning there fails with an opaque
|
|
60
|
+
* `spawn /bin/sh ENOENT` that reads as a missing shell. The loops refuse
|
|
61
|
+
* before spawning instead and release with a reason that names the path.
|
|
62
|
+
*/
|
|
63
|
+
export declare function isExistingDirectory(path: string): boolean;
|
|
47
64
|
export interface EnsureWorkDirOptions {
|
|
48
65
|
workRoot: string;
|
|
49
66
|
workflow: string;
|
|
@@ -161,6 +161,30 @@ export function isWorkdirAllowed(workdir, roots) {
|
|
|
161
161
|
return rel === '' || (!rel.startsWith('..') && !isAbsolute(rel));
|
|
162
162
|
});
|
|
163
163
|
}
|
|
164
|
+
/**
|
|
165
|
+
* Is `path` an existing DIRECTORY right now?
|
|
166
|
+
*
|
|
167
|
+
* The default behind the `dirExists` seam of both worker loops. `statSync`
|
|
168
|
+
* follows symlinks, so a link to a directory counts; a plain file does not, and
|
|
169
|
+
* neither does a path whose stat fails for any reason (ENOENT, ENOTDIR on a
|
|
170
|
+
* component that became a file, EACCES) — a directory this process cannot stat
|
|
171
|
+
* is one it cannot spawn in either.
|
|
172
|
+
*
|
|
173
|
+
* Why the loops check at all: the hub resolves `workdirFrom:` from an artifact
|
|
174
|
+
* VALUE, and that value can outlive the directory it names — a cleanup step
|
|
175
|
+
* reclaims a worktree, then a rejection re-arms an earlier step whose workdir
|
|
176
|
+
* was that worktree (owenloop #301). Spawning there fails with an opaque
|
|
177
|
+
* `spawn /bin/sh ENOENT` that reads as a missing shell. The loops refuse
|
|
178
|
+
* before spawning instead and release with a reason that names the path.
|
|
179
|
+
*/
|
|
180
|
+
export function isExistingDirectory(path) {
|
|
181
|
+
try {
|
|
182
|
+
return statSync(path).isDirectory();
|
|
183
|
+
}
|
|
184
|
+
catch {
|
|
185
|
+
return false;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
164
188
|
function defaultRunGit(args, cwd) {
|
|
165
189
|
const r = spawnSync('git', args, { cwd, encoding: 'utf8' });
|
|
166
190
|
return { status: r.status, output: `${r.stdout ?? ''}${r.stderr ?? ''}`.trim() };
|
|
@@ -22,7 +22,7 @@ import type { SshProcessAdapter } from '../../../../src/crypto/ssh.ts';
|
|
|
22
22
|
*/
|
|
23
23
|
export declare const CONSUMES_INLINE_MAX_BYTES = 65536;
|
|
24
24
|
/** Discriminated result of an exec run; the role maps these to exit codes. */
|
|
25
|
-
export type ExecOutcome = 'submitted' | 'completed' | 'misroute' | 'workdir-denied' | 'unresolved-instructions' | 'killed' | 'lease-lost' | 'ownership-error' | 'hub-unreachable' | 'submit-rejected' | 'submit-failed' | 'command-failed' | 'ask-failed' | 'rejected' | 'judge-rejected' | 'judge-no-verdict' | 'reject-failed' | 'stopped';
|
|
25
|
+
export type ExecOutcome = 'submitted' | 'completed' | 'misroute' | 'workdir-denied' | 'workdir-missing' | 'unresolved-instructions' | 'killed' | 'lease-lost' | 'ownership-error' | 'hub-unreachable' | 'submit-rejected' | 'submit-failed' | 'command-failed' | 'ask-failed' | 'rejected' | 'judge-rejected' | 'judge-no-verdict' | 'reject-failed' | 'stopped';
|
|
26
26
|
export interface ExecLoopOptions {
|
|
27
27
|
hub: HubClient;
|
|
28
28
|
runner: CommandRunner;
|
|
@@ -52,6 +52,13 @@ export interface ExecLoopOptions {
|
|
|
52
52
|
* only default that does not break every shift already running.
|
|
53
53
|
*/
|
|
54
54
|
allowedWorkdirRoots?: string[];
|
|
55
|
+
/**
|
|
56
|
+
* Is this path an existing directory? Consulted only for an order-NAMED
|
|
57
|
+
* workdir, before the command is resolved or spawned. Injected so a unit test
|
|
58
|
+
* can drive the "the worktree was reclaimed" branch without a filesystem.
|
|
59
|
+
* Defaults to `isExistingDirectory` (`agent/workdir.ts`).
|
|
60
|
+
*/
|
|
61
|
+
dirExists?: (path: string) => boolean;
|
|
55
62
|
sleep: (ms: number) => Promise<void>;
|
|
56
63
|
now: () => number;
|
|
57
64
|
random?: () => number;
|
|
@@ -44,7 +44,7 @@ import { createHash } from 'node:crypto';
|
|
|
44
44
|
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
45
45
|
import { tmpdir } from 'node:os';
|
|
46
46
|
import { join, resolve } from 'node:path';
|
|
47
|
-
import { isWorkdirAllowed } from "../agent/workdir.js";
|
|
47
|
+
import { isExistingDirectory, isWorkdirAllowed } from "../agent/workdir.js";
|
|
48
48
|
import { createLeaseLoop } from "../lease/loop.js";
|
|
49
49
|
import { HubError } from "../hub/types.js";
|
|
50
50
|
import { PAYLOAD_FILE_ENV, PAYLOAD_MAX_BYTES, readPayloadFile, resolvePayload } from "./payload.js";
|
|
@@ -874,6 +874,29 @@ export function createExecLoop(opts) {
|
|
|
874
874
|
await leasePromise;
|
|
875
875
|
return 'workdir-denied';
|
|
876
876
|
}
|
|
877
|
+
// EXISTENCE. The hub resolved `workdirFrom:` from an artifact VALUE, and a
|
|
878
|
+
// value can outlive the directory it names: a cleanup step reclaims the
|
|
879
|
+
// worktree, then a rejection re-arms an earlier step whose workdir was that
|
|
880
|
+
// worktree (owenloop #301). Spawning there fails with `spawn /bin/sh
|
|
881
|
+
// ENOENT`, which reads as a missing shell, burns an attempt, and leaves
|
|
882
|
+
// nothing in `routing alerts` naming the cause.
|
|
883
|
+
//
|
|
884
|
+
// RELEASED, not failed, through the same targeted release the roots check
|
|
885
|
+
// above uses — but with a reason that NAMES THE PATH. The lease loop
|
|
886
|
+
// forwards `stop(reason)` as the release request's `reason`, and the hub
|
|
887
|
+
// records it as an `unservable-release` routing alert, so `owenloop
|
|
888
|
+
// routing alerts --workflow <wf>` shows the vanished directory instead of a
|
|
889
|
+
// generic worker failure. Only an order-NAMED workdir is checked, for the
|
|
890
|
+
// same reason as above: the `opts.cwd` fallback is this worker's own
|
|
891
|
+
// launch directory.
|
|
892
|
+
if (order.workdir !== undefined && !(opts.dirExists ?? isExistingDirectory)(order.workdir)) {
|
|
893
|
+
const missing = resolve(order.workdir);
|
|
894
|
+
opts.err(`owenloop work exec: step '${order.step}' (${workflow}/${runId}) names workdir ` +
|
|
895
|
+
`'${missing}', which no longer exists on this machine — releasing for the pickup window`);
|
|
896
|
+
lease.stop(`step workdir no longer exists: ${missing}`); // targeted release, reason names the path
|
|
897
|
+
await leasePromise;
|
|
898
|
+
return 'workdir-missing';
|
|
899
|
+
}
|
|
877
900
|
let resolvedCommand;
|
|
878
901
|
let resolvedBundleDir;
|
|
879
902
|
try {
|
|
@@ -8,7 +8,14 @@ export interface HubClientOptions {
|
|
|
8
8
|
fetchImpl?: typeof fetch;
|
|
9
9
|
}
|
|
10
10
|
export interface HubClient {
|
|
11
|
-
|
|
11
|
+
/**
|
|
12
|
+
* `signal` on this and the other poll-loop verbs (`wake`, `presencePing`)
|
|
13
|
+
* is the shift loop's per-call deadline (issue #300): a hub call that never
|
|
14
|
+
* settles used to hang the poll loop forever while the control socket kept
|
|
15
|
+
* answering. Optional and trailing so every existing fake keeps compiling;
|
|
16
|
+
* a fake that ignores it simply cannot be cut short.
|
|
17
|
+
*/
|
|
18
|
+
whatsNext(req: WhatsNextRequest, signal?: AbortSignal): Promise<WhatsNextResponse>;
|
|
12
19
|
getOrder(req: GetOrderRequest): Promise<GetOrderResponse>;
|
|
13
20
|
heartbeat(req: HeartbeatRequest): Promise<HeartbeatResponse>;
|
|
14
21
|
release(req: ReleaseRequest): Promise<ReleaseResponse>;
|
|
@@ -56,9 +63,9 @@ export interface HubClient {
|
|
|
56
63
|
/** Read the hub's known harness/model registry. */
|
|
57
64
|
listHarnessModels?(): Promise<ListHarnessModelsResponse>;
|
|
58
65
|
/** B5 cheap wake pre-check; `cursor` rides the query string only when set. */
|
|
59
|
-
wake(cursor?: number): Promise<WakeResponse>;
|
|
66
|
+
wake(cursor?: number, signal?: AbortSignal): Promise<WakeResponse>;
|
|
60
67
|
/** B4 Shift presence register/refresh. */
|
|
61
|
-
presencePing(req: PresencePingRequest): Promise<PresencePingResponse>;
|
|
68
|
+
presencePing(req: PresencePingRequest, signal?: AbortSignal): Promise<PresencePingResponse>;
|
|
62
69
|
/**
|
|
63
70
|
* Store opaque bytes as a file artifact and return the envelope naming them.
|
|
64
71
|
*
|
|
@@ -64,11 +64,12 @@ export function createHubClient(opts) {
|
|
|
64
64
|
}
|
|
65
65
|
return JSON.parse(raw);
|
|
66
66
|
}
|
|
67
|
-
async function post(verb, body) {
|
|
67
|
+
async function post(verb, body, signal) {
|
|
68
68
|
const res = await fetchImpl(`${base}/api/${verb}`, {
|
|
69
69
|
method: 'POST',
|
|
70
70
|
headers: await authHeaders(),
|
|
71
71
|
body: JSON.stringify(body),
|
|
72
|
+
...(signal === undefined ? {} : { signal }),
|
|
72
73
|
});
|
|
73
74
|
return parse(res);
|
|
74
75
|
}
|
|
@@ -99,7 +100,7 @@ export function createHubClient(opts) {
|
|
|
99
100
|
return parse(res);
|
|
100
101
|
}
|
|
101
102
|
return {
|
|
102
|
-
whatsNext: (req) => post('whats_next', req),
|
|
103
|
+
whatsNext: (req, signal) => post('whats_next', req, signal),
|
|
103
104
|
getOrder: (req) => post('get_order', req),
|
|
104
105
|
heartbeat: (req) => post('heartbeat', req),
|
|
105
106
|
release: (req) => post('release', req),
|
|
@@ -116,8 +117,8 @@ export function createHubClient(opts) {
|
|
|
116
117
|
listHarnessModels: () => get('harness_models'),
|
|
117
118
|
// Cursor is an opaque non-negative integer; omit it entirely to bootstrap
|
|
118
119
|
// (the hub treats missing/invalid as a `changed: true` first sweep).
|
|
119
|
-
wake: (cursor) => get('wake', typeof cursor === 'number' ? `cursor=${encodeURIComponent(String(cursor))}` : undefined),
|
|
120
|
-
presencePing: (req) => post('presence_ping', req),
|
|
120
|
+
wake: (cursor, signal) => get('wake', typeof cursor === 'number' ? `cursor=${encodeURIComponent(String(cursor))}` : undefined, signal),
|
|
121
|
+
presencePing: (req, signal) => post('presence_ping', req, signal),
|
|
121
122
|
putFileArtifact: (req) => postBytes('/api/file-artifacts', req),
|
|
122
123
|
};
|
|
123
124
|
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A per-call deadline for hub traffic.
|
|
3
|
+
*
|
|
4
|
+
* Issue #300: a shift daemon went silent for hours — no log lines, no hub
|
|
5
|
+
* traffic, zero CPU — while its control socket still answered `shift status`.
|
|
6
|
+
* The poll loop was awaiting one hub call that never settled. Nothing in the
|
|
7
|
+
* loop ever bounded a call, so nothing could ever throw, log, or count it.
|
|
8
|
+
*
|
|
9
|
+
* This bounds one call two ways at once, mirroring the roster-sync deadline
|
|
10
|
+
* this was generalised from: an `AbortController` so an abort-aware transport
|
|
11
|
+
* (undici `fetch` honours `signal`) tears the request down, AND an explicit
|
|
12
|
+
* race so a client that ignores the signal — a test fake, a non-fetch
|
|
13
|
+
* transport — is still cut short. Both report the same Error, so the caller
|
|
14
|
+
* sees one stable outcome whichever fired first.
|
|
15
|
+
*
|
|
16
|
+
* The timeout is a plain `Error`, never a `HubError`: it must be counted as an
|
|
17
|
+
* ordinary (non-rate-limited) failure by whoever catches it, exactly like a
|
|
18
|
+
* connection reset would be.
|
|
19
|
+
*/
|
|
20
|
+
export declare function withHubCallTimeout<T>(label: string, run: (signal: AbortSignal) => Promise<T>, timeoutMs: number): Promise<T>;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A per-call deadline for hub traffic.
|
|
3
|
+
*
|
|
4
|
+
* Issue #300: a shift daemon went silent for hours — no log lines, no hub
|
|
5
|
+
* traffic, zero CPU — while its control socket still answered `shift status`.
|
|
6
|
+
* The poll loop was awaiting one hub call that never settled. Nothing in the
|
|
7
|
+
* loop ever bounded a call, so nothing could ever throw, log, or count it.
|
|
8
|
+
*
|
|
9
|
+
* This bounds one call two ways at once, mirroring the roster-sync deadline
|
|
10
|
+
* this was generalised from: an `AbortController` so an abort-aware transport
|
|
11
|
+
* (undici `fetch` honours `signal`) tears the request down, AND an explicit
|
|
12
|
+
* race so a client that ignores the signal — a test fake, a non-fetch
|
|
13
|
+
* transport — is still cut short. Both report the same Error, so the caller
|
|
14
|
+
* sees one stable outcome whichever fired first.
|
|
15
|
+
*
|
|
16
|
+
* The timeout is a plain `Error`, never a `HubError`: it must be counted as an
|
|
17
|
+
* ordinary (non-rate-limited) failure by whoever catches it, exactly like a
|
|
18
|
+
* connection reset would be.
|
|
19
|
+
*/
|
|
20
|
+
export async function withHubCallTimeout(label, run, timeoutMs) {
|
|
21
|
+
const controller = new AbortController();
|
|
22
|
+
let timer;
|
|
23
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
24
|
+
timer = setTimeout(() => {
|
|
25
|
+
const error = new Error(`${label} timed out after ${timeoutMs}ms`);
|
|
26
|
+
// Abort carries the same terminal error so an abort-aware transport and
|
|
27
|
+
// the explicit race report one stable, useful timeout outcome.
|
|
28
|
+
controller.abort(error);
|
|
29
|
+
reject(error);
|
|
30
|
+
}, timeoutMs);
|
|
31
|
+
// Deliberately NOT unref'd: this timer is the guarantee that the awaited
|
|
32
|
+
// call settles. An unref'd deadline lets the event loop drain while the
|
|
33
|
+
// call is still pending, so the timeout never fires (Node 22 cancels the
|
|
34
|
+
// awaiting test as "Promise resolution is still pending"). The watchdog's
|
|
35
|
+
// periodic timers are the ones that must not pin the process, not this.
|
|
36
|
+
});
|
|
37
|
+
try {
|
|
38
|
+
return await Promise.race([run(controller.signal), timeout]);
|
|
39
|
+
}
|
|
40
|
+
finally {
|
|
41
|
+
if (timer !== undefined)
|
|
42
|
+
clearTimeout(timer);
|
|
43
|
+
// A composite call can reject as soon as one request fails while a
|
|
44
|
+
// sibling fetch is still waiting. Always cancel on settlement so that
|
|
45
|
+
// sibling cannot outlive this call (and accumulate across polls).
|
|
46
|
+
if (!controller.signal.aborted)
|
|
47
|
+
controller.abort(new Error(`${label} finished`));
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -10,6 +10,7 @@ import { createHash, randomUUID } from 'node:crypto';
|
|
|
10
10
|
import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
11
11
|
import { isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
12
12
|
import { validateRoster } from "../agent/capability-model.js";
|
|
13
|
+
import { withHubCallTimeout } from "../hub/deadline.js";
|
|
13
14
|
import { owenloopConfigDir } from "../../../../src/config-dir.js";
|
|
14
15
|
import { normalizeOrigin } from "../../../../src/hub.js";
|
|
15
16
|
export const DEFAULT_HUB_ROSTER_SYNC_TIMEOUT_MS = 10_000;
|
|
@@ -54,29 +55,10 @@ function validateHubRoster(roster, context) {
|
|
|
54
55
|
}
|
|
55
56
|
/** Bound a refresh even when a test seam or non-fetch client ignores abort. */
|
|
56
57
|
export async function withHubRosterSyncTimeout(run, timeoutMs = DEFAULT_HUB_ROSTER_SYNC_TIMEOUT_MS) {
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
const error = new Error(`hub roster sync timed out after ${timeoutMs}ms`);
|
|
62
|
-
// Abort carries the same terminal error so an abort-aware transport and
|
|
63
|
-
// the explicit race report one stable, useful timeout outcome.
|
|
64
|
-
controller.abort(error);
|
|
65
|
-
reject(error);
|
|
66
|
-
}, timeoutMs);
|
|
67
|
-
});
|
|
68
|
-
try {
|
|
69
|
-
return await Promise.race([run(controller.signal), timeout]);
|
|
70
|
-
}
|
|
71
|
-
finally {
|
|
72
|
-
if (timer !== undefined)
|
|
73
|
-
clearTimeout(timer);
|
|
74
|
-
// A composite refresh can reject as soon as one request fails while a
|
|
75
|
-
// sibling fetch is still waiting. Always cancel on settlement so that
|
|
76
|
-
// sibling cannot outlive this refresh (and accumulate across polls).
|
|
77
|
-
if (!controller.signal.aborted)
|
|
78
|
-
controller.abort(new Error('hub roster sync finished'));
|
|
79
|
-
}
|
|
58
|
+
// The general form lives in `hub/deadline.ts` (issue #300 gave the poll
|
|
59
|
+
// loop's other hub calls the same deadline); this keeps the roster-specific
|
|
60
|
+
// default and error text callers already match on.
|
|
61
|
+
return withHubCallTimeout('hub roster sync', run, timeoutMs);
|
|
80
62
|
}
|
|
81
63
|
export function hubRosterCacheDir(env) {
|
|
82
64
|
return join(owenloopConfigDir(env), 'hub-rosters');
|
|
@@ -59,6 +59,36 @@ export interface ShiftLoopOptions {
|
|
|
59
59
|
rosterSyncTimeoutMs?: number;
|
|
60
60
|
/** Daemon-owned cache write, injected so failure cannot kill the loop. */
|
|
61
61
|
syncRosters?: (signal: AbortSignal) => Promise<void>;
|
|
62
|
+
/**
|
|
63
|
+
* Deadline for each poll-loop hub call (`wake`, `presence_ping`, both
|
|
64
|
+
* `whats_next` forms). Default `DEFAULT_HUB_CALL_TIMEOUT_MS`. Issue #300:
|
|
65
|
+
* without one, a call that never settles hangs the loop forever with no
|
|
66
|
+
* error, no log line and no hub traffic. A call that hits it fails on the
|
|
67
|
+
* same terms as a connection reset: logged, and recorded as `hub-error`
|
|
68
|
+
* where the call already reports one. Only a `wake` failure advances
|
|
69
|
+
* `hubFailureStreak`, exactly as before this change.
|
|
70
|
+
*/
|
|
71
|
+
hubCallTimeoutMs?: number;
|
|
72
|
+
/**
|
|
73
|
+
* WATCHDOG threshold: how long since the loop last made PROGRESS — a hub
|
|
74
|
+
* call settled, or a cycle completed — before the shift reports itself
|
|
75
|
+
* `stalled`. Default `3 * pollIntervalMs + hubCallTimeoutMs` — three missed
|
|
76
|
+
* ticks plus the longest a single call may legitimately take. Every settled
|
|
77
|
+
* call moves the reference, so a long cycle of many slow-but-answered calls
|
|
78
|
+
* never trips it; only one call that never settles can.
|
|
79
|
+
*/
|
|
80
|
+
stallThresholdMs?: number;
|
|
81
|
+
/**
|
|
82
|
+
* Cadence of the file-only `heartbeat` record. Default
|
|
83
|
+
* `DEFAULT_HEARTBEAT_INTERVAL_MS` (5 minutes); `0` disables it.
|
|
84
|
+
*/
|
|
85
|
+
heartbeatIntervalMs?: number;
|
|
86
|
+
/**
|
|
87
|
+
* Repeating-timer seam for the watchdog and heartbeat, so tests can fire
|
|
88
|
+
* them by hand. Returns the cancel function. Defaults to an UNREF'd
|
|
89
|
+
* `setInterval`: neither timer may keep the process alive on its own.
|
|
90
|
+
*/
|
|
91
|
+
schedule?: (fn: () => void, everyMs: number) => () => void;
|
|
62
92
|
/**
|
|
63
93
|
* Compute the serving set for the live serve crews. Kept injectable so the
|
|
64
94
|
* loop does no filesystem work; absent means the legitimate empty set.
|
|
@@ -266,6 +296,13 @@ export interface ShiftLoop {
|
|
|
266
296
|
* replaces, which was no alert at all for 34 minutes.
|
|
267
297
|
*/
|
|
268
298
|
noteWorkerFailure(failure: WorkerFailure): void;
|
|
299
|
+
/**
|
|
300
|
+
* Issue #300 liveness facts for `shift status`. A cycle counts once
|
|
301
|
+
* `iterate()` RETURNS, whichever entry point ran it; `getLastPollAt` is the
|
|
302
|
+
* wall clock at that moment, or undefined before the first completes.
|
|
303
|
+
*/
|
|
304
|
+
getCyclesCompleted(): number;
|
|
305
|
+
getLastPollAt(): number | undefined;
|
|
269
306
|
}
|
|
270
307
|
/** The hub reaps a never-contacted claim after this long. Every client-side
|
|
271
308
|
* re-offer latency must stay strictly under it. */
|
|
@@ -332,4 +369,26 @@ export declare const STEP_BRAKE_DECAY_MS = 600000;
|
|
|
332
369
|
* against a copy of the number, which would drift the day the hub's limit moves.
|
|
333
370
|
*/
|
|
334
371
|
export declare const MAX_RELEASE_REASON_POINTS = 1024;
|
|
372
|
+
/**
|
|
373
|
+
* Default per-call deadline for the poll loop's hub verbs (issue #300).
|
|
374
|
+
*
|
|
375
|
+
* 30 seconds, because: (1) none of these verbs long-polls — `wake` is the
|
|
376
|
+
* "cheap wake pre-check" the client documents, `presence_ping` and
|
|
377
|
+
* `whats_next` are single Durable Object round trips, and the hub answers a
|
|
378
|
+
* busy DO with an immediate "retry in 1s" error rather than by holding the
|
|
379
|
+
* request; so a legitimate call finishes in well under a second and 30s is a
|
|
380
|
+
* generous multiple of the slowest plausible one; (2) the failure this bounds
|
|
381
|
+
* lasted HOURS with no traffic at all, so any finite value fixes the bug and
|
|
382
|
+
* the question is only how much a tick may cost while it does — 30s is
|
|
383
|
+
* six default poll intervals, cheap next to the alternative; (3) it is
|
|
384
|
+
* deliberately longer than the 10s roster-sync deadline, which guards a
|
|
385
|
+
* low-priority refresh that may simply be skipped; these calls are the
|
|
386
|
+
* loop's whole purpose and deserve more patience before being counted as
|
|
387
|
+
* failed. Trade-off accepted: a `whats_next` that the hub committed but this
|
|
388
|
+
* side timed out leaves that claim to lapse through the hub's pickup window,
|
|
389
|
+
* exactly as a dropped connection after commit would.
|
|
390
|
+
*/
|
|
391
|
+
export declare const DEFAULT_HUB_CALL_TIMEOUT_MS = 30000;
|
|
392
|
+
/** Default cadence of the file-only `heartbeat` record: every five minutes. */
|
|
393
|
+
export declare const DEFAULT_HEARTBEAT_INTERVAL_MS: number;
|
|
335
394
|
export declare function createShiftLoop(opts: ShiftLoopOptions): ShiftLoop;
|