owenloop 0.5.29 → 0.5.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/packages/work/src/agent/brief.js +73 -4
- package/dist/packages/work/src/agent/loop.d.ts +5 -3
- package/dist/packages/work/src/agent/loop.js +24 -3
- package/dist/packages/work/src/agent/workdir.d.ts +17 -0
- package/dist/packages/work/src/agent/workdir.js +24 -0
- package/dist/packages/work/src/exec/loop.d.ts +8 -1
- package/dist/packages/work/src/exec/loop.js +24 -1
- package/dist/packages/work/src/hub/client.d.ts +10 -3
- package/dist/packages/work/src/hub/client.js +5 -4
- package/dist/packages/work/src/hub/deadline.d.ts +20 -0
- package/dist/packages/work/src/hub/deadline.js +49 -0
- package/dist/packages/work/src/hub/types.d.ts +17 -0
- package/dist/packages/work/src/roles/agent-run.js +1 -0
- package/dist/packages/work/src/roles/exec.js +1 -0
- package/dist/packages/work/src/settings/hub-roster-cache.js +5 -23
- package/dist/packages/work/src/shift/loop.d.ts +59 -0
- package/dist/packages/work/src/shift/loop.js +177 -19
- package/dist/packages/work/src/shift/protocol.d.ts +53 -1
- package/dist/packages/work/src/shift/runtime.js +10 -1
- package/dist/packages/work/src/shift/server.js +2 -0
- package/dist/src/bundle/manifest.d.ts +6 -1
- package/dist/src/bundle/manifest.js +38 -8
- package/dist/src/bundle/runtime.d.ts +1 -1
- package/dist/src/bundle/runtime.js +1 -0
- package/dist/src/bundle/types.d.ts +3 -2
- package/dist/src/cli.js +17 -3
- package/dist/src/crypto/records.d.ts +1 -0
- package/dist/src/crypto/records.js +11 -0
- package/dist/src/defs.js +17 -4
- package/dist/src/engine.d.ts +3 -0
- package/dist/src/engine.js +55 -10
- package/dist/src/factory.d.ts +2 -0
- package/dist/src/factory.js +2 -0
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.js +1 -1
- package/dist/src/schemas/index.d.ts +3 -0
- package/dist/src/schemas/order.v1.schema.json +3 -0
- package/dist/src/store/def-source.d.ts +1 -1
- package/dist/src/store/def-source.js +25 -0
- package/dist/src/store/gc.js +9 -7
- package/dist/src/store/index.d.ts +1 -0
- package/dist/src/store/index.js +1 -0
- package/dist/src/store/install.js +9 -3
- package/dist/src/store/instruction-source.js +9 -5
- package/dist/src/store/verified-lock.d.ts +4 -0
- package/dist/src/store/verified-lock.js +38 -0
- package/dist/src/store.js +4 -2
- package/dist/src/types.d.ts +40 -0
- package/docs/authoring.md +12 -0
- package/docs/bundles.md +3 -2
- package/docs/cli.md +39 -11
- package/docs/shift-logs.md +53 -4
- package/package.json +1 -1
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/plugin/.mcp.json +1 -1
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
- package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,27 @@ All notable changes to this project are documented here. The format is based on
|
|
|
4
4
|
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
|
|
5
5
|
adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
6
|
|
|
7
|
+
## [0.5.31](https://github.com/typicalday/owenloop/compare/v0.5.30...v0.5.31) (2026-10-01)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
### Features
|
|
11
|
+
|
|
12
|
+
* **store:** support exact calls to named bundle workflows ([#327](https://github.com/typicalday/owenloop/issues/327)) ([6336841](https://github.com/typicalday/owenloop/commit/633684192b68d005e1e65e72b8848d4bcd02f91f))
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
### Bug Fixes
|
|
16
|
+
|
|
17
|
+
* **deps:** refresh vulnerable transitive lock entries ([#331](https://github.com/typicalday/owenloop/issues/331)) ([02c3bcd](https://github.com/typicalday/owenloop/commit/02c3bcdc8751e40b7f11f746528ab30f0302df2a))
|
|
18
|
+
* **shift:** deadline every poll-loop hub call and add a stall watchdog ([#315](https://github.com/typicalday/owenloop/issues/315)) ([7e3a2a7](https://github.com/typicalday/owenloop/commit/7e3a2a7c3d9be77803ae6a618d01124a49c90997))
|
|
19
|
+
* **work:** release an order whose workdir no longer exists ([#316](https://github.com/typicalday/owenloop/issues/316)) ([d4c6adb](https://github.com/typicalday/owenloop/commit/d4c6adb4da512f9b5579c68881447eb320142293)), closes [#301](https://github.com/typicalday/owenloop/issues/301)
|
|
20
|
+
|
|
21
|
+
## [0.5.30](https://github.com/typicalday/owenloop/compare/v0.5.29...v0.5.30) (2026-09-05)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
### Features
|
|
25
|
+
|
|
26
|
+
* **engine:** send a refused path's previous value back with the re-offer ([#312](https://github.com/typicalday/owenloop/issues/312)) ([d697b6e](https://github.com/typicalday/owenloop/commit/d697b6ebb08538fa3a79f9eee0bd648fa13620a5))
|
|
27
|
+
|
|
7
28
|
## [0.5.29](https://github.com/typicalday/owenloop/compare/v0.5.28...v0.5.29) (2026-09-05)
|
|
8
29
|
|
|
9
30
|
|
|
@@ -471,6 +471,63 @@ export function renderRejection(spec) {
|
|
|
471
471
|
].join('\n\n');
|
|
472
472
|
return { message, count, ...(newest !== undefined ? { deliveredReasonAt: newest } : {}) };
|
|
473
473
|
}
|
|
474
|
+
/**
|
|
475
|
+
* Render what the refused submission actually SAID, for the paths the cold
|
|
476
|
+
* replay is about to show reasons for.
|
|
477
|
+
*
|
|
478
|
+
* WHY ONLY THE COLD PATH NEEDS THIS. A resumed session still holds its own
|
|
479
|
+
* prior submission, so `renderRejection` alone is enough there. A cold replay
|
|
480
|
+
* has no session: `renderReplayBrief`'s own contract is that without the
|
|
481
|
+
* reasons "the fresh agent repeats the rejected submission verbatim" -- but a
|
|
482
|
+
* fresh agent given only reasons cannot repeat it either. It has to rebuild the
|
|
483
|
+
* whole value from the brief to change the one part that was called out, and
|
|
484
|
+
* the rebuild quietly loses everything the reader did not object to. Sending
|
|
485
|
+
* the value back is what makes "keep everything you already have that was not
|
|
486
|
+
* called out above", which the rejection body already says, a thing the agent
|
|
487
|
+
* is able to do.
|
|
488
|
+
*
|
|
489
|
+
* SCOPE. Only paths that both carry a `previousValue` and have fresh reasons in
|
|
490
|
+
* this delta -- a value shown next to no reason is material with no correction
|
|
491
|
+
* attached, and the agent cannot tell what to do with it.
|
|
492
|
+
*
|
|
493
|
+
* HONESTY. The heading does not call this "the rejected value". After a schema
|
|
494
|
+
* reject nothing was committed, so what the hub sends is the last value that
|
|
495
|
+
* did commit, which may be older than the refusal or absent entirely. The
|
|
496
|
+
* wording says what the engine can actually promise.
|
|
497
|
+
*/
|
|
498
|
+
function renderPreviousValues(spec) {
|
|
499
|
+
const since = spec.deliveredReasonAt;
|
|
500
|
+
const wanted = spec.paths === undefined ? undefined : new Set(spec.paths);
|
|
501
|
+
const sections = [];
|
|
502
|
+
for (const owed of spec.packet.owes) {
|
|
503
|
+
if (wanted !== undefined && !wanted.has(owed.path))
|
|
504
|
+
continue;
|
|
505
|
+
if (owed.previousValue === undefined)
|
|
506
|
+
continue;
|
|
507
|
+
const fresh = owed.reasons.filter((r) => since === undefined || r.at > since);
|
|
508
|
+
if (fresh.length === 0)
|
|
509
|
+
continue;
|
|
510
|
+
let body;
|
|
511
|
+
try {
|
|
512
|
+
body = JSON.stringify(owed.previousValue, null, 2) ?? String(owed.previousValue);
|
|
513
|
+
}
|
|
514
|
+
catch {
|
|
515
|
+
// A value that arrived as JSON cannot cycle, but a projecting layer is
|
|
516
|
+
// free to hand us anything. Dropping the section beats failing the replay.
|
|
517
|
+
continue;
|
|
518
|
+
}
|
|
519
|
+
sections.push(`The value \`${owed.path}\` currently holds:\n\n\`\`\`json\n${body}\n\`\`\``);
|
|
520
|
+
}
|
|
521
|
+
if (sections.length === 0)
|
|
522
|
+
return '';
|
|
523
|
+
return [
|
|
524
|
+
'What is already on the owed paths, so you can revise it rather than rebuild it:',
|
|
525
|
+
...sections,
|
|
526
|
+
'This is what the path holds now, which is the value that was refused when the'
|
|
527
|
+
+ ' refusal came from a reader. When it came from the schema check the value was'
|
|
528
|
+
+ ' never committed, so what you see is the last one that was.',
|
|
529
|
+
].join('\n\n');
|
|
530
|
+
}
|
|
474
531
|
/**
|
|
475
532
|
* Assemble a COLD-REPLAY brief: the ordinary rendered brief plus the same
|
|
476
533
|
* rejection body as a trailing section.
|
|
@@ -497,15 +554,27 @@ export function renderReplayBrief(brief, spec) {
|
|
|
497
554
|
const full = renderRejection(spec);
|
|
498
555
|
if (full.message === '')
|
|
499
556
|
return brief;
|
|
500
|
-
const join = (body, dropped) => {
|
|
557
|
+
const join = (body, dropped, values) => {
|
|
501
558
|
const note = dropped > 0
|
|
502
559
|
? `\n\n(${dropped} older rejection reason${dropped === 1 ? '' : 's'} omitted to fit the context budget.)`
|
|
503
560
|
: '';
|
|
504
|
-
|
|
561
|
+
const prior = values === '' ? '' : `${values}\n\n---\n\n`;
|
|
562
|
+
return `${brief}\n\n---\n\n${prior}${body}${note}`;
|
|
505
563
|
};
|
|
506
|
-
|
|
564
|
+
const values = renderPreviousValues(spec);
|
|
565
|
+
let assembled = join(full.message, 0, values);
|
|
507
566
|
if (estimateTokens(assembled) <= REPLAY_TOKEN_BUDGET)
|
|
508
567
|
return assembled;
|
|
568
|
+
// First thing over budget: the previous values, whole. They are MATERIAL,
|
|
569
|
+
// and the reasons are the INSTRUCTION -- an agent holding the corrections
|
|
570
|
+
// can still rebuild the value, while one holding the value and no correction
|
|
571
|
+
// cannot know what to change. Same ordering principle as the brief itself,
|
|
572
|
+
// which is never trimmed: keep whatever the agent cannot work without.
|
|
573
|
+
if (values !== '') {
|
|
574
|
+
assembled = join(full.message, 0, '');
|
|
575
|
+
if (estimateTokens(assembled) <= REPLAY_TOKEN_BUDGET)
|
|
576
|
+
return assembled;
|
|
577
|
+
}
|
|
509
578
|
// Over budget: re-render with progressively fewer reasons, oldest dropped
|
|
510
579
|
// first. `ordered` is every candidate entry in ascending `at`, so slicing off
|
|
511
580
|
// the FRONT drops the oldest and keeps whole entries.
|
|
@@ -529,7 +598,7 @@ export function renderReplayBrief(brief, spec) {
|
|
|
529
598
|
});
|
|
530
599
|
if (trimmed.message === '')
|
|
531
600
|
break;
|
|
532
|
-
assembled = join(trimmed.message, drop);
|
|
601
|
+
assembled = join(trimmed.message, drop, '');
|
|
533
602
|
if (estimateTokens(assembled) <= REPLAY_TOKEN_BUDGET)
|
|
534
603
|
return assembled;
|
|
535
604
|
}
|
|
@@ -8,7 +8,7 @@ import { type SessionRecord, type SessionTaskRef } from '../harness/session-stor
|
|
|
8
8
|
import { type CapabilityMatch } from './capability-model.ts';
|
|
9
9
|
import type { MergedRoster } from '../settings/roster.ts';
|
|
10
10
|
/** Discriminated result of one worker life; the role maps these to exit codes. */
|
|
11
|
-
export type AgentRunOutcome = 'submitted' | 'held' | 'completed' | 'misroute' | 'workdir-denied' | 'no-template' | 'no-harness' | 'incompatible-harness-policy' | 'unstamped-order' | 'unresolvable-crew' | 'unresolvable-capability' | 'unverified-consumed' | 'session-store-failed' | 'no-submit' | 'killed' | 'lease-lost' | 'ownership-error' | 'hub-unreachable' | 'stopped';
|
|
11
|
+
export type AgentRunOutcome = 'submitted' | 'held' | 'completed' | 'misroute' | 'workdir-denied' | 'workdir-missing' | 'no-template' | 'no-harness' | 'incompatible-harness-policy' | 'unstamped-order' | 'unresolvable-crew' | 'unresolvable-capability' | 'unverified-consumed' | 'session-store-failed' | 'no-submit' | 'killed' | 'lease-lost' | 'ownership-error' | 'hub-unreachable' | 'stopped';
|
|
12
12
|
/**
|
|
13
13
|
* Locate the normalized step spec for an order. Injected because the lookup is
|
|
14
14
|
* I/O (the verified workflow store) and because resolving the local definition
|
|
@@ -123,8 +123,10 @@ export interface AgentRunLoopOptions {
|
|
|
123
123
|
/**
|
|
124
124
|
* Does this directory still exist? Injected for the same reason every other
|
|
125
125
|
* side effect here is: a unit test drives the "the work dir was reaped, so the
|
|
126
|
-
* session cannot resume" branch
|
|
127
|
-
*
|
|
126
|
+
* session cannot resume" branch — and the "the order-named workdir was
|
|
127
|
+
* reclaimed, so the order cannot be served" branch — without touching a
|
|
128
|
+
* filesystem. Defaults to `isExistingDirectory` (`./workdir.ts`): an
|
|
129
|
+
* existing DIRECTORY, so a path that became a plain file answers false.
|
|
128
130
|
*/
|
|
129
131
|
dirExists?: (path: string) => boolean;
|
|
130
132
|
sleep: (ms: number) => Promise<void>;
|
|
@@ -56,8 +56,8 @@
|
|
|
56
56
|
* cold replay WITHIN THE SAME FIRING — the order is still leased, so handing it
|
|
57
57
|
* back would waste a whole re-offer cycle to learn something already known.
|
|
58
58
|
*/
|
|
59
|
-
import {
|
|
60
|
-
import { isWorkdirAllowed } from "./workdir.js";
|
|
59
|
+
import { resolve } from 'node:path';
|
|
60
|
+
import { isExistingDirectory, isWorkdirAllowed } from "./workdir.js";
|
|
61
61
|
import { createApprovalRequester } from "./approvals.js";
|
|
62
62
|
import { createLeaseLoop } from "../lease/loop.js";
|
|
63
63
|
import { HarnessTurnError, isHarnessTurnError, isResumeUnavailable } from "../harness/contract.js";
|
|
@@ -656,6 +656,27 @@ export function createAgentRunLoop(opts) {
|
|
|
656
656
|
`which is outside every work root this machine declared (${roots}) — releasing for the pickup window`);
|
|
657
657
|
return releaseWith('workdir-denied', 'workdir-denied');
|
|
658
658
|
}
|
|
659
|
+
// EXISTENCE. The hub resolved `workdirFrom:` from an artifact VALUE, and a
|
|
660
|
+
// value can outlive the directory it names: a cleanup step reclaims the
|
|
661
|
+
// worktree, then a rejection re-arms an earlier step whose workdir was that
|
|
662
|
+
// worktree (owenloop #301). Opening a session there fails inside the
|
|
663
|
+
// harness with an error that never names the cwd, burns an attempt, and
|
|
664
|
+
// leaves nothing in `routing alerts` naming the cause.
|
|
665
|
+
//
|
|
666
|
+
// RELEASED, not failed, BEFORE the step spec is loaded or any provider
|
|
667
|
+
// session opens, through the same `releaseWith` the roots check above
|
|
668
|
+
// uses — but with a reason that NAMES THE PATH. `releaseWith` hands the
|
|
669
|
+
// reason to `lease.stop`, the lease loop forwards it as the release
|
|
670
|
+
// request's `reason`, and the hub records it as an `unservable-release`
|
|
671
|
+
// routing alert, so `owenloop routing alerts --workflow <wf>` shows the
|
|
672
|
+
// vanished directory. Only an order-NAMED workdir is checked: the fallback
|
|
673
|
+
// `<workRoot>/<workflow>/<run>/` is created by this worker on first use.
|
|
674
|
+
if (packet.workdir !== undefined && !(opts.dirExists ?? isExistingDirectory)(packet.workdir)) {
|
|
675
|
+
const missing = resolve(packet.workdir);
|
|
676
|
+
opts.err(`owenloop work agent-run: step '${packet.step}' (${order}) names workdir '${missing}', ` +
|
|
677
|
+
'which no longer exists on this machine — releasing for the pickup window');
|
|
678
|
+
return releaseWith(`step workdir no longer exists: ${missing}`, 'workdir-missing');
|
|
679
|
+
}
|
|
659
680
|
// The step spec. No bundle / no spec ⇒ we cannot brief anybody; release so
|
|
660
681
|
// the order lapses back through the hub's pickup window.
|
|
661
682
|
let material;
|
|
@@ -804,7 +825,7 @@ export function createAgentRunLoop(opts) {
|
|
|
804
825
|
// filtered to `at > prev.deliveredReasonAt` — the packet's `owes` IS the set
|
|
805
826
|
// of paths being re-armed, so no extra path filter is needed here.
|
|
806
827
|
const prev = opts.latestSession?.(task) ?? null;
|
|
807
|
-
const dirExists = opts.dirExists ??
|
|
828
|
+
const dirExists = opts.dirExists ?? isExistingDirectory;
|
|
808
829
|
const delta = renderRejection({
|
|
809
830
|
packet,
|
|
810
831
|
...(prev?.deliveredReasonAt !== undefined ? { deliveredReasonAt: prev.deliveredReasonAt } : {}),
|
|
@@ -44,6 +44,23 @@ export declare function resolveAllowedWorkdirRoots(env: Record<string, string |
|
|
|
44
44
|
* unconditionally instead of branching on whether the operator declared one.
|
|
45
45
|
*/
|
|
46
46
|
export declare function isWorkdirAllowed(workdir: string, roots: string[]): boolean;
|
|
47
|
+
/**
|
|
48
|
+
* Is `path` an existing DIRECTORY right now?
|
|
49
|
+
*
|
|
50
|
+
* The default behind the `dirExists` seam of both worker loops. `statSync`
|
|
51
|
+
* follows symlinks, so a link to a directory counts; a plain file does not, and
|
|
52
|
+
* neither does a path whose stat fails for any reason (ENOENT, ENOTDIR on a
|
|
53
|
+
* component that became a file, EACCES) — a directory this process cannot stat
|
|
54
|
+
* is one it cannot spawn in either.
|
|
55
|
+
*
|
|
56
|
+
* Why the loops check at all: the hub resolves `workdirFrom:` from an artifact
|
|
57
|
+
* VALUE, and that value can outlive the directory it names — a cleanup step
|
|
58
|
+
* reclaims a worktree, then a rejection re-arms an earlier step whose workdir
|
|
59
|
+
* was that worktree (owenloop #301). Spawning there fails with an opaque
|
|
60
|
+
* `spawn /bin/sh ENOENT` that reads as a missing shell. The loops refuse
|
|
61
|
+
* before spawning instead and release with a reason that names the path.
|
|
62
|
+
*/
|
|
63
|
+
export declare function isExistingDirectory(path: string): boolean;
|
|
47
64
|
export interface EnsureWorkDirOptions {
|
|
48
65
|
workRoot: string;
|
|
49
66
|
workflow: string;
|
|
@@ -161,6 +161,30 @@ export function isWorkdirAllowed(workdir, roots) {
|
|
|
161
161
|
return rel === '' || (!rel.startsWith('..') && !isAbsolute(rel));
|
|
162
162
|
});
|
|
163
163
|
}
|
|
164
|
+
/**
|
|
165
|
+
* Is `path` an existing DIRECTORY right now?
|
|
166
|
+
*
|
|
167
|
+
* The default behind the `dirExists` seam of both worker loops. `statSync`
|
|
168
|
+
* follows symlinks, so a link to a directory counts; a plain file does not, and
|
|
169
|
+
* neither does a path whose stat fails for any reason (ENOENT, ENOTDIR on a
|
|
170
|
+
* component that became a file, EACCES) — a directory this process cannot stat
|
|
171
|
+
* is one it cannot spawn in either.
|
|
172
|
+
*
|
|
173
|
+
* Why the loops check at all: the hub resolves `workdirFrom:` from an artifact
|
|
174
|
+
* VALUE, and that value can outlive the directory it names — a cleanup step
|
|
175
|
+
* reclaims a worktree, then a rejection re-arms an earlier step whose workdir
|
|
176
|
+
* was that worktree (owenloop #301). Spawning there fails with an opaque
|
|
177
|
+
* `spawn /bin/sh ENOENT` that reads as a missing shell. The loops refuse
|
|
178
|
+
* before spawning instead and release with a reason that names the path.
|
|
179
|
+
*/
|
|
180
|
+
export function isExistingDirectory(path) {
|
|
181
|
+
try {
|
|
182
|
+
return statSync(path).isDirectory();
|
|
183
|
+
}
|
|
184
|
+
catch {
|
|
185
|
+
return false;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
164
188
|
function defaultRunGit(args, cwd) {
|
|
165
189
|
const r = spawnSync('git', args, { cwd, encoding: 'utf8' });
|
|
166
190
|
return { status: r.status, output: `${r.stdout ?? ''}${r.stderr ?? ''}`.trim() };
|
|
@@ -22,7 +22,7 @@ import type { SshProcessAdapter } from '../../../../src/crypto/ssh.ts';
|
|
|
22
22
|
*/
|
|
23
23
|
export declare const CONSUMES_INLINE_MAX_BYTES = 65536;
|
|
24
24
|
/** Discriminated result of an exec run; the role maps these to exit codes. */
|
|
25
|
-
export type ExecOutcome = 'submitted' | 'completed' | 'misroute' | 'workdir-denied' | 'unresolved-instructions' | 'killed' | 'lease-lost' | 'ownership-error' | 'hub-unreachable' | 'submit-rejected' | 'submit-failed' | 'command-failed' | 'ask-failed' | 'rejected' | 'judge-rejected' | 'judge-no-verdict' | 'reject-failed' | 'stopped';
|
|
25
|
+
export type ExecOutcome = 'submitted' | 'completed' | 'misroute' | 'workdir-denied' | 'workdir-missing' | 'unresolved-instructions' | 'killed' | 'lease-lost' | 'ownership-error' | 'hub-unreachable' | 'submit-rejected' | 'submit-failed' | 'command-failed' | 'ask-failed' | 'rejected' | 'judge-rejected' | 'judge-no-verdict' | 'reject-failed' | 'stopped';
|
|
26
26
|
export interface ExecLoopOptions {
|
|
27
27
|
hub: HubClient;
|
|
28
28
|
runner: CommandRunner;
|
|
@@ -52,6 +52,13 @@ export interface ExecLoopOptions {
|
|
|
52
52
|
* only default that does not break every shift already running.
|
|
53
53
|
*/
|
|
54
54
|
allowedWorkdirRoots?: string[];
|
|
55
|
+
/**
|
|
56
|
+
* Is this path an existing directory? Consulted only for an order-NAMED
|
|
57
|
+
* workdir, before the command is resolved or spawned. Injected so a unit test
|
|
58
|
+
* can drive the "the worktree was reclaimed" branch without a filesystem.
|
|
59
|
+
* Defaults to `isExistingDirectory` (`agent/workdir.ts`).
|
|
60
|
+
*/
|
|
61
|
+
dirExists?: (path: string) => boolean;
|
|
55
62
|
sleep: (ms: number) => Promise<void>;
|
|
56
63
|
now: () => number;
|
|
57
64
|
random?: () => number;
|
|
@@ -44,7 +44,7 @@ import { createHash } from 'node:crypto';
|
|
|
44
44
|
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
45
45
|
import { tmpdir } from 'node:os';
|
|
46
46
|
import { join, resolve } from 'node:path';
|
|
47
|
-
import { isWorkdirAllowed } from "../agent/workdir.js";
|
|
47
|
+
import { isExistingDirectory, isWorkdirAllowed } from "../agent/workdir.js";
|
|
48
48
|
import { createLeaseLoop } from "../lease/loop.js";
|
|
49
49
|
import { HubError } from "../hub/types.js";
|
|
50
50
|
import { PAYLOAD_FILE_ENV, PAYLOAD_MAX_BYTES, readPayloadFile, resolvePayload } from "./payload.js";
|
|
@@ -874,6 +874,29 @@ export function createExecLoop(opts) {
|
|
|
874
874
|
await leasePromise;
|
|
875
875
|
return 'workdir-denied';
|
|
876
876
|
}
|
|
877
|
+
// EXISTENCE. The hub resolved `workdirFrom:` from an artifact VALUE, and a
|
|
878
|
+
// value can outlive the directory it names: a cleanup step reclaims the
|
|
879
|
+
// worktree, then a rejection re-arms an earlier step whose workdir was that
|
|
880
|
+
// worktree (owenloop #301). Spawning there fails with `spawn /bin/sh
|
|
881
|
+
// ENOENT`, which reads as a missing shell, burns an attempt, and leaves
|
|
882
|
+
// nothing in `routing alerts` naming the cause.
|
|
883
|
+
//
|
|
884
|
+
// RELEASED, not failed, through the same targeted release the roots check
|
|
885
|
+
// above uses — but with a reason that NAMES THE PATH. The lease loop
|
|
886
|
+
// forwards `stop(reason)` as the release request's `reason`, and the hub
|
|
887
|
+
// records it as an `unservable-release` routing alert, so `owenloop
|
|
888
|
+
// routing alerts --workflow <wf>` shows the vanished directory instead of a
|
|
889
|
+
// generic worker failure. Only an order-NAMED workdir is checked, for the
|
|
890
|
+
// same reason as above: the `opts.cwd` fallback is this worker's own
|
|
891
|
+
// launch directory.
|
|
892
|
+
if (order.workdir !== undefined && !(opts.dirExists ?? isExistingDirectory)(order.workdir)) {
|
|
893
|
+
const missing = resolve(order.workdir);
|
|
894
|
+
opts.err(`owenloop work exec: step '${order.step}' (${workflow}/${runId}) names workdir ` +
|
|
895
|
+
`'${missing}', which no longer exists on this machine — releasing for the pickup window`);
|
|
896
|
+
lease.stop(`step workdir no longer exists: ${missing}`); // targeted release, reason names the path
|
|
897
|
+
await leasePromise;
|
|
898
|
+
return 'workdir-missing';
|
|
899
|
+
}
|
|
877
900
|
let resolvedCommand;
|
|
878
901
|
let resolvedBundleDir;
|
|
879
902
|
try {
|
|
@@ -8,7 +8,14 @@ export interface HubClientOptions {
|
|
|
8
8
|
fetchImpl?: typeof fetch;
|
|
9
9
|
}
|
|
10
10
|
export interface HubClient {
|
|
11
|
-
|
|
11
|
+
/**
|
|
12
|
+
* `signal` on this and the other poll-loop verbs (`wake`, `presencePing`)
|
|
13
|
+
* is the shift loop's per-call deadline (issue #300): a hub call that never
|
|
14
|
+
* settles used to hang the poll loop forever while the control socket kept
|
|
15
|
+
* answering. Optional and trailing so every existing fake keeps compiling;
|
|
16
|
+
* a fake that ignores it simply cannot be cut short.
|
|
17
|
+
*/
|
|
18
|
+
whatsNext(req: WhatsNextRequest, signal?: AbortSignal): Promise<WhatsNextResponse>;
|
|
12
19
|
getOrder(req: GetOrderRequest): Promise<GetOrderResponse>;
|
|
13
20
|
heartbeat(req: HeartbeatRequest): Promise<HeartbeatResponse>;
|
|
14
21
|
release(req: ReleaseRequest): Promise<ReleaseResponse>;
|
|
@@ -56,9 +63,9 @@ export interface HubClient {
|
|
|
56
63
|
/** Read the hub's known harness/model registry. */
|
|
57
64
|
listHarnessModels?(): Promise<ListHarnessModelsResponse>;
|
|
58
65
|
/** B5 cheap wake pre-check; `cursor` rides the query string only when set. */
|
|
59
|
-
wake(cursor?: number): Promise<WakeResponse>;
|
|
66
|
+
wake(cursor?: number, signal?: AbortSignal): Promise<WakeResponse>;
|
|
60
67
|
/** B4 Shift presence register/refresh. */
|
|
61
|
-
presencePing(req: PresencePingRequest): Promise<PresencePingResponse>;
|
|
68
|
+
presencePing(req: PresencePingRequest, signal?: AbortSignal): Promise<PresencePingResponse>;
|
|
62
69
|
/**
|
|
63
70
|
* Store opaque bytes as a file artifact and return the envelope naming them.
|
|
64
71
|
*
|
|
@@ -64,11 +64,12 @@ export function createHubClient(opts) {
|
|
|
64
64
|
}
|
|
65
65
|
return JSON.parse(raw);
|
|
66
66
|
}
|
|
67
|
-
async function post(verb, body) {
|
|
67
|
+
async function post(verb, body, signal) {
|
|
68
68
|
const res = await fetchImpl(`${base}/api/${verb}`, {
|
|
69
69
|
method: 'POST',
|
|
70
70
|
headers: await authHeaders(),
|
|
71
71
|
body: JSON.stringify(body),
|
|
72
|
+
...(signal === undefined ? {} : { signal }),
|
|
72
73
|
});
|
|
73
74
|
return parse(res);
|
|
74
75
|
}
|
|
@@ -99,7 +100,7 @@ export function createHubClient(opts) {
|
|
|
99
100
|
return parse(res);
|
|
100
101
|
}
|
|
101
102
|
return {
|
|
102
|
-
whatsNext: (req) => post('whats_next', req),
|
|
103
|
+
whatsNext: (req, signal) => post('whats_next', req, signal),
|
|
103
104
|
getOrder: (req) => post('get_order', req),
|
|
104
105
|
heartbeat: (req) => post('heartbeat', req),
|
|
105
106
|
release: (req) => post('release', req),
|
|
@@ -116,8 +117,8 @@ export function createHubClient(opts) {
|
|
|
116
117
|
listHarnessModels: () => get('harness_models'),
|
|
117
118
|
// Cursor is an opaque non-negative integer; omit it entirely to bootstrap
|
|
118
119
|
// (the hub treats missing/invalid as a `changed: true` first sweep).
|
|
119
|
-
wake: (cursor) => get('wake', typeof cursor === 'number' ? `cursor=${encodeURIComponent(String(cursor))}` : undefined),
|
|
120
|
-
presencePing: (req) => post('presence_ping', req),
|
|
120
|
+
wake: (cursor, signal) => get('wake', typeof cursor === 'number' ? `cursor=${encodeURIComponent(String(cursor))}` : undefined, signal),
|
|
121
|
+
presencePing: (req, signal) => post('presence_ping', req, signal),
|
|
121
122
|
putFileArtifact: (req) => postBytes('/api/file-artifacts', req),
|
|
122
123
|
};
|
|
123
124
|
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A per-call deadline for hub traffic.
|
|
3
|
+
*
|
|
4
|
+
* Issue #300: a shift daemon went silent for hours — no log lines, no hub
|
|
5
|
+
* traffic, zero CPU — while its control socket still answered `shift status`.
|
|
6
|
+
* The poll loop was awaiting one hub call that never settled. Nothing in the
|
|
7
|
+
* loop ever bounded a call, so nothing could ever throw, log, or count it.
|
|
8
|
+
*
|
|
9
|
+
* This bounds one call two ways at once, mirroring the roster-sync deadline
|
|
10
|
+
* this was generalised from: an `AbortController` so an abort-aware transport
|
|
11
|
+
* (undici `fetch` honours `signal`) tears the request down, AND an explicit
|
|
12
|
+
* race so a client that ignores the signal — a test fake, a non-fetch
|
|
13
|
+
* transport — is still cut short. Both report the same Error, so the caller
|
|
14
|
+
* sees one stable outcome whichever fired first.
|
|
15
|
+
*
|
|
16
|
+
* The timeout is a plain `Error`, never a `HubError`: it must be counted as an
|
|
17
|
+
* ordinary (non-rate-limited) failure by whoever catches it, exactly like a
|
|
18
|
+
* connection reset would be.
|
|
19
|
+
*/
|
|
20
|
+
export declare function withHubCallTimeout<T>(label: string, run: (signal: AbortSignal) => Promise<T>, timeoutMs: number): Promise<T>;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A per-call deadline for hub traffic.
|
|
3
|
+
*
|
|
4
|
+
* Issue #300: a shift daemon went silent for hours — no log lines, no hub
|
|
5
|
+
* traffic, zero CPU — while its control socket still answered `shift status`.
|
|
6
|
+
* The poll loop was awaiting one hub call that never settled. Nothing in the
|
|
7
|
+
* loop ever bounded a call, so nothing could ever throw, log, or count it.
|
|
8
|
+
*
|
|
9
|
+
* This bounds one call two ways at once, mirroring the roster-sync deadline
|
|
10
|
+
* this was generalised from: an `AbortController` so an abort-aware transport
|
|
11
|
+
* (undici `fetch` honours `signal`) tears the request down, AND an explicit
|
|
12
|
+
* race so a client that ignores the signal — a test fake, a non-fetch
|
|
13
|
+
* transport — is still cut short. Both report the same Error, so the caller
|
|
14
|
+
* sees one stable outcome whichever fired first.
|
|
15
|
+
*
|
|
16
|
+
* The timeout is a plain `Error`, never a `HubError`: it must be counted as an
|
|
17
|
+
* ordinary (non-rate-limited) failure by whoever catches it, exactly like a
|
|
18
|
+
* connection reset would be.
|
|
19
|
+
*/
|
|
20
|
+
export async function withHubCallTimeout(label, run, timeoutMs) {
|
|
21
|
+
const controller = new AbortController();
|
|
22
|
+
let timer;
|
|
23
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
24
|
+
timer = setTimeout(() => {
|
|
25
|
+
const error = new Error(`${label} timed out after ${timeoutMs}ms`);
|
|
26
|
+
// Abort carries the same terminal error so an abort-aware transport and
|
|
27
|
+
// the explicit race report one stable, useful timeout outcome.
|
|
28
|
+
controller.abort(error);
|
|
29
|
+
reject(error);
|
|
30
|
+
}, timeoutMs);
|
|
31
|
+
// Deliberately NOT unref'd: this timer is the guarantee that the awaited
|
|
32
|
+
// call settles. An unref'd deadline lets the event loop drain while the
|
|
33
|
+
// call is still pending, so the timeout never fires (Node 22 cancels the
|
|
34
|
+
// awaiting test as "Promise resolution is still pending"). The watchdog's
|
|
35
|
+
// periodic timers are the ones that must not pin the process, not this.
|
|
36
|
+
});
|
|
37
|
+
try {
|
|
38
|
+
return await Promise.race([run(controller.signal), timeout]);
|
|
39
|
+
}
|
|
40
|
+
finally {
|
|
41
|
+
if (timer !== undefined)
|
|
42
|
+
clearTimeout(timer);
|
|
43
|
+
// A composite call can reject as soon as one request fails while a
|
|
44
|
+
// sibling fetch is still waiting. Always cancel on settlement so that
|
|
45
|
+
// sibling cannot outlive this call (and accumulate across polls).
|
|
46
|
+
if (!controller.signal.aborted)
|
|
47
|
+
controller.abort(new Error(`${label} finished`));
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -283,6 +283,23 @@ export interface OrderPacket {
|
|
|
283
283
|
judgmentRejects: number;
|
|
284
284
|
schemaRejects: number;
|
|
285
285
|
reasons: ReasonEntry[];
|
|
286
|
+
/**
|
|
287
|
+
* The value this path already holds, projected by a hub new enough to send
|
|
288
|
+
* it and ONLY on a re-offer whose reason thread is non-empty. Absent on a
|
|
289
|
+
* first offer, absent from an older hub, and absent when the refusal never
|
|
290
|
+
* committed a value.
|
|
291
|
+
*
|
|
292
|
+
* It is the REFUSED value only after a judgment or human reject, which
|
|
293
|
+
* follow a commit. A schema reject refuses the write, so after one this
|
|
294
|
+
* carries the last value that did commit, or nothing. Read `reasons` when
|
|
295
|
+
* the distinction matters; the engine's contract is on `Order.owes[]` in
|
|
296
|
+
* `src/types.ts`.
|
|
297
|
+
*
|
|
298
|
+
* Used on the COLD-REPLAY path only — see `renderReplayBrief`. A resumed
|
|
299
|
+
* session still holds its own prior submission, so sending it back there
|
|
300
|
+
* would spend tokens re-stating what the model already has.
|
|
301
|
+
*/
|
|
302
|
+
previousValue?: unknown;
|
|
286
303
|
/**
|
|
287
304
|
* The JSON Schema the engine will enforce on this output at commit time,
|
|
288
305
|
* projected off the owning produce entry by a schema-aware hub. Absent
|
|
@@ -10,6 +10,7 @@ import { createHash, randomUUID } from 'node:crypto';
|
|
|
10
10
|
import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
11
11
|
import { isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
12
12
|
import { validateRoster } from "../agent/capability-model.js";
|
|
13
|
+
import { withHubCallTimeout } from "../hub/deadline.js";
|
|
13
14
|
import { owenloopConfigDir } from "../../../../src/config-dir.js";
|
|
14
15
|
import { normalizeOrigin } from "../../../../src/hub.js";
|
|
15
16
|
export const DEFAULT_HUB_ROSTER_SYNC_TIMEOUT_MS = 10_000;
|
|
@@ -54,29 +55,10 @@ function validateHubRoster(roster, context) {
|
|
|
54
55
|
}
|
|
55
56
|
/** Bound a refresh even when a test seam or non-fetch client ignores abort. */
|
|
56
57
|
export async function withHubRosterSyncTimeout(run, timeoutMs = DEFAULT_HUB_ROSTER_SYNC_TIMEOUT_MS) {
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
const error = new Error(`hub roster sync timed out after ${timeoutMs}ms`);
|
|
62
|
-
// Abort carries the same terminal error so an abort-aware transport and
|
|
63
|
-
// the explicit race report one stable, useful timeout outcome.
|
|
64
|
-
controller.abort(error);
|
|
65
|
-
reject(error);
|
|
66
|
-
}, timeoutMs);
|
|
67
|
-
});
|
|
68
|
-
try {
|
|
69
|
-
return await Promise.race([run(controller.signal), timeout]);
|
|
70
|
-
}
|
|
71
|
-
finally {
|
|
72
|
-
if (timer !== undefined)
|
|
73
|
-
clearTimeout(timer);
|
|
74
|
-
// A composite refresh can reject as soon as one request fails while a
|
|
75
|
-
// sibling fetch is still waiting. Always cancel on settlement so that
|
|
76
|
-
// sibling cannot outlive this refresh (and accumulate across polls).
|
|
77
|
-
if (!controller.signal.aborted)
|
|
78
|
-
controller.abort(new Error('hub roster sync finished'));
|
|
79
|
-
}
|
|
58
|
+
// The general form lives in `hub/deadline.ts` (issue #300 gave the poll
|
|
59
|
+
// loop's other hub calls the same deadline); this keeps the roster-specific
|
|
60
|
+
// default and error text callers already match on.
|
|
61
|
+
return withHubCallTimeout('hub roster sync', run, timeoutMs);
|
|
80
62
|
}
|
|
81
63
|
export function hubRosterCacheDir(env) {
|
|
82
64
|
return join(owenloopConfigDir(env), 'hub-rosters');
|
|
@@ -59,6 +59,36 @@ export interface ShiftLoopOptions {
|
|
|
59
59
|
rosterSyncTimeoutMs?: number;
|
|
60
60
|
/** Daemon-owned cache write, injected so failure cannot kill the loop. */
|
|
61
61
|
syncRosters?: (signal: AbortSignal) => Promise<void>;
|
|
62
|
+
/**
|
|
63
|
+
* Deadline for each poll-loop hub call (`wake`, `presence_ping`, both
|
|
64
|
+
* `whats_next` forms). Default `DEFAULT_HUB_CALL_TIMEOUT_MS`. Issue #300:
|
|
65
|
+
* without one, a call that never settles hangs the loop forever with no
|
|
66
|
+
* error, no log line and no hub traffic. A call that hits it fails on the
|
|
67
|
+
* same terms as a connection reset: logged, and recorded as `hub-error`
|
|
68
|
+
* where the call already reports one. Only a `wake` failure advances
|
|
69
|
+
* `hubFailureStreak`, exactly as before this change.
|
|
70
|
+
*/
|
|
71
|
+
hubCallTimeoutMs?: number;
|
|
72
|
+
/**
|
|
73
|
+
* WATCHDOG threshold: how long since the loop last made PROGRESS — a hub
|
|
74
|
+
* call settled, or a cycle completed — before the shift reports itself
|
|
75
|
+
* `stalled`. Default `3 * pollIntervalMs + hubCallTimeoutMs` — three missed
|
|
76
|
+
* ticks plus the longest a single call may legitimately take. Every settled
|
|
77
|
+
* call moves the reference, so a long cycle of many slow-but-answered calls
|
|
78
|
+
* never trips it; only one call that never settles can.
|
|
79
|
+
*/
|
|
80
|
+
stallThresholdMs?: number;
|
|
81
|
+
/**
|
|
82
|
+
* Cadence of the file-only `heartbeat` record. Default
|
|
83
|
+
* `DEFAULT_HEARTBEAT_INTERVAL_MS` (5 minutes); `0` disables it.
|
|
84
|
+
*/
|
|
85
|
+
heartbeatIntervalMs?: number;
|
|
86
|
+
/**
|
|
87
|
+
* Repeating-timer seam for the watchdog and heartbeat, so tests can fire
|
|
88
|
+
* them by hand. Returns the cancel function. Defaults to an UNREF'd
|
|
89
|
+
* `setInterval`: neither timer may keep the process alive on its own.
|
|
90
|
+
*/
|
|
91
|
+
schedule?: (fn: () => void, everyMs: number) => () => void;
|
|
62
92
|
/**
|
|
63
93
|
* Compute the serving set for the live serve crews. Kept injectable so the
|
|
64
94
|
* loop does no filesystem work; absent means the legitimate empty set.
|
|
@@ -266,6 +296,13 @@ export interface ShiftLoop {
|
|
|
266
296
|
* replaces, which was no alert at all for 34 minutes.
|
|
267
297
|
*/
|
|
268
298
|
noteWorkerFailure(failure: WorkerFailure): void;
|
|
299
|
+
/**
|
|
300
|
+
* Issue #300 liveness facts for `shift status`. A cycle counts once
|
|
301
|
+
* `iterate()` RETURNS, whichever entry point ran it; `getLastPollAt` is the
|
|
302
|
+
* wall clock at that moment, or undefined before the first completes.
|
|
303
|
+
*/
|
|
304
|
+
getCyclesCompleted(): number;
|
|
305
|
+
getLastPollAt(): number | undefined;
|
|
269
306
|
}
|
|
270
307
|
/** The hub reaps a never-contacted claim after this long. Every client-side
|
|
271
308
|
* re-offer latency must stay strictly under it. */
|
|
@@ -332,4 +369,26 @@ export declare const STEP_BRAKE_DECAY_MS = 600000;
|
|
|
332
369
|
* against a copy of the number, which would drift the day the hub's limit moves.
|
|
333
370
|
*/
|
|
334
371
|
export declare const MAX_RELEASE_REASON_POINTS = 1024;
|
|
372
|
+
/**
|
|
373
|
+
* Default per-call deadline for the poll loop's hub verbs (issue #300).
|
|
374
|
+
*
|
|
375
|
+
* 30 seconds, because: (1) none of these verbs long-polls — `wake` is the
|
|
376
|
+
* "cheap wake pre-check" the client documents, `presence_ping` and
|
|
377
|
+
* `whats_next` are single Durable Object round trips, and the hub answers a
|
|
378
|
+
* busy DO with an immediate "retry in 1s" error rather than by holding the
|
|
379
|
+
* request; so a legitimate call finishes in well under a second and 30s is a
|
|
380
|
+
* generous multiple of the slowest plausible one; (2) the failure this bounds
|
|
381
|
+
* lasted HOURS with no traffic at all, so any finite value fixes the bug and
|
|
382
|
+
* the question is only how much a tick may cost while it does — 30s is
|
|
383
|
+
* six default poll intervals, cheap next to the alternative; (3) it is
|
|
384
|
+
* deliberately longer than the 10s roster-sync deadline, which guards a
|
|
385
|
+
* low-priority refresh that may simply be skipped; these calls are the
|
|
386
|
+
* loop's whole purpose and deserve more patience before being counted as
|
|
387
|
+
* failed. Trade-off accepted: a `whats_next` that the hub committed but this
|
|
388
|
+
* side timed out leaves that claim to lapse through the hub's pickup window,
|
|
389
|
+
* exactly as a dropped connection after commit would.
|
|
390
|
+
*/
|
|
391
|
+
export declare const DEFAULT_HUB_CALL_TIMEOUT_MS = 30000;
|
|
392
|
+
/** Default cadence of the file-only `heartbeat` record: every five minutes. */
|
|
393
|
+
export declare const DEFAULT_HEARTBEAT_INTERVAL_MS: number;
|
|
335
394
|
export declare function createShiftLoop(opts: ShiftLoopOptions): ShiftLoop;
|