owenloop 0.5.8 → 0.5.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +81 -39
- package/bin/owenloop.mjs +16 -1
- package/dist/packages/work/src/agent/loop.d.ts +23 -8
- package/dist/packages/work/src/agent/loop.js +55 -6
- package/dist/packages/work/src/bundle/pull.d.ts +21 -0
- package/dist/packages/work/src/bundle/pull.js +118 -0
- package/dist/packages/work/src/consumed-verifier.js +6 -17
- package/dist/packages/work/src/exec/instructions.d.ts +4 -1
- package/dist/packages/work/src/exec/instructions.js +3 -0
- package/dist/packages/work/src/exec/loop.d.ts +2 -2
- package/dist/packages/work/src/exec/loop.js +273 -46
- package/dist/packages/work/src/exec/receipt.d.ts +7 -7
- package/dist/packages/work/src/exec/receipt.js +6 -6
- package/dist/packages/work/src/harness/claude.js +116 -0
- package/dist/packages/work/src/harness/codex.d.ts +8 -0
- package/dist/packages/work/src/harness/codex.js +27 -1
- package/dist/packages/work/src/harness/contract.d.ts +9 -0
- package/dist/packages/work/src/hub/client.d.ts +19 -4
- package/dist/packages/work/src/hub/client.js +7 -3
- package/dist/packages/work/src/hub/types.d.ts +84 -0
- package/dist/packages/work/src/main.js +1 -0
- package/dist/packages/work/src/roles/agent-run.js +57 -32
- package/dist/packages/work/src/roles/exec.js +27 -8
- package/dist/packages/work/src/roles/settings.js +2 -0
- package/dist/packages/work/src/roles/shift.js +14 -2
- package/dist/packages/work/src/roles/util.d.ts +1 -0
- package/dist/packages/work/src/roles/util.js +10 -0
- package/dist/packages/work/src/settings/hub-roster-cache.d.ts +68 -0
- package/dist/packages/work/src/settings/hub-roster-cache.js +370 -0
- package/dist/packages/work/src/settings/roster.d.ts +84 -0
- package/dist/packages/work/src/settings/roster.js +364 -5
- package/dist/packages/work/src/settings/serving.d.ts +17 -0
- package/dist/packages/work/src/settings/serving.js +79 -0
- package/dist/packages/work/src/settings/settings.d.ts +14 -1
- package/dist/packages/work/src/settings/settings.js +14 -0
- package/dist/packages/work/src/shift/loop.d.ts +108 -5
- package/dist/packages/work/src/shift/loop.js +416 -54
- package/dist/packages/work/src/shift/protocol.d.ts +7 -5
- package/dist/packages/work/src/shift/runtime.d.ts +18 -0
- package/dist/packages/work/src/shift/runtime.js +104 -5
- package/dist/packages/work/src/shift/server.js +2 -0
- package/dist/packages/work/src/shift/spawn.d.ts +17 -2
- package/dist/packages/work/src/shift/spawn.js +16 -13
- package/dist/packages/work/src/shift/state.d.ts +29 -2
- package/dist/packages/work/src/shift/state.js +45 -15
- package/dist/packages/work/src/submit-proof.d.ts +11 -8
- package/dist/packages/work/src/submit-proof.js +11 -8
- package/dist/packages/work/src/usage.d.ts +1 -1
- package/dist/packages/work/src/usage.js +7 -1
- package/dist/packages/work/src/util/modifier-init.d.ts +7 -0
- package/dist/packages/work/src/util/modifier-init.js +73 -0
- package/dist/src/bundle/tar.d.ts +9 -4
- package/dist/src/bundle/tar.js +9 -4
- package/dist/src/capabilities.d.ts +76 -6
- package/dist/src/capabilities.js +49 -6
- package/dist/src/capability-mapping-client.d.ts +29 -0
- package/dist/src/capability-mapping-client.js +160 -0
- package/dist/src/cli.d.ts +16 -3
- package/dist/src/cli.js +1925 -281
- package/dist/src/config-dir.d.ts +1 -1
- package/dist/src/config-dir.js +1 -1
- package/dist/src/crypto/index.d.ts +1 -1
- package/dist/src/crypto/index.js +1 -1
- package/dist/src/crypto/org-root.d.ts +30 -12
- package/dist/src/crypto/org-root.js +162 -14
- package/dist/src/crypto/records.d.ts +2 -0
- package/dist/src/crypto/records.js +2 -0
- package/dist/src/defs.d.ts +38 -2
- package/dist/src/defs.js +593 -86
- package/dist/src/engine.d.ts +103 -25
- package/dist/src/engine.js +331 -79
- package/dist/src/hub.d.ts +77 -0
- package/dist/src/hub.js +162 -0
- package/dist/src/implements.d.ts +56 -0
- package/dist/src/implements.js +790 -0
- package/dist/src/index.d.ts +11 -9
- package/dist/src/index.js +6 -5
- package/dist/src/mcp/serve.d.ts +22 -0
- package/dist/src/mcp/serve.js +330 -41
- package/dist/src/mcp/server.d.ts +10 -7
- package/dist/src/mcp/server.js +53 -8
- package/dist/src/model.d.ts +48 -10
- package/dist/src/model.js +252 -99
- package/dist/src/order-resolver.d.ts +1 -0
- package/dist/src/order-resolver.js +8 -1
- package/dist/src/schemas/index.d.ts +11 -0
- package/dist/src/schemas/order.v1.schema.json +7 -1
- package/dist/src/store/def-source.d.ts +19 -0
- package/dist/src/store/def-source.js +87 -17
- package/dist/src/store/gc-recovery.d.ts +46 -0
- package/dist/src/store/gc-recovery.js +258 -0
- package/dist/src/store/gc.d.ts +94 -0
- package/dist/src/store/gc.js +562 -0
- package/dist/src/store/index.d.ts +3 -1
- package/dist/src/store/index.js +2 -0
- package/dist/src/store/install.d.ts +34 -5
- package/dist/src/store/install.js +73 -11
- package/dist/src/store/pre-commit-verifier.js +42 -2
- package/dist/src/store/snapshot-guard.d.ts +18 -0
- package/dist/src/store/snapshot-guard.js +94 -0
- package/dist/src/store.d.ts +69 -4
- package/dist/src/store.js +254 -16
- package/dist/src/types.d.ts +101 -20
- package/dist/src/util.js +4 -1
- package/docs/authoring.md +235 -10
- package/docs/bundles.md +45 -2
- package/docs/cli.md +437 -55
- package/docs/crypto.md +32 -10
- package/docs/design.md +177 -47
- package/docs/evals/mcp-charter-baseline.json +428 -0
- package/docs/mcp-charter-eval.md +124 -0
- package/docs/routing.md +189 -34
- package/docs/shift-logs.md +38 -9
- package/docs/wire-contracts.md +78 -14
- package/examples/workflows/compile-dev-playbook.yaml +1 -1
- package/examples/workflows/delivery.yaml +31 -14
- package/examples/workflows/ship.yaml +38 -4
- package/package.json +2 -1
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/plugin/.mcp.json +1 -1
- package/plugins/claude-code/plugin/skills/author/SKILL.md +22 -0
- package/plugins/claude-code/plugin/skills/ephemeral/SKILL.md +95 -0
- package/plugins/claude-code/plugin/skills/graduate/SKILL.md +156 -0
- package/plugins/claude-code/plugin/skills/plan/SKILL.md +236 -0
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
- package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
- package/plugins/codex/plugins/owenloop/skills/author/SKILL.md +22 -0
- package/plugins/codex/plugins/owenloop/skills/ephemeral/SKILL.md +95 -0
- package/plugins/codex/plugins/owenloop/skills/graduate/SKILL.md +156 -0
- package/plugins/codex/plugins/owenloop/skills/plan/SKILL.md +236 -0
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* authoritative. The command resolver returns the exact authored command and
|
|
8
8
|
* does not perform placeholder substitution.
|
|
9
9
|
*/
|
|
10
|
-
import type { BundleIngestor, StoreInstructionSource } from '../../../../src/store/index.ts';
|
|
10
|
+
import type { BundleIngestor, MissingObjectHandler, StoreInstructionSource } from '../../../../src/store/index.ts';
|
|
11
11
|
import type { StepDef, WorkflowDef } from '../../../../src/types.ts';
|
|
12
12
|
import type { DefPolicy, DefVerdict } from '../../../../src/crypto/verify-publication.ts';
|
|
13
13
|
import type { OriginRules } from '../../../../src/crypto/origin-rules.ts';
|
|
@@ -63,6 +63,8 @@ export interface StoreInstructionResolverOptions {
|
|
|
63
63
|
globalRoot: string;
|
|
64
64
|
verifier: BundleIngestor;
|
|
65
65
|
source?: StoreInstructionSource;
|
|
66
|
+
/** Optional one-shot authenticated recovery hook for a true store miss. */
|
|
67
|
+
onMissing?: MissingObjectHandler;
|
|
66
68
|
/** Optional execution-time publication verifier. Omitted means publication trust is unverifiable and command orders refuse. */
|
|
67
69
|
definitionVerifier?: DefinitionVerifier;
|
|
68
70
|
/** Optional execution-time origin verifier. */
|
|
@@ -88,6 +90,7 @@ export declare function createDefaultStoreInstructionResolver(args: {
|
|
|
88
90
|
cwd: string;
|
|
89
91
|
env: Record<string, string | undefined>;
|
|
90
92
|
verifier?: BundleIngestor;
|
|
93
|
+
onMissing?: MissingObjectHandler;
|
|
91
94
|
definitionVerifier?: DefinitionVerifier;
|
|
92
95
|
originVerifier?: OriginVerifier;
|
|
93
96
|
consumedVerifier?: ConsumedVerifier;
|
|
@@ -44,6 +44,7 @@ export function createStoreInstructionResolver(options) {
|
|
|
44
44
|
projectRoot: options.projectRoot,
|
|
45
45
|
globalRoot: options.globalRoot,
|
|
46
46
|
verifier: options.verifier,
|
|
47
|
+
...(options.onMissing !== undefined ? { onMissing: options.onMissing } : {}),
|
|
47
48
|
});
|
|
48
49
|
const env = options.env ?? {};
|
|
49
50
|
let mergedPolicies;
|
|
@@ -304,12 +305,14 @@ export function createDefaultStoreInstructionResolver(args) {
|
|
|
304
305
|
projectRoot,
|
|
305
306
|
globalRoot,
|
|
306
307
|
verifier,
|
|
308
|
+
...(args.onMissing !== undefined ? { onMissing: args.onMissing } : {}),
|
|
307
309
|
});
|
|
308
310
|
return createStoreInstructionResolver({
|
|
309
311
|
projectRoot,
|
|
310
312
|
globalRoot,
|
|
311
313
|
verifier,
|
|
312
314
|
source,
|
|
315
|
+
...(args.onMissing !== undefined ? { onMissing: args.onMissing } : {}),
|
|
313
316
|
definitionVerifier: args.definitionVerifier ?? createExecutionDefinitionVerifier({ env: args.env }),
|
|
314
317
|
originVerifier: args.originVerifier ?? createExecutionOriginVerifier({ env: args.env }),
|
|
315
318
|
...(args.consumedVerifier !== undefined ? { consumedVerifier: args.consumedVerifier } : {}),
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { HubClient } from '../hub/client.ts';
|
|
2
|
-
import type
|
|
2
|
+
import { type ContactHolder } from '../hub/types.ts';
|
|
3
3
|
import type { CommandRunner } from './runner.ts';
|
|
4
4
|
import type { InstructionResolver } from './instructions.ts';
|
|
5
5
|
import { type SubmissionKeyManager } from '../submit-proof.ts';
|
|
@@ -22,7 +22,7 @@ import type { SshProcessAdapter } from '../../../../src/crypto/ssh.ts';
|
|
|
22
22
|
*/
|
|
23
23
|
export declare const CONSUMES_INLINE_MAX_BYTES = 65536;
|
|
24
24
|
/** Discriminated result of an exec run; the role maps these to exit codes. */
|
|
25
|
-
export type ExecOutcome = 'submitted' | 'completed' | 'misroute' | 'workdir-denied' | 'unresolved-instructions' | 'killed' | 'lease-lost' | 'ownership-error' | 'hub-unreachable' | 'submit-rejected' | 'submit-failed' | 'rejected' | 'judge-rejected' | 'judge-no-verdict' | 'reject-failed' | 'stopped';
|
|
25
|
+
export type ExecOutcome = 'submitted' | 'completed' | 'misroute' | 'workdir-denied' | 'unresolved-instructions' | 'killed' | 'lease-lost' | 'ownership-error' | 'hub-unreachable' | 'submit-rejected' | 'submit-failed' | 'command-failed' | 'ask-failed' | 'rejected' | 'judge-rejected' | 'judge-no-verdict' | 'reject-failed' | 'stopped';
|
|
26
26
|
export interface ExecLoopOptions {
|
|
27
27
|
hub: HubClient;
|
|
28
28
|
runner: CommandRunner;
|
|
@@ -19,13 +19,12 @@
|
|
|
19
19
|
* misroute, and no child process is started.
|
|
20
20
|
* 3. RUN + RACE — the runner shells the command out while the lease loop runs.
|
|
21
21
|
* Whichever settles first wins (plan decision 9):
|
|
22
|
-
* - an ordinary command settles
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
* carries the command result.
|
|
22
|
+
* - an ordinary command settles ⇒ build a receipt. A payload reject is
|
|
23
|
+
* delivered FIRST (the hub refuses a reject once the claim has closed,
|
|
24
|
+
* and the last owed submit is what closes it). A successful command
|
|
25
|
+
* submits that receipt to every owed path; a failed command raises an
|
|
26
|
+
* operator question with the receipt as diagnostic context and submits
|
|
27
|
+
* nothing.
|
|
29
28
|
* - a judge command exits 0 ⇒ submit its receipt; a non-zero exit ⇒ send
|
|
30
29
|
* `reject` for `order.judge` without a receipt; signal or machinery
|
|
31
30
|
* failure ⇒ no verdict, leave the claim for the reap path.
|
|
@@ -47,11 +46,17 @@ import { tmpdir } from 'node:os';
|
|
|
47
46
|
import { join, resolve } from 'node:path';
|
|
48
47
|
import { isWorkdirAllowed } from "../agent/workdir.js";
|
|
49
48
|
import { createLeaseLoop } from "../lease/loop.js";
|
|
49
|
+
import { HubError } from "../hub/types.js";
|
|
50
50
|
import { parsePayloadLine } from "./payload.js";
|
|
51
51
|
import { buildReceipt } from "./receipt.js";
|
|
52
52
|
import { buildSubmitProof } from "../submit-proof.js";
|
|
53
53
|
/** sha256 of the empty byte string — the hash for a run with no captured output. */
|
|
54
54
|
const EMPTY_HASH = createHash('sha256').digest('hex');
|
|
55
|
+
/** Submit retries are intentionally narrow: 429 is known not to reach the verb. */
|
|
56
|
+
const SUBMIT_MAX_ATTEMPTS = 3;
|
|
57
|
+
const SUBMIT_RETRY_WINDOW_MS = 30_000;
|
|
58
|
+
const SUBMIT_FALLBACK_DELAYS_MS = [5_000, 10_000];
|
|
59
|
+
const SUBMIT_ERROR_DETAIL_MAX_CHARS = 160;
|
|
55
60
|
/**
|
|
56
61
|
* The largest serialized `consumes` payload delivered inline in
|
|
57
62
|
* `OWENLOOP_CONSUMES`; anything strictly larger goes to a file named by
|
|
@@ -72,6 +77,64 @@ export const CONSUMES_INLINE_MAX_BYTES = 65_536;
|
|
|
72
77
|
function errMsg(e) {
|
|
73
78
|
return e instanceof Error ? e.message : String(e);
|
|
74
79
|
}
|
|
80
|
+
/**
|
|
81
|
+
* Keep edge response bodies out of the run log without changing HubError's
|
|
82
|
+
* transport contract for other callers. In particular, Cloudflare's HTML 1015
|
|
83
|
+
* page contributes no useful diagnosis beyond the 429 classification.
|
|
84
|
+
*/
|
|
85
|
+
function formatSubmitError(e) {
|
|
86
|
+
if (!(e instanceof HubError))
|
|
87
|
+
return errMsg(e);
|
|
88
|
+
if (e.status === 429)
|
|
89
|
+
return 'HTTP 429 (rate limited)';
|
|
90
|
+
const detail = e.message.replace(/\s+/g, ' ').trim();
|
|
91
|
+
const boundedDetail = detail.length <= SUBMIT_ERROR_DETAIL_MAX_CHARS
|
|
92
|
+
? detail
|
|
93
|
+
: `${detail.slice(0, SUBMIT_ERROR_DETAIL_MAX_CHARS - 1)}…`;
|
|
94
|
+
return `HTTP ${e.status}${e.code === undefined ? '' : ` (${e.code})`}${boundedDetail === '' ? '' : `: ${boundedDetail}`}`;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* The one predicate that decides both how child output is logged and whether
|
|
98
|
+
* its receipt may be submitted.
|
|
99
|
+
*
|
|
100
|
+
* runner.ts reports a signalled child with `exitCode: null`, so a separate
|
|
101
|
+
* signal check is unnecessary.
|
|
102
|
+
*/
|
|
103
|
+
function commandSucceeded(result) {
|
|
104
|
+
return result.exitCode === 0 && result.error === undefined;
|
|
105
|
+
}
|
|
106
|
+
/** How a command failed, as an English predicate. */
|
|
107
|
+
function describeCommandFailure(result) {
|
|
108
|
+
if (result.error !== undefined)
|
|
109
|
+
return `could not be run (${result.error})`;
|
|
110
|
+
if (result.signal !== undefined)
|
|
111
|
+
return `was killed by ${result.signal}`;
|
|
112
|
+
return `exited ${result.exitCode}`;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Append the child's captured output to a payload reject reason when it gives
|
|
116
|
+
* the consumer diagnosis that the script's one-line reason cannot.
|
|
117
|
+
*
|
|
118
|
+
* A reject consumer receives only this text. The same `outputTail` already
|
|
119
|
+
* reaches the worker log through `relayChildOutput` and the receipt through
|
|
120
|
+
* `receipt.ts`, but a step that consumes a rejected artifact sees neither:
|
|
121
|
+
* a delivery builder otherwise learns only that local checks failed. Matching
|
|
122
|
+
* `relayChildOutput`'s trailing-newline trim makes the log and reject carry
|
|
123
|
+
* identical bytes, while the label prevents the child's words from being
|
|
124
|
+
* mistaken for the script's reason.
|
|
125
|
+
*
|
|
126
|
+
* Scripts normally print their `##owenloop:payload##` directive to stdout, so
|
|
127
|
+
* that marker remains in the tail just as it does in `relayChildOutput`.
|
|
128
|
+
* Stripping it here would duplicate the runner's bounded marker scanner. The
|
|
129
|
+
* runner caps the tail at 4 KiB, and a large final payload can crowd out its
|
|
130
|
+
* diagnostics; that pre-existing limit belongs in the runner, not this relay.
|
|
131
|
+
*/
|
|
132
|
+
function withCommandOutput(text, outputTail) {
|
|
133
|
+
const tail = outputTail.replace(/\n+$/, '');
|
|
134
|
+
if (tail === '')
|
|
135
|
+
return text;
|
|
136
|
+
return `${text}\n\n--- command output (last ${Buffer.byteLength(tail, 'utf8')} bytes) ---\n${tail}`;
|
|
137
|
+
}
|
|
75
138
|
/**
|
|
76
139
|
* Put the order's consumed inputs on `childEnv`, and return the temp DIRECTORY
|
|
77
140
|
* holding the overflow file — or `undefined` when the payload went inline.
|
|
@@ -141,6 +204,42 @@ function deliverConsumes(childEnv, consumes) {
|
|
|
141
204
|
throw e;
|
|
142
205
|
}
|
|
143
206
|
}
|
|
207
|
+
/**
|
|
208
|
+
* Deliver the owed-artifact reason threads to a command child. The same
|
|
209
|
+
* inline-or-file collision rule as consumes applies: a nested command must
|
|
210
|
+
* never inherit stale feedback from its parent order.
|
|
211
|
+
*/
|
|
212
|
+
function deliverFeedback(childEnv, feedback) {
|
|
213
|
+
if (feedback === undefined) {
|
|
214
|
+
delete childEnv['OWENLOOP_FEEDBACK'];
|
|
215
|
+
delete childEnv['OWENLOOP_FEEDBACK_FILE'];
|
|
216
|
+
return undefined;
|
|
217
|
+
}
|
|
218
|
+
let json;
|
|
219
|
+
try {
|
|
220
|
+
json = JSON.stringify(feedback);
|
|
221
|
+
}
|
|
222
|
+
catch (e) {
|
|
223
|
+
throw new Error(`cannot serialize the order's feedback for OWENLOOP_FEEDBACK: ${errMsg(e)}`);
|
|
224
|
+
}
|
|
225
|
+
if (Buffer.byteLength(json, 'utf8') <= CONSUMES_INLINE_MAX_BYTES) {
|
|
226
|
+
childEnv['OWENLOOP_FEEDBACK'] = json;
|
|
227
|
+
delete childEnv['OWENLOOP_FEEDBACK_FILE'];
|
|
228
|
+
return undefined;
|
|
229
|
+
}
|
|
230
|
+
const dir = mkdtempSync(join(tmpdir(), 'owenloop-consumes-'));
|
|
231
|
+
try {
|
|
232
|
+
const file = join(dir, 'feedback.json');
|
|
233
|
+
writeFileSync(file, json, { encoding: 'utf8', mode: 0o600 });
|
|
234
|
+
childEnv['OWENLOOP_FEEDBACK_FILE'] = file;
|
|
235
|
+
delete childEnv['OWENLOOP_FEEDBACK'];
|
|
236
|
+
return dir;
|
|
237
|
+
}
|
|
238
|
+
catch (e) {
|
|
239
|
+
removeConsumesDir(dir);
|
|
240
|
+
throw e;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
144
243
|
/** Best-effort removal of an overflow directory; a cleanup failure never fails a step. */
|
|
145
244
|
function removeConsumesDir(dir) {
|
|
146
245
|
if (dir === undefined)
|
|
@@ -222,6 +321,24 @@ export function createExecLoop(opts) {
|
|
|
222
321
|
return 'hub-unreachable';
|
|
223
322
|
return 'lease-lost';
|
|
224
323
|
}
|
|
324
|
+
/**
|
|
325
|
+
* Wait for a server-approved submit retry without continuing after the
|
|
326
|
+
* concurrent lease loop has made the claim terminal.
|
|
327
|
+
*/
|
|
328
|
+
async function waitForSubmitRetry(ms) {
|
|
329
|
+
const settled = await Promise.race([
|
|
330
|
+
opts.sleep(ms).then(() => ({ kind: 'slept' })),
|
|
331
|
+
leasePromise.then((outcome) => ({ kind: 'lease', outcome })),
|
|
332
|
+
]);
|
|
333
|
+
return settled.kind === 'lease' ? settled.outcome : undefined;
|
|
334
|
+
}
|
|
335
|
+
/** Stop a failed submit exactly as the pre-retry implementation did. */
|
|
336
|
+
async function failSubmit(path, error) {
|
|
337
|
+
opts.err(`owenloop work exec: submit to ${path} failed: ${formatSubmitError(error)}`);
|
|
338
|
+
lease.stop('submit-failed'); // targeted release (idempotent) — best effort
|
|
339
|
+
await leasePromise;
|
|
340
|
+
return 'submit-failed';
|
|
341
|
+
}
|
|
225
342
|
/** Deliver one reject verb and settle the lease after the response arrives. */
|
|
226
343
|
async function issueReject(path, text, successOutcome) {
|
|
227
344
|
try {
|
|
@@ -251,10 +368,15 @@ export function createExecLoop(opts) {
|
|
|
251
368
|
* Deliver a PAYLOAD reject before the owed submits, and say whether this
|
|
252
369
|
* worker is finished.
|
|
253
370
|
*
|
|
371
|
+
* Its text is the script's reason plus a labeled child-output tail. That
|
|
372
|
+
* tail makes a reject actionable for a consumer that never sees the receipt,
|
|
373
|
+
* where the runner's output would otherwise be the only diagnosis.
|
|
374
|
+
*
|
|
254
375
|
* The distinction from `issueReject` is the third outcome. A judge reject is
|
|
255
376
|
* always the end of the run; a payload reject may leave the claim open, and
|
|
256
|
-
* when it does the
|
|
257
|
-
* terminal `ExecOutcome` or the
|
|
377
|
+
* when it does the caller decides whether the command result may deliver
|
|
378
|
+
* owed receipts. So this returns either a terminal `ExecOutcome` or the
|
|
379
|
+
* sentinel `'continue'`.
|
|
258
380
|
*
|
|
259
381
|
* refused / threw → terminal. `issueReject` has already logged it and
|
|
260
382
|
* settled the lease. Nothing is submitted: a reject that
|
|
@@ -265,17 +387,19 @@ export function createExecLoop(opts) {
|
|
|
265
387
|
* the owed paths stay debts for the next firing.
|
|
266
388
|
* landed, still open → `'continue'`. The rejected path was not consumed
|
|
267
389
|
* here, so the claim and the consume fingerprint are
|
|
268
|
-
* both intact
|
|
390
|
+
* both intact. A successful command may submit its
|
|
391
|
+
* owed receipts; a failed command escalates instead.
|
|
269
392
|
*
|
|
270
393
|
* `closed` is read from the hub's own response rather than re-derived from the
|
|
271
394
|
* order, because the consumed-input test lives in the hub
|
|
272
395
|
* (`reject-artifact.ts` checks `runRow.fingerprint`, which is what the engine
|
|
273
396
|
* recorded at claim time — not what the def declares the step consumes).
|
|
274
397
|
*/
|
|
275
|
-
async function relayPayloadReject(path, text, owed) {
|
|
398
|
+
async function relayPayloadReject(path, text, outputTail, owed) {
|
|
399
|
+
const body = withCommandOutput(text, outputTail);
|
|
276
400
|
let res;
|
|
277
401
|
try {
|
|
278
|
-
res = await hub.reject({ workflow, run: runId, path, text });
|
|
402
|
+
res = await hub.reject({ workflow, run: runId, path, text: body });
|
|
279
403
|
}
|
|
280
404
|
catch (e) {
|
|
281
405
|
opts.err(`owenloop work exec: reject of ${path} failed: ${errMsg(e)}`);
|
|
@@ -298,7 +422,7 @@ export function createExecLoop(opts) {
|
|
|
298
422
|
await leasePromise;
|
|
299
423
|
return 'rejected';
|
|
300
424
|
}
|
|
301
|
-
opts.out(`owenloop work exec: rejected ${path} (not consumed by this firing) —
|
|
425
|
+
opts.out(`owenloop work exec: rejected ${path} (not consumed by this firing) — claim remains open`);
|
|
302
426
|
return 'continue';
|
|
303
427
|
}
|
|
304
428
|
/**
|
|
@@ -322,11 +446,12 @@ export function createExecLoop(opts) {
|
|
|
322
446
|
* schema-rejected — with no trace of what the child said. Reproducing the
|
|
323
447
|
* script by hand outside the engine was the only way to see its stderr.
|
|
324
448
|
*
|
|
325
|
-
* The success half matters because a non-judge command
|
|
326
|
-
*
|
|
327
|
-
* `merge-gate`
|
|
328
|
-
* relay, every deferral produced three identical log
|
|
329
|
-
* measured 40-minute case is recorded in
|
|
449
|
+
* The success half matters because a non-judge command can defer with exit 0
|
|
450
|
+
* and no payload. Failed commands now escalate through `ask`, but
|
|
451
|
+
* `merge-gate` still uses that exit-0 deferral on its `cadence: 2m`
|
|
452
|
+
* re-offer. Before this relay, every deferral produced three identical log
|
|
453
|
+
* lines and no reason; the measured 40-minute case is recorded in
|
|
454
|
+
* `src/config-dir.ts:19-26`.
|
|
330
455
|
*
|
|
331
456
|
* Silence is printed rather than skipped because "this gate printed nothing"
|
|
332
457
|
* is itself the diagnosis, and an absent line is indistinguishable from the
|
|
@@ -337,9 +462,9 @@ export function createExecLoop(opts) {
|
|
|
337
462
|
* 44 KiB per run. That cap is why no volume knob exists: a knob would let a
|
|
338
463
|
* machine silently reintroduce this exact bug. Each line is prefixed so the
|
|
339
464
|
* child's words are never mistaken for exec's own.
|
|
340
|
-
|
|
465
|
+
*/
|
|
341
466
|
function relayChildOutput(result, step) {
|
|
342
|
-
const succeeded = result
|
|
467
|
+
const succeeded = commandSucceeded(result);
|
|
343
468
|
// TWO CHANNELS ON PURPOSE, and this is not tidiness to be refactored away.
|
|
344
469
|
// `opts.err` is the channel a reader treats as trouble; routing a green
|
|
345
470
|
// step's routine output there would make every successful step look like a
|
|
@@ -350,12 +475,7 @@ export function createExecLoop(opts) {
|
|
|
350
475
|
write(`owenloop work exec: the command for step '${step}' succeeded; its output follows`);
|
|
351
476
|
}
|
|
352
477
|
else {
|
|
353
|
-
|
|
354
|
-
? `could not be run (${result.error})`
|
|
355
|
-
: result.signal !== undefined
|
|
356
|
-
? `was killed by ${result.signal}`
|
|
357
|
-
: `exited ${result.exitCode}`;
|
|
358
|
-
write(`owenloop work exec: the command for step '${step}' ${how}; its last output follows`);
|
|
478
|
+
write(`owenloop work exec: the command for step '${step}' ${describeCommandFailure(result)}; its last output follows`);
|
|
359
479
|
}
|
|
360
480
|
const tail = result.outputTail.replace(/\n+$/, '');
|
|
361
481
|
if (tail === '') {
|
|
@@ -365,8 +485,64 @@ export function createExecLoop(opts) {
|
|
|
365
485
|
for (const line of tail.split('\n'))
|
|
366
486
|
write(` | ${line}`);
|
|
367
487
|
}
|
|
368
|
-
/**
|
|
369
|
-
|
|
488
|
+
/** The operator-facing failure text shown by `owenloop inbox`. */
|
|
489
|
+
function failureQuestion(receipt, path, resolvedCommand) {
|
|
490
|
+
const head = `the command for step '${receipt.step}' ${describeCommandFailure(receipt)}, ` +
|
|
491
|
+
`so '${path}' was not produced and no receipt was submitted.\n` +
|
|
492
|
+
`command: ${resolvedCommand}\n` +
|
|
493
|
+
`exit code: ${receipt.exitCode ?? 'none'}` +
|
|
494
|
+
(receipt.signal !== undefined ? `, signal: ${receipt.signal}` : '') +
|
|
495
|
+
(receipt.error !== undefined ? `, error: ${receipt.error}` : '') +
|
|
496
|
+
`\noutput: ${receipt.stdoutBytes} stdout byte(s), ${receipt.stderrBytes} stderr byte(s), ` +
|
|
497
|
+
`hash ${receipt.outputHash}`;
|
|
498
|
+
return withCommandOutput(head, receipt.outputTail);
|
|
499
|
+
}
|
|
500
|
+
/**
|
|
501
|
+
* A failed command does not green anything. Raise a question on its own owed
|
|
502
|
+
* path and retain the receipt as diagnostic context instead.
|
|
503
|
+
*/
|
|
504
|
+
async function escalateCommandFailure(receipt, order, resolvedCommand) {
|
|
505
|
+
// run() returns misroute when order.owes is empty, so this index is safe.
|
|
506
|
+
const path = order.owes[0].path;
|
|
507
|
+
const how = describeCommandFailure(receipt);
|
|
508
|
+
opts.err(`owenloop work exec: the command for step '${order.step}' ${how} — ` +
|
|
509
|
+
`no receipt will be submitted; escalating on ${path}`);
|
|
510
|
+
// ask closes the run, so one firing can raise only one question. The
|
|
511
|
+
// remaining owed paths stay debts for the next firing.
|
|
512
|
+
if (order.owes.length > 1) {
|
|
513
|
+
const rest = order.owes.slice(1).map((owe) => owe.path).join(', ');
|
|
514
|
+
opts.err(`owenloop work exec: ${order.owes.length - 1} other owed path(s) were not escalated ` +
|
|
515
|
+
`because ask closes the run: ${rest}`);
|
|
516
|
+
}
|
|
517
|
+
let res;
|
|
518
|
+
try {
|
|
519
|
+
res = await hub.ask({
|
|
520
|
+
workflow,
|
|
521
|
+
run: runId,
|
|
522
|
+
path,
|
|
523
|
+
question: failureQuestion(receipt, path, resolvedCommand),
|
|
524
|
+
context: JSON.stringify(receipt),
|
|
525
|
+
});
|
|
526
|
+
}
|
|
527
|
+
catch (e) {
|
|
528
|
+
opts.err(`owenloop work exec: ask on ${path} failed: ${errMsg(e)}`);
|
|
529
|
+
lease.stop('ask-failed');
|
|
530
|
+
await leasePromise;
|
|
531
|
+
return 'ask-failed';
|
|
532
|
+
}
|
|
533
|
+
if (res.ok !== true) {
|
|
534
|
+
opts.err(`owenloop work exec: ask on ${path} was refused: ${res.text}`);
|
|
535
|
+
lease.stop('ask-failed', res.closed === true ? { release: false } : undefined);
|
|
536
|
+
await leasePromise;
|
|
537
|
+
return 'ask-failed';
|
|
538
|
+
}
|
|
539
|
+
opts.out(`owenloop work exec: asked ${path} — the artifact is held for a human and nothing was greened`);
|
|
540
|
+
lease.stop('command-failed', res.closed === true ? { release: false } : undefined);
|
|
541
|
+
await leasePromise;
|
|
542
|
+
return 'command-failed';
|
|
543
|
+
}
|
|
544
|
+
/** Build the command receipt, then deliver, reject, or escalate it. */
|
|
545
|
+
async function deliverCommandResult(result, order, resolvedCommand) {
|
|
370
546
|
if (signalled) {
|
|
371
547
|
// The operator killed the work and the command settled before the lease
|
|
372
548
|
// (the release HTTP round-trip is slower than a TERM'd child dying), so
|
|
@@ -450,19 +626,21 @@ export function createExecLoop(opts) {
|
|
|
450
626
|
//
|
|
451
627
|
// `closed: false` — the rejected path was NOT consumed by this firing, so
|
|
452
628
|
// the claim is still open and the fingerprint has not moved. Fall through
|
|
453
|
-
//
|
|
629
|
+
// to the command-result gate: a success submits every owed path, while a
|
|
630
|
+
// failure escalates through ask and submits nothing.
|
|
454
631
|
//
|
|
455
632
|
// A REFUSED reject now submits nothing. That is the point: previously the
|
|
456
633
|
// receipt had already landed and greened a path whose gate had failed.
|
|
457
634
|
if (order.judge === undefined && parsedPayload.reject !== undefined) {
|
|
458
|
-
const rejected = await relayPayloadReject(parsedPayload.reject.path, parsedPayload.reject.text, order.owes.length);
|
|
635
|
+
const rejected = await relayPayloadReject(parsedPayload.reject.path, parsedPayload.reject.text, result.outputTail, order.owes.length);
|
|
459
636
|
if (rejected !== 'continue')
|
|
460
637
|
return rejected;
|
|
461
638
|
}
|
|
639
|
+
if (!commandSucceeded(result))
|
|
640
|
+
return escalateCommandFailure(receipt, order, resolvedCommand);
|
|
462
641
|
for (const owe of order.owes) {
|
|
463
|
-
let
|
|
642
|
+
let proof;
|
|
464
643
|
try {
|
|
465
|
-
let proof;
|
|
466
644
|
if (opts.origin !== undefined) {
|
|
467
645
|
// The immutable order can authorize a judge proof only when the owed
|
|
468
646
|
// path is the fingerprinted judged artifact. Producer command receipts
|
|
@@ -479,21 +657,60 @@ export function createExecLoop(opts) {
|
|
|
479
657
|
...(opts.sshProcess !== undefined ? { sshProcess: opts.sshProcess } : {}),
|
|
480
658
|
});
|
|
481
659
|
}
|
|
482
|
-
res = await hub.submit({
|
|
483
|
-
workflow,
|
|
484
|
-
run: runId,
|
|
485
|
-
path: owe.path,
|
|
486
|
-
value: receipt,
|
|
487
|
-
...(proof !== undefined ? { proof } : {}),
|
|
488
|
-
holder: opts.holder,
|
|
489
|
-
});
|
|
490
660
|
}
|
|
491
661
|
catch (e) {
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
662
|
+
return failSubmit(owe.path, e);
|
|
663
|
+
}
|
|
664
|
+
// One logical receipt gets one request object. A retry replays these same
|
|
665
|
+
// fields (including the proof) rather than producing a fresh signature.
|
|
666
|
+
const submitRequest = Object.freeze({
|
|
667
|
+
workflow,
|
|
668
|
+
run: runId,
|
|
669
|
+
path: owe.path,
|
|
670
|
+
value: receipt,
|
|
671
|
+
...(proof !== undefined ? { proof } : {}),
|
|
672
|
+
holder: opts.holder,
|
|
673
|
+
});
|
|
674
|
+
const retryStartedAt = opts.now();
|
|
675
|
+
let res;
|
|
676
|
+
for (let attempt = 1; attempt <= SUBMIT_MAX_ATTEMPTS; attempt++) {
|
|
677
|
+
try {
|
|
678
|
+
res = await hub.submit(submitRequest);
|
|
679
|
+
break;
|
|
680
|
+
}
|
|
681
|
+
catch (e) {
|
|
682
|
+
const retryable = e instanceof HubError && e.status === 429;
|
|
683
|
+
if (!retryable || attempt === SUBMIT_MAX_ATTEMPTS)
|
|
684
|
+
return failSubmit(owe.path, e);
|
|
685
|
+
const delay = e.retryAfterMs ?? SUBMIT_FALLBACK_DELAYS_MS[attempt - 1];
|
|
686
|
+
const remaining = SUBMIT_RETRY_WINDOW_MS - (opts.now() - retryStartedAt);
|
|
687
|
+
// Never retry ahead of a server delay, nor hold this receipt beyond
|
|
688
|
+
// the bounded retry window just because a malformed/large header said to.
|
|
689
|
+
if (delay > remaining)
|
|
690
|
+
return failSubmit(owe.path, e);
|
|
691
|
+
opts.err(`owenloop work exec: submit to ${owe.path} failed: ${formatSubmitError(e)}; ` +
|
|
692
|
+
`retrying attempt ${attempt + 1}/${SUBMIT_MAX_ATTEMPTS} after ${delay}ms`);
|
|
693
|
+
const leaseOutcome = await waitForSubmitRetry(delay);
|
|
694
|
+
// stop() marks this worker as signalled before its best-effort release
|
|
695
|
+
// finishes. The retry sleep can therefore win this race while the
|
|
696
|
+
// release is still in flight; do not issue a new submit in that gap.
|
|
697
|
+
if (signalled) {
|
|
698
|
+
opts.err(`owenloop work exec: signalled while waiting to retry submit to ${owe.path} — no further receipt sent`);
|
|
699
|
+
// The role exits the process as soon as this loop resolves. Wait for
|
|
700
|
+
// lease finalBreath so that exit cannot abandon its targeted release.
|
|
701
|
+
await leasePromise;
|
|
702
|
+
return 'killed';
|
|
703
|
+
}
|
|
704
|
+
if (leaseOutcome !== undefined) {
|
|
705
|
+
opts.err(`owenloop work exec: lease ${leaseOutcome} while waiting to retry submit to ${owe.path} — no further receipt sent`);
|
|
706
|
+
return mapLeaseDuringRun(leaseOutcome);
|
|
707
|
+
}
|
|
708
|
+
}
|
|
496
709
|
}
|
|
710
|
+
// Each non-success attempt returns from this function, so this only
|
|
711
|
+
// satisfies TypeScript's control-flow analysis after the bounded loop.
|
|
712
|
+
if (res === undefined)
|
|
713
|
+
return failSubmit(owe.path, new Error('submit retry ended without a response'));
|
|
497
714
|
const outcome = res.outcome;
|
|
498
715
|
if (outcome !== 'green' && outcome !== 'submitted') {
|
|
499
716
|
opts.err(`owenloop work exec: submit to ${owe.path} rejected (${outcome ?? 'unknown'}): ${res.text}`);
|
|
@@ -594,6 +811,7 @@ export function createExecLoop(opts) {
|
|
|
594
811
|
// no receipt — but it is not a guarantee to build on. Do not add work here
|
|
595
812
|
// that assumes the child is gone.
|
|
596
813
|
let consumesDir;
|
|
814
|
+
let feedbackDir;
|
|
597
815
|
try {
|
|
598
816
|
let cmd;
|
|
599
817
|
try {
|
|
@@ -618,6 +836,14 @@ export function createExecLoop(opts) {
|
|
|
618
836
|
childEnv['OWENLOOP_WORKFLOW'] = workflow;
|
|
619
837
|
childEnv['OWENLOOP_RUN'] = runId;
|
|
620
838
|
consumesDir = deliverConsumes(childEnv, order.consumes);
|
|
839
|
+
const feedback = order.owes
|
|
840
|
+
.filter((owe) => owe.reasons.length > 0)
|
|
841
|
+
.map((owe) => ({ path: owe.path, reasons: owe.reasons }));
|
|
842
|
+
feedbackDir = deliverFeedback(childEnv, feedback.length > 0 ? feedback : undefined);
|
|
843
|
+
if (order.modifier === undefined)
|
|
844
|
+
delete childEnv['OWENLOOP_MODIFIER'];
|
|
845
|
+
else
|
|
846
|
+
childEnv['OWENLOOP_MODIFIER'] = order.modifier;
|
|
621
847
|
// The order carries no workdir when the step declared neither `workdir:`
|
|
622
848
|
// nor `workdirFrom:` (the hub resolves `workdirFrom` and ships the
|
|
623
849
|
// result, so an absent field IS that signal). Substituting the shift's
|
|
@@ -655,7 +881,7 @@ export function createExecLoop(opts) {
|
|
|
655
881
|
cmd = runner.start(resolvedCommand, startOptions);
|
|
656
882
|
}
|
|
657
883
|
catch (e) {
|
|
658
|
-
return
|
|
884
|
+
return deliverCommandResult(machineryFailure(e), order, resolvedCommand);
|
|
659
885
|
}
|
|
660
886
|
running = cmd;
|
|
661
887
|
opts.out(`owenloop work exec: running ${workflow}/${runId} (step '${order.step}')`);
|
|
@@ -672,10 +898,11 @@ export function createExecLoop(opts) {
|
|
|
672
898
|
opts.err(`owenloop work exec: lease ${outcome.o} while the command ran — killed the command, no receipt submitted`);
|
|
673
899
|
return mapLeaseDuringRun(outcome.o);
|
|
674
900
|
}
|
|
675
|
-
return
|
|
901
|
+
return deliverCommandResult(outcome.r, order, resolvedCommand);
|
|
676
902
|
}
|
|
677
903
|
finally {
|
|
678
904
|
removeConsumesDir(consumesDir);
|
|
905
|
+
removeConsumesDir(feedbackDir);
|
|
679
906
|
}
|
|
680
907
|
}
|
|
681
908
|
function stop(reason) {
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The command receipt (C5, plan decision 8) — the
|
|
3
|
-
*
|
|
4
|
-
* verb has no failure channel,
|
|
5
|
-
* `
|
|
6
|
-
* exit uses `reject` instead
|
|
7
|
-
* Pure construction so it is trivially asserted in tests.
|
|
2
|
+
* The command receipt (C5, plan decision 8) — the artifact value exec submits
|
|
3
|
+
* only for a successful ordinary command and for a judge that exits 0. Because
|
|
4
|
+
* the hub `submit` verb has no failure channel, a failed ordinary command puts
|
|
5
|
+
* this same receipt in `hub.ask`'s diagnostic context and submits nothing. A
|
|
6
|
+
* non-zero judge exit uses `reject` instead, and signal-killed work submits
|
|
7
|
+
* neither. Pure construction so it is trivially asserted in tests.
|
|
8
8
|
*/
|
|
9
9
|
import { type ParsedPayload } from './payload.ts';
|
|
10
10
|
import type { CommandResult } from './runner.ts';
|
|
11
|
-
/**
|
|
11
|
+
/** A command's artifact value on success, or failure diagnostics in `hub.ask` context. */
|
|
12
12
|
export interface CommandReceipt {
|
|
13
13
|
kind: 'command-receipt';
|
|
14
14
|
command: string;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The command receipt (C5, plan decision 8) — the
|
|
3
|
-
*
|
|
4
|
-
* verb has no failure channel,
|
|
5
|
-
* `
|
|
6
|
-
* exit uses `reject` instead
|
|
7
|
-
* Pure construction so it is trivially asserted in tests.
|
|
2
|
+
* The command receipt (C5, plan decision 8) — the artifact value exec submits
|
|
3
|
+
* only for a successful ordinary command and for a judge that exits 0. Because
|
|
4
|
+
* the hub `submit` verb has no failure channel, a failed ordinary command puts
|
|
5
|
+
* this same receipt in `hub.ask`'s diagnostic context and submits nothing. A
|
|
6
|
+
* non-zero judge exit uses `reject` instead, and signal-killed work submits
|
|
7
|
+
* neither. Pure construction so it is trivially asserted in tests.
|
|
8
8
|
*/
|
|
9
9
|
import { parsePayloadLine } from "./payload.js";
|
|
10
10
|
/** Fold a raw `CommandResult` and its order context into the submitted receipt. */
|