owenloop 0.5.24 → 0.5.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/dist/packages/work/src/exec/loop.js +110 -6
- package/dist/packages/work/src/exec/payload.d.ts +107 -5
- package/dist/packages/work/src/exec/payload.js +222 -17
- package/dist/packages/work/src/exec/receipt.d.ts +12 -3
- package/dist/packages/work/src/exec/receipt.js +9 -9
- package/dist/packages/work/src/shift/loop.d.ts +76 -12
- package/dist/packages/work/src/shift/loop.js +157 -24
- package/dist/src/bundle/runtime.d.ts +1 -1
- package/dist/src/bundle/runtime.js +13 -0
- package/dist/src/defs.js +106 -11
- package/dist/src/engine.d.ts +2 -8
- package/dist/src/engine.js +40 -28
- package/dist/src/model.d.ts +12 -2
- package/dist/src/model.js +29 -22
- package/dist/src/order-resolver.js +1 -0
- package/dist/src/types.d.ts +8 -0
- package/docs/authoring.md +31 -1
- package/docs/bundles.md +66 -0
- package/docs/cli.md +6 -2
- package/docs/design.md +36 -5
- package/docs/routing.md +3 -2
- package/docs/wire-contracts.md +22 -1
- package/package.json +1 -1
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/plugin/.mcp.json +1 -1
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
- package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,25 @@ All notable changes to this project are documented here. The format is based on
|
|
|
4
4
|
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
|
|
5
5
|
adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
6
|
|
|
7
|
+
## [0.5.26](https://github.com/typicalday/owenloop/compare/v0.5.25...v0.5.26) (2026-09-01)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
### Features
|
|
11
|
+
|
|
12
|
+
* **work:** return a command step's payload through a file when stdout is too small ([#298](https://github.com/typicalday/owenloop/issues/298)) ([047313b](https://github.com/typicalday/owenloop/commit/047313bfff7ff188f4b3d9ec26ffcedaf6b6c42f))
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
### Bug Fixes
|
|
16
|
+
|
|
17
|
+
* **shift:** hand a dead worker's claim back instead of letting it sit INFLIGHT ([#297](https://github.com/typicalday/owenloop/issues/297)) ([995b15b](https://github.com/typicalday/owenloop/commit/995b15be4b7dbd286f32d70bdcb71a6401c92d48))
|
|
18
|
+
|
|
19
|
+
## [0.5.25](https://github.com/typicalday/owenloop/compare/v0.5.24...v0.5.25) (2026-08-23)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
### Features
|
|
23
|
+
|
|
24
|
+
* **engine:** scope native judges to workflow modifiers ([#295](https://github.com/typicalday/owenloop/issues/295)) ([d1cb9d5](https://github.com/typicalday/owenloop/commit/d1cb9d5f711b7ea5ed545202a0d2e37189ba1d06))
|
|
25
|
+
|
|
7
26
|
## [0.5.24](https://github.com/typicalday/owenloop/compare/v0.5.23...v0.5.24) (2026-08-23)
|
|
8
27
|
|
|
9
28
|
|
|
@@ -47,7 +47,7 @@ import { join, resolve } from 'node:path';
|
|
|
47
47
|
import { isWorkdirAllowed } from "../agent/workdir.js";
|
|
48
48
|
import { createLeaseLoop } from "../lease/loop.js";
|
|
49
49
|
import { HubError } from "../hub/types.js";
|
|
50
|
-
import {
|
|
50
|
+
import { PAYLOAD_FILE_ENV, PAYLOAD_MAX_BYTES, readPayloadFile, resolvePayload } from "./payload.js";
|
|
51
51
|
import { buildReceipt } from "./receipt.js";
|
|
52
52
|
import { buildSubmitProof } from "../submit-proof.js";
|
|
53
53
|
/** sha256 of the empty byte string — the hash for a run with no captured output. */
|
|
@@ -57,6 +57,14 @@ const SUBMIT_MAX_ATTEMPTS = 3;
|
|
|
57
57
|
const SUBMIT_RETRY_WINDOW_MS = 30_000;
|
|
58
58
|
const SUBMIT_FALLBACK_DELAYS_MS = [5_000, 10_000];
|
|
59
59
|
const SUBMIT_ERROR_DETAIL_MAX_CHARS = 160;
|
|
60
|
+
/**
|
|
61
|
+
* The largest payload carried whole into a failed command's `hub.ask` context;
|
|
62
|
+
* see `escalationContext`. 8 KiB is the same order as the 4 KiB `outputTail`
|
|
63
|
+
* the runner already caps for the same reason — big enough that an ordinary
|
|
64
|
+
* structured result still arrives intact for triage, small enough that a run
|
|
65
|
+
* that escalates repeatedly cannot grow its own reason thread without bound.
|
|
66
|
+
*/
|
|
67
|
+
const ASK_CONTEXT_PAYLOAD_MAX_BYTES = 8 * 1024;
|
|
60
68
|
/**
|
|
61
69
|
* The largest serialized `consumes` payload delivered inline in
|
|
62
70
|
* `OWENLOOP_CONSUMES`; anything strictly larger goes to a file named by
|
|
@@ -240,6 +248,52 @@ function deliverFeedback(childEnv, feedback) {
|
|
|
240
248
|
throw e;
|
|
241
249
|
}
|
|
242
250
|
}
|
|
251
|
+
/**
|
|
252
|
+
* Give the command a private path to write its payload to, and return both the
|
|
253
|
+
* path and the directory holding it.
|
|
254
|
+
*
|
|
255
|
+
* ALWAYS ASSIGNED, for the same reason the consumes and feedback names are:
|
|
256
|
+
* `childEnv` starts from `process.env`, and a shift can itself have been
|
|
257
|
+
* launched from inside another command step, so `OWENLOOP_PAYLOAD_FILE` may
|
|
258
|
+
* already carry the PARENT order's path. A conditional assignment would let a
|
|
259
|
+
* child write its result into its parent's file. There is no inline counterpart
|
|
260
|
+
* to `delete` here, because this channel runs the other way: the worker supplies
|
|
261
|
+
* a location, not a value.
|
|
262
|
+
*
|
|
263
|
+
* THE FILE IS NOT CREATED. Its absence after the command exits is exactly the
|
|
264
|
+
* signal that the command returned through stdout instead. The `0700` directory
|
|
265
|
+
* is created, so the command can write without owning a mkdir, and so the
|
|
266
|
+
* payload is unreadable by other users on the machine whatever umask the command
|
|
267
|
+
* writes it under — the worker cannot choose the file's mode when the command is
|
|
268
|
+
* the one creating it.
|
|
269
|
+
*
|
|
270
|
+
* A FAILURE TO CREATE THE DIRECTORY DEGRADES LOUDLY; IT DOES NOT FAIL THE STEP.
|
|
271
|
+
* This runs on every command spawn, where the two overflow directories are
|
|
272
|
+
* created only above 64 KiB, so a read-only rootfs or a full `/tmp` would
|
|
273
|
+
* otherwise turn every previously-working command step on the machine into an
|
|
274
|
+
* escalation carrying a raw ENOSPC — a total outage in exchange for a channel
|
|
275
|
+
* most commands do not use. Instead the variable is UNSET and the command falls
|
|
276
|
+
* back to the stdout marker, which is what it had before this channel existed.
|
|
277
|
+
* Unsetting is not optional: leaving the inherited parent value in place is the
|
|
278
|
+
* one outcome worse than having no channel, because the child would then write
|
|
279
|
+
* its result into its parent order's file.
|
|
280
|
+
*/
|
|
281
|
+
function deliverPayloadFile(childEnv, warn) {
|
|
282
|
+
let dir;
|
|
283
|
+
try {
|
|
284
|
+
dir = mkdtempSync(join(tmpdir(), 'owenloop-payload-'));
|
|
285
|
+
}
|
|
286
|
+
catch (e) {
|
|
287
|
+
delete childEnv[PAYLOAD_FILE_ENV];
|
|
288
|
+
warn(`owenloop work exec: could not create a payload directory under ${tmpdir()}: ${errMsg(e)} — ` +
|
|
289
|
+
`${PAYLOAD_FILE_ENV} is unset for this command, so a payload larger than ` +
|
|
290
|
+
`${PAYLOAD_MAX_BYTES} bytes has no way to return`);
|
|
291
|
+
return {};
|
|
292
|
+
}
|
|
293
|
+
const file = join(dir, 'payload.json');
|
|
294
|
+
childEnv[PAYLOAD_FILE_ENV] = file;
|
|
295
|
+
return { dir, file };
|
|
296
|
+
}
|
|
243
297
|
/** Best-effort removal of an overflow directory; a cleanup failure never fails a step. */
|
|
244
298
|
function removeConsumesDir(dir) {
|
|
245
299
|
if (dir === undefined)
|
|
@@ -497,6 +551,41 @@ export function createExecLoop(opts) {
|
|
|
497
551
|
`hash ${receipt.outputHash}`;
|
|
498
552
|
return withCommandOutput(head, receipt.outputTail);
|
|
499
553
|
}
|
|
554
|
+
/**
|
|
555
|
+
* Serialize a failed command's receipt for `hub.ask`'s diagnostic context,
|
|
556
|
+
* with the payload summarised rather than carried whole.
|
|
557
|
+
*
|
|
558
|
+
* ASK CONTEXT IS PERMANENT RUN STATE, AND NOTHING ELSE BOUNDS IT. The hub
|
|
559
|
+
* concatenates this string into the question text, stores it as a reason entry
|
|
560
|
+
* on the artifact, and redelivers that reason thread inside every subsequent
|
|
561
|
+
* order packet for the run. Unlike `submit`, which refuses an oversized value
|
|
562
|
+
* by name, neither the worker nor the hub caps a question — so whatever goes
|
|
563
|
+
* in here is paid for on every later firing, forever.
|
|
564
|
+
*
|
|
565
|
+
* That was harmless while the only large field was `outputTail`, which the
|
|
566
|
+
* runner already caps at 4 KiB for exactly this reason. The payload file broke
|
|
567
|
+
* the assumption: a payload is read before the exit code is checked, so a
|
|
568
|
+
* command that writes a large report and THEN exits non-zero — a test runner
|
|
569
|
+
* emitting a full JSON report on failure is the ordinary case, not an
|
|
570
|
+
* adversarial one — would push megabytes of it into permanent state.
|
|
571
|
+
*
|
|
572
|
+
* Summarised, not dropped: its size and the fact that one arrived is what a
|
|
573
|
+
* human triaging the failure needs, and the value itself is not lost, because
|
|
574
|
+
* a failed command is re-run rather than resumed. `payloadError` is always
|
|
575
|
+
* kept whole — it is a diagnosis, and it is bounded by construction.
|
|
576
|
+
*/
|
|
577
|
+
function escalationContext(receipt) {
|
|
578
|
+
if (!('payload' in receipt))
|
|
579
|
+
return JSON.stringify(receipt);
|
|
580
|
+
const serialized = JSON.stringify(receipt.payload) ?? 'undefined';
|
|
581
|
+
const bytes = Buffer.byteLength(serialized, 'utf8');
|
|
582
|
+
if (bytes <= ASK_CONTEXT_PAYLOAD_MAX_BYTES)
|
|
583
|
+
return JSON.stringify(receipt);
|
|
584
|
+
return JSON.stringify({
|
|
585
|
+
...receipt,
|
|
586
|
+
payload: `[${bytes} bytes, omitted: over the ${ASK_CONTEXT_PAYLOAD_MAX_BYTES} byte cap on ask context]`,
|
|
587
|
+
});
|
|
588
|
+
}
|
|
500
589
|
/**
|
|
501
590
|
* A failed command does not green anything. Raise a question on its own owed
|
|
502
591
|
* path and retain the receipt as diagnostic context instead.
|
|
@@ -521,7 +610,7 @@ export function createExecLoop(opts) {
|
|
|
521
610
|
run: runId,
|
|
522
611
|
path,
|
|
523
612
|
question: failureQuestion(receipt, path, resolvedCommand),
|
|
524
|
-
context:
|
|
613
|
+
context: escalationContext(receipt),
|
|
525
614
|
});
|
|
526
615
|
}
|
|
527
616
|
catch (e) {
|
|
@@ -542,7 +631,7 @@ export function createExecLoop(opts) {
|
|
|
542
631
|
return 'command-failed';
|
|
543
632
|
}
|
|
544
633
|
/** Build the command receipt, then deliver, reject, or escalate it. */
|
|
545
|
-
async function deliverCommandResult(result, order, resolvedCommand) {
|
|
634
|
+
async function deliverCommandResult(result, order, resolvedCommand, payloadFile) {
|
|
546
635
|
if (signalled) {
|
|
547
636
|
// The operator killed the work and the command settled before the lease
|
|
548
637
|
// (the release HTTP round-trip is slower than a TERM'd child dying), so
|
|
@@ -556,7 +645,15 @@ export function createExecLoop(opts) {
|
|
|
556
645
|
// Before any of the branches below decide what to do about the failure.
|
|
557
646
|
// Every one of them is reachable with a useless log otherwise.
|
|
558
647
|
relayChildOutput(result, order.step);
|
|
559
|
-
|
|
648
|
+
// Read AFTER the child exited, and only on this path. The lease-terminal
|
|
649
|
+
// path never reaches here (see the `finally` at the spawn site: it can unlink
|
|
650
|
+
// before the killed process group is fully reaped), so nothing in this read
|
|
651
|
+
// assumes the child is gone on a path where it might not be.
|
|
652
|
+
const parsedPayload = resolvePayload({
|
|
653
|
+
payloadLine: result.payloadLine,
|
|
654
|
+
payloadOverCap: result.payloadOverCap,
|
|
655
|
+
file: readPayloadFile(payloadFile),
|
|
656
|
+
});
|
|
560
657
|
if (order.judge !== undefined) {
|
|
561
658
|
if (result.exitCode === null) {
|
|
562
659
|
// A signal or machinery failure is not a verdict. Leave the claim for
|
|
@@ -812,6 +909,11 @@ export function createExecLoop(opts) {
|
|
|
812
909
|
// that assumes the child is gone.
|
|
813
910
|
let consumesDir;
|
|
814
911
|
let feedbackDir;
|
|
912
|
+
// The payload directory differs from the two above: it is created on EVERY
|
|
913
|
+
// command spawn, not only on overflow, because the command has to be told
|
|
914
|
+
// where it may write before anyone knows whether it will.
|
|
915
|
+
let payloadDir;
|
|
916
|
+
let payloadFile;
|
|
815
917
|
try {
|
|
816
918
|
let cmd;
|
|
817
919
|
try {
|
|
@@ -836,6 +938,7 @@ export function createExecLoop(opts) {
|
|
|
836
938
|
childEnv['OWENLOOP_WORKFLOW'] = workflow;
|
|
837
939
|
childEnv['OWENLOOP_RUN'] = runId;
|
|
838
940
|
consumesDir = deliverConsumes(childEnv, order.consumes);
|
|
941
|
+
({ dir: payloadDir, file: payloadFile } = deliverPayloadFile(childEnv, opts.err));
|
|
839
942
|
const feedback = order.owes
|
|
840
943
|
.filter((owe) => owe.reasons.length > 0)
|
|
841
944
|
.map((owe) => ({ path: owe.path, reasons: owe.reasons }));
|
|
@@ -881,7 +984,7 @@ export function createExecLoop(opts) {
|
|
|
881
984
|
cmd = runner.start(resolvedCommand, startOptions);
|
|
882
985
|
}
|
|
883
986
|
catch (e) {
|
|
884
|
-
return deliverCommandResult(machineryFailure(e), order, resolvedCommand);
|
|
987
|
+
return deliverCommandResult(machineryFailure(e), order, resolvedCommand, payloadFile);
|
|
885
988
|
}
|
|
886
989
|
running = cmd;
|
|
887
990
|
opts.out(`owenloop work exec: running ${workflow}/${runId} (step '${order.step}')`);
|
|
@@ -898,11 +1001,12 @@ export function createExecLoop(opts) {
|
|
|
898
1001
|
opts.err(`owenloop work exec: lease ${outcome.o} while the command ran — killed the command, no receipt submitted`);
|
|
899
1002
|
return mapLeaseDuringRun(outcome.o);
|
|
900
1003
|
}
|
|
901
|
-
return deliverCommandResult(outcome.r, order, resolvedCommand);
|
|
1004
|
+
return deliverCommandResult(outcome.r, order, resolvedCommand, payloadFile);
|
|
902
1005
|
}
|
|
903
1006
|
finally {
|
|
904
1007
|
removeConsumesDir(consumesDir);
|
|
905
1008
|
removeConsumesDir(feedbackDir);
|
|
1009
|
+
removeConsumesDir(payloadDir);
|
|
906
1010
|
}
|
|
907
1011
|
}
|
|
908
1012
|
function stop(reason) {
|
|
@@ -1,20 +1,66 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The command payload channel.
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* The command payload channel. A command returns its payload through exactly
|
|
3
|
+
* one of two transports, and this module parses whichever it used.
|
|
4
|
+
*
|
|
5
|
+
* STDOUT MARKER. The command prints `PAYLOAD_MARKER` followed by JSON on one
|
|
6
|
+
* line. The runner captures the bounded raw text after the marker; this module
|
|
7
|
+
* parses it. Bounded at `PAYLOAD_MAX_BYTES` because the scanner holds the line
|
|
8
|
+
* in memory while it reads it, and an unbounded line is an unbounded buffer.
|
|
9
|
+
*
|
|
10
|
+
* PAYLOAD FILE. The command writes the same JSON to the path in
|
|
11
|
+
* `OWENLOOP_PAYLOAD_FILE`, which exec always sets. Bounded at
|
|
12
|
+
* `PAYLOAD_FILE_MAX_BYTES` instead, because the bytes are already on disk and
|
|
13
|
+
* the only cost of reading them is the read itself.
|
|
14
|
+
*
|
|
15
|
+
* WHY THE FILE EXISTS AT ALL. The two caps are not cosmetic. A command step
|
|
16
|
+
* whose result exceeds 64 KiB used to have no way to return it: the stdout line
|
|
17
|
+
* was dropped, and the step produced nothing. The hub, meanwhile, has offloaded
|
|
18
|
+
* artifacts above 64 KiB to R2 since long before this channel existed
|
|
19
|
+
* (`hub-core/src/artifacts.ts`), accepting up to 25 MB. So the transport, not
|
|
20
|
+
* the storage, was the ceiling, and only for command steps — an agent step
|
|
21
|
+
* submits over HTTP and never met it.
|
|
22
|
+
*
|
|
23
|
+
* BOTH AT ONCE IS A REFUSAL, NOT A PRECEDENCE RULE. A command that prints a
|
|
24
|
+
* marker line AND writes the file has stated its result twice, and nothing in
|
|
25
|
+
* either transport says which statement it meant. Picking one silently would
|
|
26
|
+
* publish a value the author did not choose. `resolvePayload` refuses and names
|
|
27
|
+
* the conflict instead.
|
|
5
28
|
*/
|
|
6
29
|
/** The exact line prefix a command must print on stdout to emit a payload. */
|
|
7
30
|
export declare const PAYLOAD_MARKER = "##owenloop:payload##";
|
|
8
31
|
/** Maximum UTF-8 byte length of the JSON text after the marker. */
|
|
9
32
|
export declare const PAYLOAD_MAX_BYTES: number;
|
|
33
|
+
/**
|
|
34
|
+
* Maximum byte length of `OWENLOOP_PAYLOAD_FILE`.
|
|
35
|
+
*
|
|
36
|
+
* A COARSE PRE-FILTER, NOT THE AUTHORITATIVE LIMIT. The hub's ceiling is 25 MB
|
|
37
|
+
* (`MAX_ARTIFACT_BYTES` in `hub-core/src/artifacts.ts`), but it is measured on
|
|
38
|
+
* the whole serialized artifact value, which for a command step is the entire
|
|
39
|
+
* `CommandReceipt` — the payload plus the command, up to 4 KiB of `outputTail`,
|
|
40
|
+
* an output hash and seven id fields. A payload at exactly 25 MB therefore
|
|
41
|
+
* serializes to strictly more than 25 MB and is refused hub-side with certainty.
|
|
42
|
+
* This value reserves ~1 MB of envelope headroom so that a payload the worker
|
|
43
|
+
* accepts is not one the hub is guaranteed to reject.
|
|
44
|
+
*
|
|
45
|
+
* It cannot be more than a pre-filter, because the hub's effective cap is
|
|
46
|
+
* `min(tier.artifactMaxBytes, MAX_ARTIFACT_BYTES)` and the lowest tier is 5 MB
|
|
47
|
+
* (`shared/src/tiers.ts`). The worker does not know the org's tier, so the hub
|
|
48
|
+
* stays authoritative and may refuse well below this. What the worker buys by
|
|
49
|
+
* checking at all is the cheap, specific half of the refusal: it can name the
|
|
50
|
+
* file and its exact size before spending an HTTP round trip to be told a
|
|
51
|
+
* serialized value was too large.
|
|
52
|
+
*/
|
|
53
|
+
export declare const PAYLOAD_FILE_MAX_BYTES = 24000000;
|
|
54
|
+
/** The environment variable naming the payload file. Always set on a command spawn. */
|
|
55
|
+
export declare const PAYLOAD_FILE_ENV = "OWENLOOP_PAYLOAD_FILE";
|
|
10
56
|
export interface RejectDirective {
|
|
11
57
|
path: string;
|
|
12
58
|
text: string;
|
|
13
59
|
}
|
|
14
60
|
export interface ParsedPayload {
|
|
15
|
-
/** Present when the
|
|
61
|
+
/** Present when the source contained valid JSON, including `null`/primitives. */
|
|
16
62
|
payload?: unknown;
|
|
17
|
-
/** Present when the
|
|
63
|
+
/** Present when the source or its reject directive could not be used. */
|
|
18
64
|
payloadError?: string;
|
|
19
65
|
/** Present only for a strictly valid reject directive. */
|
|
20
66
|
reject?: RejectDirective;
|
|
@@ -27,3 +73,59 @@ export interface ParsedPayload {
|
|
|
27
73
|
* that a later byte made the line too large.
|
|
28
74
|
*/
|
|
29
75
|
export declare function parsePayloadLine(payloadLine?: string, overCap?: boolean): ParsedPayload;
|
|
76
|
+
/** What the payload file held, as three mutually exclusive outcomes. */
|
|
77
|
+
export interface PayloadFileRead {
|
|
78
|
+
/** The file's trimmed text. Absent when the command did not use this channel. */
|
|
79
|
+
text?: string;
|
|
80
|
+
/** Why the file could not be used. Absent when there was nothing wrong with it. */
|
|
81
|
+
error?: string;
|
|
82
|
+
/**
|
|
83
|
+
* The file's size in bytes, whenever it was measured — including on the
|
|
84
|
+
* over-cap refusal, where the size is the whole point. Absent when the file
|
|
85
|
+
* could not be opened or stat'ed at all. Reported so a conflict can name it
|
|
86
|
+
* after the temp directory is gone.
|
|
87
|
+
*/
|
|
88
|
+
bytes?: number;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Read `OWENLOOP_PAYLOAD_FILE` back after the command exits.
|
|
92
|
+
*
|
|
93
|
+
* ABSENT AND EMPTY BOTH MEAN "THIS CHANNEL WAS NOT USED", and the difference
|
|
94
|
+
* from the marker line is deliberate. Printing the marker is an affirmative act
|
|
95
|
+
* with nothing after it, so an empty marker line is an error. The file already
|
|
96
|
+
* has a path the worker chose and handed over, and `> "$OWENLOOP_PAYLOAD_FILE"`
|
|
97
|
+
* is what a shell does to a path it is about to maybe write — truncating it
|
|
98
|
+
* says nothing about intent. Treating that as an error would make the natural
|
|
99
|
+
* shell idiom fail for commands that then return through stdout.
|
|
100
|
+
*
|
|
101
|
+
* ANYTHING OTHER THAN A REGULAR FILE AT THAT PATH IS AN ERROR, not an absence:
|
|
102
|
+
* exec created an empty private directory and named a file inside it, so a
|
|
103
|
+
* directory, a symlink or a FIFO there is the command having done something
|
|
104
|
+
* unintended.
|
|
105
|
+
*
|
|
106
|
+
* THE WHOLE READ GOES THROUGH ONE DESCRIPTOR, opened once and checked from its
|
|
107
|
+
* own `fstat`. Opening and then re-resolving the path — `statSync` followed by
|
|
108
|
+
* `readFileSync` — leaves a window in which the file the checks passed and the
|
|
109
|
+
* file the bytes came from are not the same file. The child that wrote this path
|
|
110
|
+
* is not necessarily gone: a command may background a grandchild that redirects
|
|
111
|
+
* its own descriptors, and the shell exits while it keeps running.
|
|
112
|
+
*/
|
|
113
|
+
export declare function readPayloadFile(path: string | undefined): PayloadFileRead;
|
|
114
|
+
/** The two transports as the caller observed them, before either is trusted. */
|
|
115
|
+
export interface PayloadSources {
|
|
116
|
+
/** Raw text after the stdout marker, from the runner's bounded scanner. */
|
|
117
|
+
payloadLine?: string;
|
|
118
|
+
/** The scanner saw a marker line that exceeded its cap. */
|
|
119
|
+
payloadOverCap?: boolean;
|
|
120
|
+
/** The result of reading `OWENLOOP_PAYLOAD_FILE`. */
|
|
121
|
+
file?: PayloadFileRead;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Resolve the one payload a command returned, across both transports.
|
|
125
|
+
*
|
|
126
|
+
* The conflict case is a refusal by design; see this module's header. Note that
|
|
127
|
+
* an over-cap marker line still counts as "the command used stdout" — it stated
|
|
128
|
+
* a result there, and the fact that the statement was too long to read does not
|
|
129
|
+
* turn it into a command that only used the file.
|
|
130
|
+
*/
|
|
131
|
+
export declare function resolvePayload(sources: PayloadSources): ParsedPayload;
|
|
@@ -1,18 +1,95 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The command payload channel.
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* The command payload channel. A command returns its payload through exactly
|
|
3
|
+
* one of two transports, and this module parses whichever it used.
|
|
4
|
+
*
|
|
5
|
+
* STDOUT MARKER. The command prints `PAYLOAD_MARKER` followed by JSON on one
|
|
6
|
+
* line. The runner captures the bounded raw text after the marker; this module
|
|
7
|
+
* parses it. Bounded at `PAYLOAD_MAX_BYTES` because the scanner holds the line
|
|
8
|
+
* in memory while it reads it, and an unbounded line is an unbounded buffer.
|
|
9
|
+
*
|
|
10
|
+
* PAYLOAD FILE. The command writes the same JSON to the path in
|
|
11
|
+
* `OWENLOOP_PAYLOAD_FILE`, which exec always sets. Bounded at
|
|
12
|
+
* `PAYLOAD_FILE_MAX_BYTES` instead, because the bytes are already on disk and
|
|
13
|
+
* the only cost of reading them is the read itself.
|
|
14
|
+
*
|
|
15
|
+
* WHY THE FILE EXISTS AT ALL. The two caps are not cosmetic. A command step
|
|
16
|
+
* whose result exceeds 64 KiB used to have no way to return it: the stdout line
|
|
17
|
+
* was dropped, and the step produced nothing. The hub, meanwhile, has offloaded
|
|
18
|
+
* artifacts above 64 KiB to R2 since long before this channel existed
|
|
19
|
+
* (`hub-core/src/artifacts.ts`), accepting up to 25 MB. So the transport, not
|
|
20
|
+
* the storage, was the ceiling, and only for command steps — an agent step
|
|
21
|
+
* submits over HTTP and never met it.
|
|
22
|
+
*
|
|
23
|
+
* BOTH AT ONCE IS A REFUSAL, NOT A PRECEDENCE RULE. A command that prints a
|
|
24
|
+
* marker line AND writes the file has stated its result twice, and nothing in
|
|
25
|
+
* either transport says which statement it meant. Picking one silently would
|
|
26
|
+
* publish a value the author did not choose. `resolvePayload` refuses and names
|
|
27
|
+
* the conflict instead.
|
|
5
28
|
*/
|
|
29
|
+
import { closeSync, constants, fstatSync, openSync, readSync } from 'node:fs';
|
|
6
30
|
/** The exact line prefix a command must print on stdout to emit a payload. */
|
|
7
31
|
export const PAYLOAD_MARKER = '##owenloop:payload##';
|
|
8
32
|
/** Maximum UTF-8 byte length of the JSON text after the marker. */
|
|
9
33
|
export const PAYLOAD_MAX_BYTES = 64 * 1024;
|
|
34
|
+
/**
|
|
35
|
+
* Maximum byte length of `OWENLOOP_PAYLOAD_FILE`.
|
|
36
|
+
*
|
|
37
|
+
* A COARSE PRE-FILTER, NOT THE AUTHORITATIVE LIMIT. The hub's ceiling is 25 MB
|
|
38
|
+
* (`MAX_ARTIFACT_BYTES` in `hub-core/src/artifacts.ts`), but it is measured on
|
|
39
|
+
* the whole serialized artifact value, which for a command step is the entire
|
|
40
|
+
* `CommandReceipt` — the payload plus the command, up to 4 KiB of `outputTail`,
|
|
41
|
+
* an output hash and seven id fields. A payload at exactly 25 MB therefore
|
|
42
|
+
* serializes to strictly more than 25 MB and is refused hub-side with certainty.
|
|
43
|
+
* This value reserves ~1 MB of envelope headroom so that a payload the worker
|
|
44
|
+
* accepts is not one the hub is guaranteed to reject.
|
|
45
|
+
*
|
|
46
|
+
* It cannot be more than a pre-filter, because the hub's effective cap is
|
|
47
|
+
* `min(tier.artifactMaxBytes, MAX_ARTIFACT_BYTES)` and the lowest tier is 5 MB
|
|
48
|
+
* (`shared/src/tiers.ts`). The worker does not know the org's tier, so the hub
|
|
49
|
+
* stays authoritative and may refuse well below this. What the worker buys by
|
|
50
|
+
* checking at all is the cheap, specific half of the refusal: it can name the
|
|
51
|
+
* file and its exact size before spending an HTTP round trip to be told a
|
|
52
|
+
* serialized value was too large.
|
|
53
|
+
*/
|
|
54
|
+
export const PAYLOAD_FILE_MAX_BYTES = 24_000_000;
|
|
55
|
+
/** The environment variable naming the payload file. Always set on a command spawn. */
|
|
56
|
+
export const PAYLOAD_FILE_ENV = 'OWENLOOP_PAYLOAD_FILE';
|
|
10
57
|
function isRecord(value) {
|
|
11
58
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
12
59
|
}
|
|
13
60
|
function hasNonEmptyString(value) {
|
|
14
61
|
return typeof value === 'string' && value.trim().length > 0;
|
|
15
62
|
}
|
|
63
|
+
function errorMessage(error) {
|
|
64
|
+
return error instanceof Error ? error.message : String(error);
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Parse payload JSON that is already known to be non-empty and within its
|
|
68
|
+
* source's cap. Shared by both transports so a reject directive means the same
|
|
69
|
+
* thing however it arrived.
|
|
70
|
+
*/
|
|
71
|
+
function parsePayloadJson(jsonText) {
|
|
72
|
+
let payload;
|
|
73
|
+
try {
|
|
74
|
+
payload = JSON.parse(jsonText);
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
78
|
+
return { payloadError: `payload JSON is malformed: ${message}` };
|
|
79
|
+
}
|
|
80
|
+
if (!isRecord(payload) || !('reject' in payload))
|
|
81
|
+
return { payload };
|
|
82
|
+
const rawReject = payload['reject'];
|
|
83
|
+
if (!isRecord(rawReject)) {
|
|
84
|
+
return { payload, payloadError: 'payload reject directive must be an object' };
|
|
85
|
+
}
|
|
86
|
+
const path = rawReject['path'];
|
|
87
|
+
const text = rawReject['text'];
|
|
88
|
+
if (!hasNonEmptyString(path) || !hasNonEmptyString(text)) {
|
|
89
|
+
return { payload, payloadError: 'payload reject directive requires non-empty string path and text' };
|
|
90
|
+
}
|
|
91
|
+
return { payload, reject: { path, text } };
|
|
92
|
+
}
|
|
16
93
|
/**
|
|
17
94
|
* Parse the raw text after `PAYLOAD_MARKER`.
|
|
18
95
|
*
|
|
@@ -31,24 +108,152 @@ export function parsePayloadLine(payloadLine, overCap = false) {
|
|
|
31
108
|
if (Buffer.byteLength(jsonText, 'utf8') > PAYLOAD_MAX_BYTES) {
|
|
32
109
|
return { payloadError: `payload JSON exceeds the ${PAYLOAD_MAX_BYTES / 1024} KiB cap` };
|
|
33
110
|
}
|
|
34
|
-
|
|
111
|
+
return parsePayloadJson(jsonText);
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Read `OWENLOOP_PAYLOAD_FILE` back after the command exits.
|
|
115
|
+
*
|
|
116
|
+
* ABSENT AND EMPTY BOTH MEAN "THIS CHANNEL WAS NOT USED", and the difference
|
|
117
|
+
* from the marker line is deliberate. Printing the marker is an affirmative act
|
|
118
|
+
* with nothing after it, so an empty marker line is an error. The file already
|
|
119
|
+
* has a path the worker chose and handed over, and `> "$OWENLOOP_PAYLOAD_FILE"`
|
|
120
|
+
* is what a shell does to a path it is about to maybe write — truncating it
|
|
121
|
+
* says nothing about intent. Treating that as an error would make the natural
|
|
122
|
+
* shell idiom fail for commands that then return through stdout.
|
|
123
|
+
*
|
|
124
|
+
* ANYTHING OTHER THAN A REGULAR FILE AT THAT PATH IS AN ERROR, not an absence:
|
|
125
|
+
* exec created an empty private directory and named a file inside it, so a
|
|
126
|
+
* directory, a symlink or a FIFO there is the command having done something
|
|
127
|
+
* unintended.
|
|
128
|
+
*
|
|
129
|
+
* THE WHOLE READ GOES THROUGH ONE DESCRIPTOR, opened once and checked from its
|
|
130
|
+
* own `fstat`. Opening and then re-resolving the path — `statSync` followed by
|
|
131
|
+
* `readFileSync` — leaves a window in which the file the checks passed and the
|
|
132
|
+
* file the bytes came from are not the same file. The child that wrote this path
|
|
133
|
+
* is not necessarily gone: a command may background a grandchild that redirects
|
|
134
|
+
* its own descriptors, and the shell exits while it keeps running.
|
|
135
|
+
*/
|
|
136
|
+
export function readPayloadFile(path) {
|
|
137
|
+
if (path === undefined)
|
|
138
|
+
return {};
|
|
139
|
+
let fd;
|
|
35
140
|
try {
|
|
36
|
-
|
|
141
|
+
// O_NONBLOCK and O_NOFOLLOW are both load-bearing.
|
|
142
|
+
//
|
|
143
|
+
// O_NONBLOCK: a FIFO at this path makes a blocking open wait for a writer
|
|
144
|
+
// that may never arrive. That wait is synchronous and on the worker's only
|
|
145
|
+
// thread, so it stops heartbeats, holds the lease and cannot time out —
|
|
146
|
+
// an operator kill is the only recovery. Opening non-blocking returns a
|
|
147
|
+
// descriptor immediately, and the `isFile` check below then refuses it.
|
|
148
|
+
// On a regular file the flag has no effect at all.
|
|
149
|
+
//
|
|
150
|
+
// O_NOFOLLOW: `stat` follows symlinks, so a regular-file check alone would
|
|
151
|
+
// publish whatever the link pointed at. Same user, so this is not a
|
|
152
|
+
// privilege boundary — but the guard says "a regular file exec named", and
|
|
153
|
+
// following a link silently makes that false.
|
|
154
|
+
fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW);
|
|
37
155
|
}
|
|
38
156
|
catch (error) {
|
|
39
|
-
const
|
|
40
|
-
|
|
157
|
+
const code = error?.code;
|
|
158
|
+
// ENOENT is the ordinary case — the command returned through stdout, or
|
|
159
|
+
// returned nothing. Any other open failure is reported rather than silently
|
|
160
|
+
// read as an absence, because "we could not look" is not "there was
|
|
161
|
+
// nothing there".
|
|
162
|
+
if (code === 'ENOENT')
|
|
163
|
+
return {};
|
|
164
|
+
if (code === 'ELOOP' || code === 'EMLINK') {
|
|
165
|
+
return { error: `${PAYLOAD_FILE_ENV} is a symbolic link, not a regular file` };
|
|
166
|
+
}
|
|
167
|
+
return { error: `cannot read ${PAYLOAD_FILE_ENV}: ${errorMessage(error)}` };
|
|
41
168
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
169
|
+
try {
|
|
170
|
+
const stat = fstatSync(fd);
|
|
171
|
+
if (!stat.isFile()) {
|
|
172
|
+
return { error: `${PAYLOAD_FILE_ENV} is not a regular file` };
|
|
173
|
+
}
|
|
174
|
+
// Checked from the descriptor's own stat rather than after reading, so an
|
|
175
|
+
// oversized file is refused without pulling it into memory first.
|
|
176
|
+
const size = stat.size;
|
|
177
|
+
if (size > PAYLOAD_FILE_MAX_BYTES) {
|
|
178
|
+
return {
|
|
179
|
+
bytes: size,
|
|
180
|
+
error: `${PAYLOAD_FILE_ENV} is ${size} bytes, over the ${PAYLOAD_FILE_MAX_BYTES} byte cap ` +
|
|
181
|
+
'— a payload this large must be written as an artifact, not returned through the payload channel',
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
// Bounded by the size just measured on THIS descriptor, so a file that keeps
|
|
185
|
+
// growing cannot be read past the cap however large it becomes. A file that
|
|
186
|
+
// shrank yields a short read, and a truncated JSON document is reported as
|
|
187
|
+
// malformed — the honest outcome for a file that changed while it was read.
|
|
188
|
+
const buffer = Buffer.alloc(size);
|
|
189
|
+
let filled = 0;
|
|
190
|
+
while (filled < size) {
|
|
191
|
+
const chunk = readSync(fd, buffer, filled, size - filled, null);
|
|
192
|
+
if (chunk === 0)
|
|
193
|
+
break;
|
|
194
|
+
filled += chunk;
|
|
195
|
+
}
|
|
196
|
+
const text = buffer.subarray(0, filled).toString('utf8').trim();
|
|
197
|
+
if (text === '')
|
|
198
|
+
return { bytes: filled };
|
|
199
|
+
return { bytes: filled, text };
|
|
47
200
|
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
if (!hasNonEmptyString(path) || !hasNonEmptyString(text)) {
|
|
51
|
-
return { payload, payloadError: 'payload reject directive requires non-empty string path and text' };
|
|
201
|
+
catch (error) {
|
|
202
|
+
return { error: `cannot read ${PAYLOAD_FILE_ENV}: ${errorMessage(error)}` };
|
|
52
203
|
}
|
|
53
|
-
|
|
204
|
+
finally {
|
|
205
|
+
try {
|
|
206
|
+
closeSync(fd);
|
|
207
|
+
}
|
|
208
|
+
catch {
|
|
209
|
+
// A descriptor that will not close changes nothing about what was read,
|
|
210
|
+
// and throwing here would replace a good result with an unrelated error.
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Describe a both-transports conflict in enough detail to act on it from the
|
|
216
|
+
* receipt alone.
|
|
217
|
+
*
|
|
218
|
+
* The sizes are the point. By the time anyone reads this the private temp
|
|
219
|
+
* directory has been removed, so the file cannot be inspected; and the marker
|
|
220
|
+
* line is frequently printed by a nested helper the author did not write, while
|
|
221
|
+
* the receipt keeps only the last 4 KiB of stdout. Naming both byte counts is
|
|
222
|
+
* usually enough to identify which half of a script did what.
|
|
223
|
+
*
|
|
224
|
+
* A file that could not be read is still a use of the file transport, so it
|
|
225
|
+
* stays a conflict rather than falling back to the marker. Its error rides along
|
|
226
|
+
* instead of being discarded: it is the more specific and more actionable of the
|
|
227
|
+
* two facts, and a caller told only "you used both" would never learn that the
|
|
228
|
+
* file was, say, over the cap.
|
|
229
|
+
*/
|
|
230
|
+
function describeConflict(sources, file) {
|
|
231
|
+
const marker = sources.payloadOverCap === true
|
|
232
|
+
? `over the ${PAYLOAD_MAX_BYTES} byte cap`
|
|
233
|
+
: `${Buffer.byteLength(sources.payloadLine ?? '', 'utf8')} bytes`;
|
|
234
|
+
const fileSize = file.bytes === undefined ? 'unreadable' : `${file.bytes} bytes`;
|
|
235
|
+
const detail = file.error === undefined ? '' : ` The file itself was also unusable: ${file.error}`;
|
|
236
|
+
return (`a payload arrived through both the stdout ${PAYLOAD_MARKER} line (${marker}) and ` +
|
|
237
|
+
`${PAYLOAD_FILE_ENV} (${fileSize}) — a command must return its payload through exactly one of them, ` +
|
|
238
|
+
`because nothing here says which one it meant.${detail}`);
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Resolve the one payload a command returned, across both transports.
|
|
242
|
+
*
|
|
243
|
+
* The conflict case is a refusal by design; see this module's header. Note that
|
|
244
|
+
* an over-cap marker line still counts as "the command used stdout" — it stated
|
|
245
|
+
* a result there, and the fact that the statement was too long to read does not
|
|
246
|
+
* turn it into a command that only used the file.
|
|
247
|
+
*/
|
|
248
|
+
export function resolvePayload(sources) {
|
|
249
|
+
const file = sources.file ?? {};
|
|
250
|
+
const usedMarker = sources.payloadLine !== undefined || sources.payloadOverCap === true;
|
|
251
|
+
const usedFile = file.text !== undefined || file.error !== undefined;
|
|
252
|
+
if (usedMarker && usedFile)
|
|
253
|
+
return { payloadError: describeConflict(sources, file) };
|
|
254
|
+
if (file.error !== undefined)
|
|
255
|
+
return { payloadError: file.error };
|
|
256
|
+
if (file.text !== undefined)
|
|
257
|
+
return parsePayloadJson(file.text);
|
|
258
|
+
return parsePayloadLine(sources.payloadLine, sources.payloadOverCap ?? false);
|
|
54
259
|
}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* non-zero judge exit uses `reject` instead, and signal-killed work submits
|
|
7
7
|
* neither. Pure construction so it is trivially asserted in tests.
|
|
8
8
|
*/
|
|
9
|
-
import {
|
|
9
|
+
import type { ParsedPayload } from './payload.ts';
|
|
10
10
|
import type { CommandResult } from './runner.ts';
|
|
11
11
|
/** A command's artifact value on success, or failure diagnostics in `hub.ask` context. */
|
|
12
12
|
export interface CommandReceipt {
|
|
@@ -37,5 +37,14 @@ export interface ReceiptContext {
|
|
|
37
37
|
run: string;
|
|
38
38
|
step: string;
|
|
39
39
|
}
|
|
40
|
-
/**
|
|
41
|
-
|
|
40
|
+
/**
|
|
41
|
+
* Fold a raw `CommandResult` and its order context into the submitted receipt.
|
|
42
|
+
*
|
|
43
|
+
* `parsedPayload` is required rather than defaulted. A default could only parse
|
|
44
|
+
* the stdout marker, which stopped being the whole story when the payload file
|
|
45
|
+
* was added: it would silently drop a file payload, or publish the marker half
|
|
46
|
+
* of a conflict, and in both cases produce a receipt with no `payloadError` to
|
|
47
|
+
* show anything went wrong. Resolving both transports needs the file read, which
|
|
48
|
+
* only the caller holds, so the caller states the answer.
|
|
49
|
+
*/
|
|
50
|
+
export declare function buildReceipt(result: CommandResult, ctx: ReceiptContext, parsedPayload: ParsedPayload): CommandReceipt;
|