owenloop 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -0
- package/README.md +94 -7
- package/bin/owenloop.mjs +2 -2
- package/dist/add.d.ts +412 -0
- package/dist/add.js +999 -0
- package/dist/cli.d.ts +47 -0
- package/dist/cli.js +1809 -19
- package/dist/defs.d.ts +33 -0
- package/dist/defs.js +171 -21
- package/dist/engine.d.ts +395 -40
- package/dist/engine.js +1118 -135
- package/dist/factory.d.ts +19 -0
- package/dist/factory.js +41 -5
- package/dist/hub.d.ts +290 -0
- package/dist/hub.js +408 -0
- package/dist/index.d.ts +3 -2
- package/dist/index.js +2 -1
- package/dist/model.d.ts +36 -0
- package/dist/model.js +20 -9
- package/dist/store.d.ts +14 -2
- package/dist/store.js +285 -35
- package/dist/types.d.ts +120 -0
- package/dist/untar.d.ts +47 -0
- package/dist/untar.js +164 -0
- package/dist/util.d.ts +76 -0
- package/dist/util.js +96 -0
- package/docs/authoring.md +118 -1
- package/docs/cli.md +446 -3
- package/docs/design.md +259 -32
- package/docs/embedding.md +81 -10
- package/examples/workflows/command-worker.yaml +56 -0
- package/package.json +1 -1
package/dist/untar.d.ts
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal, dependency-free `.tar.gz` reader for GitHub codeload tarballs.
|
|
3
|
+
*
|
|
4
|
+
* GitHub's tarball endpoint (`/repos/:owner/:repo/tarball/:ref`) emits a
|
|
5
|
+
* gzipped USTAR/pax stream: short ASCII paths as plain USTAR headers, and
|
|
6
|
+
* paths over 100 chars as a pax extended header (typeflag `'x'`) followed by
|
|
7
|
+
* the real entry. GitHub does NOT emit GNU longname (`'L'`) entries — this
|
|
8
|
+
* reader does not implement that typeflag; a repo path that happens to need
|
|
9
|
+
* it would be silently skipped (unrepresentable, not corrupt).
|
|
10
|
+
*
|
|
11
|
+
* This is intentionally narrow: just enough tar to unpack a GitHub tarball,
|
|
12
|
+
* not a general-purpose tar library.
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* Resource bounds enforced while unpacking a tarball. Defaults are sized for
|
|
16
|
+
* GitHub codeload output and are deliberately generous; tests inject tiny
|
|
17
|
+
* values to exercise the limits without building giant fixtures.
|
|
18
|
+
*
|
|
19
|
+
* - `maxCompressedBytes` (256 MiB): the buffered tarball download itself.
|
|
20
|
+
* - `maxExpandedBytes` (1 GiB): total inflated size — kills gzip bombs at
|
|
21
|
+
* inflate time. GitHub's own recommended repo size sits comfortably below.
|
|
22
|
+
* - `maxFileCount` (50k): number of regular files kept.
|
|
23
|
+
* - `maxFileBytes` (100 MiB): GitHub blocks pushing files larger than this,
|
|
24
|
+
* and LFS content is never in the tarball, so no legit entry exceeds it.
|
|
25
|
+
* - `maxPathLength` (1024): guards against absurdly long entry paths (pax or
|
|
26
|
+
* USTAR prefix-composed) before they are ever used as a filesystem path.
|
|
27
|
+
*/
|
|
28
|
+
export interface TarLimits {
|
|
29
|
+
maxCompressedBytes: number;
|
|
30
|
+
maxExpandedBytes: number;
|
|
31
|
+
maxFileCount: number;
|
|
32
|
+
maxFileBytes: number;
|
|
33
|
+
maxPathLength: number;
|
|
34
|
+
}
|
|
35
|
+
export declare const DEFAULT_TAR_LIMITS: TarLimits;
|
|
36
|
+
/**
|
|
37
|
+
* Extract a gzipped tar archive into a flat map of in-archive path → file
|
|
38
|
+
* bytes. Directory entries are dropped; only regular files are kept. Pax
|
|
39
|
+
* extended headers (typeflag 'x') are honored for long paths; pax global
|
|
40
|
+
* headers (typeflag 'g') are skipped. Any other typeflag is skipped (its
|
|
41
|
+
* data block(s) consumed and discarded so the stream stays in sync).
|
|
42
|
+
*
|
|
43
|
+
* Returns keys as the raw in-archive paths, including the leading
|
|
44
|
+
* `<owner>-<repo>-<sha>/` root-dir component GitHub tarballs always have —
|
|
45
|
+
* callers strip that themselves.
|
|
46
|
+
*/
|
|
47
|
+
export declare function extractTarGz(bytes: Uint8Array, limits?: Partial<TarLimits>): Map<string, Uint8Array>;
|
package/dist/untar.js
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal, dependency-free `.tar.gz` reader for GitHub codeload tarballs.
|
|
3
|
+
*
|
|
4
|
+
* GitHub's tarball endpoint (`/repos/:owner/:repo/tarball/:ref`) emits a
|
|
5
|
+
* gzipped USTAR/pax stream: short ASCII paths as plain USTAR headers, and
|
|
6
|
+
* paths over 100 chars as a pax extended header (typeflag `'x'`) followed by
|
|
7
|
+
* the real entry. GitHub does NOT emit GNU longname (`'L'`) entries — this
|
|
8
|
+
* reader does not implement that typeflag; a repo path that happens to need
|
|
9
|
+
* it would be silently skipped (unrepresentable, not corrupt).
|
|
10
|
+
*
|
|
11
|
+
* This is intentionally narrow: just enough tar to unpack a GitHub tarball,
|
|
12
|
+
* not a general-purpose tar library.
|
|
13
|
+
*/
|
|
14
|
+
import { gunzipSync } from 'node:zlib';
|
|
15
|
+
const BLOCK = 512;
|
|
16
|
+
export const DEFAULT_TAR_LIMITS = {
|
|
17
|
+
maxCompressedBytes: 256 * 1024 * 1024,
|
|
18
|
+
maxExpandedBytes: 1024 * 1024 * 1024,
|
|
19
|
+
maxFileCount: 50_000,
|
|
20
|
+
maxFileBytes: 100 * 1024 * 1024,
|
|
21
|
+
maxPathLength: 1024,
|
|
22
|
+
};
|
|
23
|
+
/** Parse a NUL/space-padded octal field (tar's numeric encoding) into a number. */
|
|
24
|
+
function parseOctal(bytes) {
|
|
25
|
+
let s = '';
|
|
26
|
+
for (const b of bytes) {
|
|
27
|
+
if (b === 0 || b === 32 /* space */)
|
|
28
|
+
continue;
|
|
29
|
+
s += String.fromCharCode(b);
|
|
30
|
+
}
|
|
31
|
+
return s.length ? parseInt(s, 8) : 0;
|
|
32
|
+
}
|
|
33
|
+
function readCString(bytes) {
|
|
34
|
+
const nul = bytes.indexOf(0);
|
|
35
|
+
const slice = nul >= 0 ? bytes.subarray(0, nul) : bytes;
|
|
36
|
+
return Buffer.from(slice).toString('utf8');
|
|
37
|
+
}
|
|
38
|
+
/** Parse a pax extended-header data block's `key=value` records, e.g. `path=...`. */
|
|
39
|
+
function parsePaxRecords(data) {
|
|
40
|
+
const text = Buffer.from(data).toString('utf8');
|
|
41
|
+
const out = new Map();
|
|
42
|
+
let i = 0;
|
|
43
|
+
while (i < text.length) {
|
|
44
|
+
// Each record is "<len> <key>=<value>\n", <len> counts the whole record
|
|
45
|
+
// (including the length prefix itself and the trailing newline).
|
|
46
|
+
const spaceIdx = text.indexOf(' ', i);
|
|
47
|
+
if (spaceIdx < 0)
|
|
48
|
+
break;
|
|
49
|
+
const len = parseInt(text.slice(i, spaceIdx), 10);
|
|
50
|
+
if (!Number.isFinite(len) || len <= 0)
|
|
51
|
+
break;
|
|
52
|
+
const record = text.slice(i, i + len);
|
|
53
|
+
const eq = record.indexOf('=', spaceIdx - i + 1);
|
|
54
|
+
if (eq >= 0) {
|
|
55
|
+
const key = record.slice(spaceIdx - i + 1, eq);
|
|
56
|
+
// record.slice(eq + 1) still has the trailing '\n' the pax format
|
|
57
|
+
// mandates — strip it to get the bare value.
|
|
58
|
+
const value = record.slice(eq + 1).replace(/\n$/, '');
|
|
59
|
+
out.set(key, value);
|
|
60
|
+
}
|
|
61
|
+
i += len;
|
|
62
|
+
}
|
|
63
|
+
return out;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Extract a gzipped tar archive into a flat map of in-archive path → file
|
|
67
|
+
* bytes. Directory entries are dropped; only regular files are kept. Pax
|
|
68
|
+
* extended headers (typeflag 'x') are honored for long paths; pax global
|
|
69
|
+
* headers (typeflag 'g') are skipped. Any other typeflag is skipped (its
|
|
70
|
+
* data block(s) consumed and discarded so the stream stays in sync).
|
|
71
|
+
*
|
|
72
|
+
* Returns keys as the raw in-archive paths, including the leading
|
|
73
|
+
* `<owner>-<repo>-<sha>/` root-dir component GitHub tarballs always have —
|
|
74
|
+
* callers strip that themselves.
|
|
75
|
+
*/
|
|
76
|
+
export function extractTarGz(bytes, limits = {}) {
|
|
77
|
+
const lim = { ...DEFAULT_TAR_LIMITS, ...limits };
|
|
78
|
+
if (bytes.length > lim.maxCompressedBytes) {
|
|
79
|
+
throw new Error(`compressed archive size ${bytes.length} exceeds limit of ${lim.maxCompressedBytes} bytes`);
|
|
80
|
+
}
|
|
81
|
+
let tar;
|
|
82
|
+
try {
|
|
83
|
+
// maxOutputLength caps the inflated size — a gzip bomb aborts here with
|
|
84
|
+
// ERR_BUFFER_TOO_LARGE rather than exhausting memory.
|
|
85
|
+
tar = gunzipSync(bytes, { maxOutputLength: lim.maxExpandedBytes });
|
|
86
|
+
}
|
|
87
|
+
catch (e) {
|
|
88
|
+
if (e.code === 'ERR_BUFFER_TOO_LARGE') {
|
|
89
|
+
throw new Error(`expanded archive size exceeds limit of ${lim.maxExpandedBytes} bytes`);
|
|
90
|
+
}
|
|
91
|
+
throw e;
|
|
92
|
+
}
|
|
93
|
+
const out = new Map();
|
|
94
|
+
let offset = 0;
|
|
95
|
+
let fileCount = 0;
|
|
96
|
+
let pendingLongPath;
|
|
97
|
+
while (offset + BLOCK <= tar.length) {
|
|
98
|
+
const header = tar.subarray(offset, offset + BLOCK);
|
|
99
|
+
// Two consecutive all-zero blocks mark the end of the archive.
|
|
100
|
+
if (header.every((b) => b === 0))
|
|
101
|
+
break;
|
|
102
|
+
const nameField = readCString(header.subarray(0, 100));
|
|
103
|
+
const sizeField = parseOctal(header.subarray(124, 136));
|
|
104
|
+
const typeflag = String.fromCharCode(header[156] ?? 0);
|
|
105
|
+
const prefixField = readCString(header.subarray(345, 500));
|
|
106
|
+
// parseOctal on a corrupt size field can yield NaN; a valid entry is a
|
|
107
|
+
// non-negative integer.
|
|
108
|
+
if (!Number.isInteger(sizeField) || sizeField < 0) {
|
|
109
|
+
throw new Error(`corrupt tar header: invalid size field (got ${sizeField})`);
|
|
110
|
+
}
|
|
111
|
+
offset += BLOCK;
|
|
112
|
+
const dataStart = offset;
|
|
113
|
+
const dataEnd = dataStart + sizeField;
|
|
114
|
+
// subarray silently clamps a past-the-end range, so a truncated archive
|
|
115
|
+
// would otherwise yield a short read instead of an error — guard it.
|
|
116
|
+
if (dataEnd > tar.length) {
|
|
117
|
+
throw new Error(`corrupt tar archive: entry data (${sizeField} bytes) extends past end of archive`);
|
|
118
|
+
}
|
|
119
|
+
const data = tar.subarray(dataStart, dataEnd);
|
|
120
|
+
// Data is padded up to the next 512-byte boundary.
|
|
121
|
+
offset += Math.ceil(sizeField / BLOCK) * BLOCK;
|
|
122
|
+
if (typeflag === '5') {
|
|
123
|
+
// directory — nothing to keep
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
if (typeflag === 'g') {
|
|
127
|
+
// pax global extended header — applies to the whole archive; we don't
|
|
128
|
+
// need any of its records (uid/gid/etc), just skip its data.
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (typeflag === 'x') {
|
|
132
|
+
// pax extended header for the NEXT entry — currently only 'path' matters.
|
|
133
|
+
const records = parsePaxRecords(data);
|
|
134
|
+
const p = records.get('path');
|
|
135
|
+
if (p !== undefined)
|
|
136
|
+
pendingLongPath = p;
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
if (typeflag !== '0' && typeflag !== '\0') {
|
|
140
|
+
// unsupported typeflag (symlink, hardlink, GNU longname, ...) — skip.
|
|
141
|
+
pendingLongPath = undefined;
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
// Regular file.
|
|
145
|
+
if (sizeField > lim.maxFileBytes) {
|
|
146
|
+
throw new Error(`archive entry exceeds per-file size limit of ${lim.maxFileBytes} bytes (${sizeField} bytes)`);
|
|
147
|
+
}
|
|
148
|
+
// Validate the FINAL, fully-resolved name (pax 'path' override or USTAR
|
|
149
|
+
// prefix+name), not the raw 100-byte field — that is what a caller joins.
|
|
150
|
+
const name = pendingLongPath ?? (prefixField ? `${prefixField}/${nameField}` : nameField);
|
|
151
|
+
pendingLongPath = undefined;
|
|
152
|
+
if (name) {
|
|
153
|
+
if (name.length > lim.maxPathLength) {
|
|
154
|
+
throw new Error(`archive entry path length ${name.length} exceeds limit of ${lim.maxPathLength} chars`);
|
|
155
|
+
}
|
|
156
|
+
fileCount += 1;
|
|
157
|
+
if (fileCount > lim.maxFileCount) {
|
|
158
|
+
throw new Error(`archive file count exceeds limit of ${lim.maxFileCount}`);
|
|
159
|
+
}
|
|
160
|
+
out.set(name, new Uint8Array(data));
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return out;
|
|
164
|
+
}
|
package/dist/util.d.ts
CHANGED
|
@@ -1,4 +1,80 @@
|
|
|
1
1
|
/** Small dependency-light helpers. */
|
|
2
|
+
/**
|
|
3
|
+
* `mkdir -p` for a project-state directory (e.g. `.owenloop`), refusing to
|
|
4
|
+
* create it through a symlinked component (SEC-3, parent-directory half).
|
|
5
|
+
*
|
|
6
|
+
* The destination-FILE symlink defense lives in `hub.ts`'s `writeFileAtomic`;
|
|
7
|
+
* this closes the sibling hole where the PARENT directory itself is a symlink.
|
|
8
|
+
* If `.owenloop` is a symlink to a directory elsewhere, `mkdirSync(dir, {
|
|
9
|
+
* recursive: true })` silently succeeds (the target already exists) and every
|
|
10
|
+
* subsequent write lands beneath the link target, outside the project checkout
|
|
11
|
+
* — a hostile checkout can ship `.owenloop -> /somewhere/else` to redirect
|
|
12
|
+
* writes that way.
|
|
13
|
+
*
|
|
14
|
+
* Lives in `util.ts` (engine core) rather than `hub.ts` so the core store
|
|
15
|
+
* factory (`factory.ts`) can share the one helper without core depending on a
|
|
16
|
+
* hub/CLI module (the `boundaries.test.ts` core→hub import boundary).
|
|
17
|
+
*
|
|
18
|
+
* Behavior:
|
|
19
|
+
* 1. `lstat` the dir. If it is a symbolic link (to a directory, to a file, or
|
|
20
|
+
* dangling), throw a clear error naming the path. A dangling symlink matters
|
|
21
|
+
* too: recursive `mkdir` would otherwise CREATE the link target.
|
|
22
|
+
* 2. If it exists and is not a directory (a plain file), throw a clear
|
|
23
|
+
* "not a directory" error rather than letting `mkdirSync` surface a raw
|
|
24
|
+
* EEXIST/ENOTDIR — mirroring `writeFileAtomic`'s directory-refusal courtesy.
|
|
25
|
+
* 3. Otherwise `mkdirSync(dir, { recursive: true })` — creates it fresh when
|
|
26
|
+
* missing, no-op when a real directory already exists.
|
|
27
|
+
*
|
|
28
|
+
* TOCTOU stance matches `writeFileAtomic`: the `lstat` immediately before the
|
|
29
|
+
* `mkdir` defends the STATIC hostile-checkout threat (the attacker controls
|
|
30
|
+
* repo contents at clone time). Unlike `rename`, `mkdir` does not neutralize a
|
|
31
|
+
* symlink raced into place between the `lstat` and the `mkdir` — a live local
|
|
32
|
+
* attacker racing the process is outside this threat model, the same caveat
|
|
33
|
+
* framing `writeFileAtomic` carries.
|
|
34
|
+
*/
|
|
35
|
+
export declare function mkdirRefusingSymlink(dir: string): void;
|
|
36
|
+
/**
|
|
37
|
+
* File-level half of SEC-3: refuse to open the DEFAULT `state.db` (and its
|
|
38
|
+
* SQLite sidecars) through a symlink. PR #47 / `mkdirRefusingSymlink` closed
|
|
39
|
+
* the PARENT-directory hole; this closes the sibling one where `.owenloop` is a
|
|
40
|
+
* REAL directory but `state.db` (or a sidecar) is a symlink to a file
|
|
41
|
+
* elsewhere. Opening the database follows that file symlink, redirecting every
|
|
42
|
+
* write outside the project checkout — a hostile checkout can ship
|
|
43
|
+
* `.owenloop/state.db -> /elsewhere` to do exactly that.
|
|
44
|
+
*
|
|
45
|
+
* Sidecars are guarded, not just the main file: `store.ts` runs
|
|
46
|
+
* `PRAGMA journal_mode = WAL`, so writes flow through `state.db-wal` and
|
|
47
|
+
* coordinate via the mmap'd `state.db-shm`. SQLite's unix VFS opens both with
|
|
48
|
+
* plain `open(2)`, which follows file symlinks unless `SQLITE_OPEN_NOFOLLOW` is
|
|
49
|
+
* passed — and `node:sqlite`'s `DatabaseSync` does not pass it. So a regular
|
|
50
|
+
* (or absent) `state.db` with `state.db-wal -> /elsewhere` still redirects
|
|
51
|
+
* writes; WAL mode does not make the sidecars moot. `state.db-journal` only
|
|
52
|
+
* matters in rollback-journal modes, but a pre-existing hostile db opens in its
|
|
53
|
+
* on-disk journal mode until the WAL switch runs, so its journal can be touched
|
|
54
|
+
* transiently during open — one more loop iteration guards it too.
|
|
55
|
+
*
|
|
56
|
+
* Lives in `util.ts` (engine core) alongside `mkdirRefusingSymlink` so both
|
|
57
|
+
* `factory.ts` (core) and `cli.ts` share the one helper without core depending
|
|
58
|
+
* on a hub/CLI module (the `boundaries.test.ts` core→hub import boundary).
|
|
59
|
+
*
|
|
60
|
+
* Behavior, for each of `<dbPath>`, `-wal`, `-shm`, `-journal`:
|
|
61
|
+
* 1. `lstat` it. Missing → fine (a fresh create).
|
|
62
|
+
* 2. A symbolic link (to a file, a directory, or dangling — `lstat` reports the
|
|
63
|
+
* link itself either way) → throw a clear error naming the path. A dangling
|
|
64
|
+
* link matters too: opening it would otherwise CREATE the link target.
|
|
65
|
+
* 3. Exists but is not a regular file (a directory, FIFO, …) → throw a clear
|
|
66
|
+
* "not a regular file" error rather than letting SQLite surface a rawer one
|
|
67
|
+
* — a symmetry courtesy mirroring `mkdirRefusingSymlink`'s "not a directory"
|
|
68
|
+
* branch.
|
|
69
|
+
* 4. A regular file → fine (a normal reopen).
|
|
70
|
+
*
|
|
71
|
+
* TOCTOU stance matches `writeFileAtomic`/`mkdirRefusingSymlink`: the `lstat`
|
|
72
|
+
* defends the STATIC hostile-checkout threat (the attacker controls repo
|
|
73
|
+
* contents at clone time). A live local attacker racing a symlink into place
|
|
74
|
+
* between the `lstat` and SQLite's open is outside this threat model, the same
|
|
75
|
+
* caveat those siblings carry.
|
|
76
|
+
*/
|
|
77
|
+
export declare function dbPathRefusingSymlink(dbPath: string): void;
|
|
2
78
|
/** Deterministic id from parts — same parts always yield the same id. */
|
|
3
79
|
export declare function detId(prefix: string, ...parts: string[]): string;
|
|
4
80
|
/** Random unique id — for instances (workflow/run) that must not collide. */
|
package/dist/util.js
CHANGED
|
@@ -1,5 +1,101 @@
|
|
|
1
1
|
/** Small dependency-light helpers. */
|
|
2
2
|
import { createHash, randomBytes } from 'node:crypto';
|
|
3
|
+
import { lstatSync, mkdirSync } from 'node:fs';
|
|
4
|
+
/**
|
|
5
|
+
* `mkdir -p` for a project-state directory (e.g. `.owenloop`), refusing to
|
|
6
|
+
* create it through a symlinked component (SEC-3, parent-directory half).
|
|
7
|
+
*
|
|
8
|
+
* The destination-FILE symlink defense lives in `hub.ts`'s `writeFileAtomic`;
|
|
9
|
+
* this closes the sibling hole where the PARENT directory itself is a symlink.
|
|
10
|
+
* If `.owenloop` is a symlink to a directory elsewhere, `mkdirSync(dir, {
|
|
11
|
+
* recursive: true })` silently succeeds (the target already exists) and every
|
|
12
|
+
* subsequent write lands beneath the link target, outside the project checkout
|
|
13
|
+
* — a hostile checkout can ship `.owenloop -> /somewhere/else` to redirect
|
|
14
|
+
* writes that way.
|
|
15
|
+
*
|
|
16
|
+
* Lives in `util.ts` (engine core) rather than `hub.ts` so the core store
|
|
17
|
+
* factory (`factory.ts`) can share the one helper without core depending on a
|
|
18
|
+
* hub/CLI module (the `boundaries.test.ts` core→hub import boundary).
|
|
19
|
+
*
|
|
20
|
+
* Behavior:
|
|
21
|
+
* 1. `lstat` the dir. If it is a symbolic link (to a directory, to a file, or
|
|
22
|
+
* dangling), throw a clear error naming the path. A dangling symlink matters
|
|
23
|
+
* too: recursive `mkdir` would otherwise CREATE the link target.
|
|
24
|
+
* 2. If it exists and is not a directory (a plain file), throw a clear
|
|
25
|
+
* "not a directory" error rather than letting `mkdirSync` surface a raw
|
|
26
|
+
* EEXIST/ENOTDIR — mirroring `writeFileAtomic`'s directory-refusal courtesy.
|
|
27
|
+
* 3. Otherwise `mkdirSync(dir, { recursive: true })` — creates it fresh when
|
|
28
|
+
* missing, no-op when a real directory already exists.
|
|
29
|
+
*
|
|
30
|
+
* TOCTOU stance matches `writeFileAtomic`: the `lstat` immediately before the
|
|
31
|
+
* `mkdir` defends the STATIC hostile-checkout threat (the attacker controls
|
|
32
|
+
* repo contents at clone time). Unlike `rename`, `mkdir` does not neutralize a
|
|
33
|
+
* symlink raced into place between the `lstat` and the `mkdir` — a live local
|
|
34
|
+
* attacker racing the process is outside this threat model, the same caveat
|
|
35
|
+
* framing `writeFileAtomic` carries.
|
|
36
|
+
*/
|
|
37
|
+
export function mkdirRefusingSymlink(dir) {
|
|
38
|
+
const existing = lstatSync(dir, { throwIfNoEntry: false });
|
|
39
|
+
if (existing?.isSymbolicLink()) {
|
|
40
|
+
throw new Error(`refusing to write under ${dir}: it is a symbolic link`);
|
|
41
|
+
}
|
|
42
|
+
if (existing && !existing.isDirectory()) {
|
|
43
|
+
throw new Error(`refusing to write under ${dir}: it is not a directory`);
|
|
44
|
+
}
|
|
45
|
+
mkdirSync(dir, { recursive: true });
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* File-level half of SEC-3: refuse to open the DEFAULT `state.db` (and its
|
|
49
|
+
* SQLite sidecars) through a symlink. PR #47 / `mkdirRefusingSymlink` closed
|
|
50
|
+
* the PARENT-directory hole; this closes the sibling one where `.owenloop` is a
|
|
51
|
+
* REAL directory but `state.db` (or a sidecar) is a symlink to a file
|
|
52
|
+
* elsewhere. Opening the database follows that file symlink, redirecting every
|
|
53
|
+
* write outside the project checkout — a hostile checkout can ship
|
|
54
|
+
* `.owenloop/state.db -> /elsewhere` to do exactly that.
|
|
55
|
+
*
|
|
56
|
+
* Sidecars are guarded, not just the main file: `store.ts` runs
|
|
57
|
+
* `PRAGMA journal_mode = WAL`, so writes flow through `state.db-wal` and
|
|
58
|
+
* coordinate via the mmap'd `state.db-shm`. SQLite's unix VFS opens both with
|
|
59
|
+
* plain `open(2)`, which follows file symlinks unless `SQLITE_OPEN_NOFOLLOW` is
|
|
60
|
+
* passed — and `node:sqlite`'s `DatabaseSync` does not pass it. So a regular
|
|
61
|
+
* (or absent) `state.db` with `state.db-wal -> /elsewhere` still redirects
|
|
62
|
+
* writes; WAL mode does not make the sidecars moot. `state.db-journal` only
|
|
63
|
+
* matters in rollback-journal modes, but a pre-existing hostile db opens in its
|
|
64
|
+
* on-disk journal mode until the WAL switch runs, so its journal can be touched
|
|
65
|
+
* transiently during open — one more loop iteration guards it too.
|
|
66
|
+
*
|
|
67
|
+
* Lives in `util.ts` (engine core) alongside `mkdirRefusingSymlink` so both
|
|
68
|
+
* `factory.ts` (core) and `cli.ts` share the one helper without core depending
|
|
69
|
+
* on a hub/CLI module (the `boundaries.test.ts` core→hub import boundary).
|
|
70
|
+
*
|
|
71
|
+
* Behavior, for each of `<dbPath>`, `-wal`, `-shm`, `-journal`:
|
|
72
|
+
* 1. `lstat` it. Missing → fine (a fresh create).
|
|
73
|
+
* 2. A symbolic link (to a file, a directory, or dangling — `lstat` reports the
|
|
74
|
+
* link itself either way) → throw a clear error naming the path. A dangling
|
|
75
|
+
* link matters too: opening it would otherwise CREATE the link target.
|
|
76
|
+
* 3. Exists but is not a regular file (a directory, FIFO, …) → throw a clear
|
|
77
|
+
* "not a regular file" error rather than letting SQLite surface a rawer one
|
|
78
|
+
* — a symmetry courtesy mirroring `mkdirRefusingSymlink`'s "not a directory"
|
|
79
|
+
* branch.
|
|
80
|
+
* 4. A regular file → fine (a normal reopen).
|
|
81
|
+
*
|
|
82
|
+
* TOCTOU stance matches `writeFileAtomic`/`mkdirRefusingSymlink`: the `lstat`
|
|
83
|
+
* defends the STATIC hostile-checkout threat (the attacker controls repo
|
|
84
|
+
* contents at clone time). A live local attacker racing a symlink into place
|
|
85
|
+
* between the `lstat` and SQLite's open is outside this threat model, the same
|
|
86
|
+
* caveat those siblings carry.
|
|
87
|
+
*/
|
|
88
|
+
export function dbPathRefusingSymlink(dbPath) {
|
|
89
|
+
for (const p of [dbPath, `${dbPath}-wal`, `${dbPath}-shm`, `${dbPath}-journal`]) {
|
|
90
|
+
const existing = lstatSync(p, { throwIfNoEntry: false });
|
|
91
|
+
if (existing?.isSymbolicLink()) {
|
|
92
|
+
throw new Error(`refusing to write to ${p}: it is a symbolic link`);
|
|
93
|
+
}
|
|
94
|
+
if (existing && !existing.isFile()) {
|
|
95
|
+
throw new Error(`refusing to write to ${p}: it is not a regular file`);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
3
99
|
/** Deterministic id from parts — same parts always yield the same id. */
|
|
4
100
|
export function detId(prefix, ...parts) {
|
|
5
101
|
const h = createHash('sha1').update(parts.join(' ')).digest('hex');
|
package/docs/authoring.md
CHANGED
|
@@ -43,7 +43,7 @@ steps:
|
|
|
43
43
|
# maxSchemaFailures: 1 # just for this produce — see design.md §6
|
|
44
44
|
body: | # the prompt; ${WORKFLOW} ${RUN} ${INDEX} are filled in
|
|
45
45
|
Read the proposal and produce a `plan`.
|
|
46
|
-
bodyFile: path/to.md # load body from a file, relative to this workflow's dir; mutually exclusive with body
|
|
46
|
+
bodyFile: path/to.md # load body from a file, relative to this workflow's dir (must resolve inside it); mutually exclusive with body
|
|
47
47
|
|
|
48
48
|
generates: # optional; outputs this step makes that NO step
|
|
49
49
|
- audit_log # consumes. Exempt from dead-end lint; otherwise
|
|
@@ -65,11 +65,25 @@ steps:
|
|
|
65
65
|
invalidates: [plan] # which input stems this step may invalidate
|
|
66
66
|
cadence: "0s" # min spacing between runs (e.g. "30m")
|
|
67
67
|
maxRunsPerDay: 1000
|
|
68
|
+
reapTtl: 30m # duration; per-step stranded-lease TTL (overrides
|
|
69
|
+
# engine reapTtlMs; default 2h). A beat within it
|
|
70
|
+
# keeps the lease fresh.
|
|
71
|
+
maxLease: 2h # duration; opt-in cap on TOTAL lease lifetime,
|
|
72
|
+
# enforced regardless of heartbeats. Omitted =
|
|
73
|
+
# no cap (heartbeats extend indefinitely). Set it
|
|
74
|
+
# only as a runaway backstop — it can reap a
|
|
75
|
+
# still-beating job.
|
|
68
76
|
model: standard # quality tier (fast | standard | strong |
|
|
69
77
|
# strongest) or a literal model id — opaque
|
|
70
78
|
# to the engine, passed through on the order
|
|
71
79
|
# (see below)
|
|
72
80
|
workdir: … # opaque hint passed through on the order; omitted when unset
|
|
81
|
+
worker: agent # opaque executor label (default: agent); see below
|
|
82
|
+
command: … # required when worker: command; opaque, never shelled out
|
|
83
|
+
spec: # optional opaque config map, shape-checked like x: (a plain map)
|
|
84
|
+
anything: goes # contents never read by the engine; see below
|
|
85
|
+
labels: [nightly, batch] # optional routing labels for peer-orchestrator
|
|
86
|
+
# claim filtering (tick --label); empty = absent; see below
|
|
73
87
|
x: # optional; opaque extension map, passed through
|
|
74
88
|
anything: goes # untouched onto the order (Order.x); see design.md §27.3
|
|
75
89
|
```
|
|
@@ -98,6 +112,88 @@ a literal model id. Pin an exact model when you need reproducibility — just
|
|
|
98
112
|
know the def is now host-specific, on purpose. Omit `model:` entirely and the
|
|
99
113
|
dispatcher uses its default.
|
|
100
114
|
|
|
115
|
+
## `worker:` — declaring the executor
|
|
116
|
+
|
|
117
|
+
Every step in every def written before this feature dispatches to an LLM
|
|
118
|
+
agent — that's still the default, and omitting `worker:` entirely leaves a
|
|
119
|
+
def byte-for-byte unchanged. `worker:` lets a step opt into a *different*
|
|
120
|
+
kind of executor instead: a shell command, a webhook, a browser-automation
|
|
121
|
+
runner, anything a dispatcher on the other end of `tick` knows how to run.
|
|
122
|
+
The engine never executes anything itself — `worker:` is an opaque label
|
|
123
|
+
that rides the order (same pass-through contract as `model`) for whatever
|
|
124
|
+
drives your workers to switch on.
|
|
125
|
+
|
|
126
|
+
```yaml
|
|
127
|
+
steps:
|
|
128
|
+
- name: tester
|
|
129
|
+
consumes: [reviewed_plan]
|
|
130
|
+
produces: [result]
|
|
131
|
+
worker: command # opaque to the engine; default is 'agent'
|
|
132
|
+
command: npm test # required when worker: command; never parsed or shelled out
|
|
133
|
+
spec: # optional opaque config for the dispatcher
|
|
134
|
+
timeout: 300
|
|
135
|
+
workdir: .
|
|
136
|
+
body: "" # not required for worker: command
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Two shape rules, both enforced at load time, everything else opaque:
|
|
140
|
+
|
|
141
|
+
- **`worker: command` requires `command:`.** A `command`-worker step with no
|
|
142
|
+
`command:` is a load-time `DefError` — the dispatcher would have nothing to
|
|
143
|
+
run.
|
|
144
|
+
- **`worker: agent` (explicit) requires a real `body:`.** Omitting `worker:`
|
|
145
|
+
still defaults to `'agent'` and still needs no body-shape check beyond
|
|
146
|
+
today's behavior — this rule only fires when a step *explicitly* writes
|
|
147
|
+
`worker: agent` and leaves `body:` empty, which is almost certainly a
|
|
148
|
+
mistake (an agent step with no prompt).
|
|
149
|
+
|
|
150
|
+
Any other `worker` value (`browser-automation`, `webhook`, your own label) is
|
|
151
|
+
fully opaque — no `body:`/`command:` requirement at all. A definition can
|
|
152
|
+
optionally declare `workers: [agent, command]` at the top level as a typo
|
|
153
|
+
guard; when present, `validateDef` rejects any step (or judge — see below)
|
|
154
|
+
whose effective worker (after the `agent` default) isn't in the list:
|
|
155
|
+
|
|
156
|
+
```yaml
|
|
157
|
+
name: my-workflow
|
|
158
|
+
workers: [agent, command] # optional allow-list; absent = any worker string accepted
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
`command:` and `spec:` are opaque the same way `x:` is: `command` is
|
|
162
|
+
shape-checked as a string, `spec` as a plain map, and neither is ever read or
|
|
163
|
+
interpreted by the engine beyond that. All three (`worker`, `command`,
|
|
164
|
+
`spec`) ride through `buildOrder` onto the emitted `Order` untouched, exactly
|
|
165
|
+
like `model` and `workdir`. See [`docs/design.md` §27.4](design.md) for the
|
|
166
|
+
full contract.
|
|
167
|
+
|
|
168
|
+
## `labels:` — routing a step to a particular tick caller
|
|
169
|
+
|
|
170
|
+
`labels:` is an optional list of opaque strings on a step. It exists so several
|
|
171
|
+
orchestrators ticking the *same* database can split the work between them: a
|
|
172
|
+
tick caller may pass one or more `--label <x>` filters (CLI) or
|
|
173
|
+
`engine.tick(wf, { labels: [...] })` (embedding), and a step is left for a
|
|
174
|
+
*different* caller only when **both** the caller's filter and the step's own
|
|
175
|
+
`labels:` are non-empty and share no value — that step's firings defer with
|
|
176
|
+
reason `label-mismatch` until a caller whose filter matches (or a caller with
|
|
177
|
+
no filter at all) ticks it. An empty `labels:` list normalizes to absent.
|
|
178
|
+
|
|
179
|
+
Two consequences are worth stating plainly:
|
|
180
|
+
|
|
181
|
+
- **Routing, not authorization.** A caller that passes *no* filter claims every
|
|
182
|
+
step, labeled or not — and any worker that can reach the database can tick
|
|
183
|
+
without a filter. Labels are a work-splitting convenience, never a security
|
|
184
|
+
boundary; never rely on them to keep a step away from a worker that shouldn't
|
|
185
|
+
run it.
|
|
186
|
+
- **Starvation hazard.** If every live caller ticks with a label filter and
|
|
187
|
+
none of them intersects a given step's labels, that step's orders sit
|
|
188
|
+
deferred forever with `label-mismatch` — no caller ever claims them.
|
|
189
|
+
Operators running labeled callers should monitor the deferred/blocked entries
|
|
190
|
+
(each folded deferral in a `TickResult` carries its `reason`) so a
|
|
191
|
+
permanently unclaimed step is caught rather than silently stuck.
|
|
192
|
+
|
|
193
|
+
`labels:` is distinct from [`worker:`](#worker--declaring-the-executor):
|
|
194
|
+
`worker:` says what *kind* of executor should run an order once it's claimed;
|
|
195
|
+
`labels:` says *which tick caller* is allowed to claim it in the first place.
|
|
196
|
+
|
|
101
197
|
## `produces:` vs `generates:`
|
|
102
198
|
|
|
103
199
|
A stem under `produces:` is expected to be consumed downstream — owenloop's lint warns
|
|
@@ -137,11 +233,24 @@ steps:
|
|
|
137
233
|
model: strong # optional, per-judge model tier
|
|
138
234
|
inputs: true # optional, default false — judge also
|
|
139
235
|
# reads the producer's inputs (question)
|
|
236
|
+
- name: ci-gate
|
|
237
|
+
body: "unused by a command judge, but still a required field" # every
|
|
238
|
+
# judge needs body/bodyFile regardless of worker
|
|
239
|
+
worker: command # judges can be deterministic too — same
|
|
240
|
+
command: scripts/ci-gate.sh # worker:/command: contract as a normal step
|
|
140
241
|
maxAttempts: 5 # producer's cap (default for every produce on this step)
|
|
141
242
|
# — also bounds judge-reject → rebuild loops; `report`
|
|
142
243
|
# above could set its own maxAttempts: to override it
|
|
143
244
|
```
|
|
144
245
|
|
|
246
|
+
A judge entry accepts the same `worker:`/`command:`/`spec:` fields as a
|
|
247
|
+
normal step (see [`worker:` above](#worker--declaring-the-executor)) — a
|
|
248
|
+
judge can be a deterministic check (a script's exit status) instead of an
|
|
249
|
+
LLM verdict. Note `body:`/`bodyFile:` is still required on every judge
|
|
250
|
+
regardless of `worker:` — that requirement is orthogonal to this feature and
|
|
251
|
+
applies even to a `worker: command` judge (the field just goes unread by a
|
|
252
|
+
non-agent dispatcher).
|
|
253
|
+
|
|
145
254
|
Each judge is a real step under the hood — it fires its own worker order
|
|
146
255
|
through the normal pipeline, with its own throttles (`cadence:`,
|
|
147
256
|
`maxRunsPerDay:`) and retry/timeout behavior. When `researcher` commits
|
|
@@ -370,3 +479,11 @@ Don't put credentials or secrets in an artifact value. Values are persisted
|
|
|
370
479
|
as plaintext in the SQLite store (no encryption at rest) and are copied
|
|
371
480
|
verbatim into the prompt/context of every order that consumes them — anyone
|
|
372
481
|
who can read the database or a downstream job's prompt can read it.
|
|
482
|
+
|
|
483
|
+
Those values are also *retained*, not just transiently persisted: every
|
|
484
|
+
artifact version is kept (the store never overwrites a prior version in place),
|
|
485
|
+
and every run's issued order packet — prompt plus the input values it consumed
|
|
486
|
+
— is written to the SQLite `run` table at claim time and stays there after the
|
|
487
|
+
run closes and the workflow finishes. A secret that flowed through an input
|
|
488
|
+
therefore lives on in the database until the file itself is disposed of — see
|
|
489
|
+
[Storage](../README.md#storage) for the operator's disposal responsibility.
|