owenloop 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +82 -7
- package/bin/owenloop.mjs +2 -2
- package/dist/add.d.ts +412 -0
- package/dist/add.js +999 -0
- package/dist/cli.d.ts +43 -0
- package/dist/cli.js +1865 -19
- package/dist/defs.d.ts +33 -0
- package/dist/defs.js +112 -19
- package/dist/engine.d.ts +338 -50
- package/dist/engine.js +854 -170
- package/dist/factory.d.ts +19 -0
- package/dist/factory.js +41 -5
- package/dist/hub.d.ts +514 -0
- package/dist/hub.js +783 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2 -1
- package/dist/model.d.ts +36 -0
- package/dist/store.d.ts +14 -2
- package/dist/store.js +269 -21
- package/dist/types.d.ts +92 -0
- package/dist/untar.d.ts +47 -0
- package/dist/untar.js +164 -0
- package/dist/util.d.ts +76 -0
- package/dist/util.js +96 -0
- package/docs/authoring.md +48 -1
- package/docs/cli.md +509 -3
- package/docs/design.md +72 -9
- package/docs/embedding.md +87 -7
- package/package.json +1 -1
package/dist/untar.d.ts
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal, dependency-free `.tar.gz` reader for GitHub codeload tarballs.
|
|
3
|
+
*
|
|
4
|
+
* GitHub's tarball endpoint (`/repos/:owner/:repo/tarball/:ref`) emits a
|
|
5
|
+
* gzipped USTAR/pax stream: short ASCII paths as plain USTAR headers, and
|
|
6
|
+
* paths over 100 chars as a pax extended header (typeflag `'x'`) followed by
|
|
7
|
+
* the real entry. GitHub does NOT emit GNU longname (`'L'`) entries — this
|
|
8
|
+
* reader does not implement that typeflag; a repo path that happens to need
|
|
9
|
+
* it would be silently skipped (unrepresentable, not corrupt).
|
|
10
|
+
*
|
|
11
|
+
* This is intentionally narrow: just enough tar to unpack a GitHub tarball,
|
|
12
|
+
* not a general-purpose tar library.
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* Resource bounds enforced while unpacking a tarball. Defaults are sized for
|
|
16
|
+
* GitHub codeload output and are deliberately generous; tests inject tiny
|
|
17
|
+
* values to exercise the limits without building giant fixtures.
|
|
18
|
+
*
|
|
19
|
+
* - `maxCompressedBytes` (256 MiB): the buffered tarball download itself.
|
|
20
|
+
* - `maxExpandedBytes` (1 GiB): total inflated size — kills gzip bombs at
|
|
21
|
+
* inflate time. GitHub's own recommended repo size sits comfortably below.
|
|
22
|
+
* - `maxFileCount` (50k): number of regular files kept.
|
|
23
|
+
* - `maxFileBytes` (100 MiB): GitHub blocks pushing files larger than this,
|
|
24
|
+
* and LFS content is never in the tarball, so no legit entry exceeds it.
|
|
25
|
+
* - `maxPathLength` (1024): guards against absurdly long entry paths (pax or
|
|
26
|
+
* USTAR prefix-composed) before they are ever used as a filesystem path.
|
|
27
|
+
*/
|
|
28
|
+
export interface TarLimits {
|
|
29
|
+
maxCompressedBytes: number;
|
|
30
|
+
maxExpandedBytes: number;
|
|
31
|
+
maxFileCount: number;
|
|
32
|
+
maxFileBytes: number;
|
|
33
|
+
maxPathLength: number;
|
|
34
|
+
}
|
|
35
|
+
export declare const DEFAULT_TAR_LIMITS: TarLimits;
|
|
36
|
+
/**
|
|
37
|
+
* Extract a gzipped tar archive into a flat map of in-archive path → file
|
|
38
|
+
* bytes. Directory entries are dropped; only regular files are kept. Pax
|
|
39
|
+
* extended headers (typeflag 'x') are honored for long paths; pax global
|
|
40
|
+
* headers (typeflag 'g') are skipped. Any other typeflag is skipped (its
|
|
41
|
+
* data block(s) consumed and discarded so the stream stays in sync).
|
|
42
|
+
*
|
|
43
|
+
* Returns keys as the raw in-archive paths, including the leading
|
|
44
|
+
* `<owner>-<repo>-<sha>/` root-dir component GitHub tarballs always have —
|
|
45
|
+
* callers strip that themselves.
|
|
46
|
+
*/
|
|
47
|
+
export declare function extractTarGz(bytes: Uint8Array, limits?: Partial<TarLimits>): Map<string, Uint8Array>;
|
package/dist/untar.js
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal, dependency-free `.tar.gz` reader for GitHub codeload tarballs.
|
|
3
|
+
*
|
|
4
|
+
* GitHub's tarball endpoint (`/repos/:owner/:repo/tarball/:ref`) emits a
|
|
5
|
+
* gzipped USTAR/pax stream: short ASCII paths as plain USTAR headers, and
|
|
6
|
+
* paths over 100 chars as a pax extended header (typeflag `'x'`) followed by
|
|
7
|
+
* the real entry. GitHub does NOT emit GNU longname (`'L'`) entries — this
|
|
8
|
+
* reader does not implement that typeflag; a repo path that happens to need
|
|
9
|
+
* it would be silently skipped (unrepresentable, not corrupt).
|
|
10
|
+
*
|
|
11
|
+
* This is intentionally narrow: just enough tar to unpack a GitHub tarball,
|
|
12
|
+
* not a general-purpose tar library.
|
|
13
|
+
*/
|
|
14
|
+
import { gunzipSync } from 'node:zlib';
|
|
15
|
+
const BLOCK = 512;
|
|
16
|
+
export const DEFAULT_TAR_LIMITS = {
|
|
17
|
+
maxCompressedBytes: 256 * 1024 * 1024,
|
|
18
|
+
maxExpandedBytes: 1024 * 1024 * 1024,
|
|
19
|
+
maxFileCount: 50_000,
|
|
20
|
+
maxFileBytes: 100 * 1024 * 1024,
|
|
21
|
+
maxPathLength: 1024,
|
|
22
|
+
};
|
|
23
|
+
/** Parse a NUL/space-padded octal field (tar's numeric encoding) into a number. */
|
|
24
|
+
function parseOctal(bytes) {
|
|
25
|
+
let s = '';
|
|
26
|
+
for (const b of bytes) {
|
|
27
|
+
if (b === 0 || b === 32 /* space */)
|
|
28
|
+
continue;
|
|
29
|
+
s += String.fromCharCode(b);
|
|
30
|
+
}
|
|
31
|
+
return s.length ? parseInt(s, 8) : 0;
|
|
32
|
+
}
|
|
33
|
+
function readCString(bytes) {
|
|
34
|
+
const nul = bytes.indexOf(0);
|
|
35
|
+
const slice = nul >= 0 ? bytes.subarray(0, nul) : bytes;
|
|
36
|
+
return Buffer.from(slice).toString('utf8');
|
|
37
|
+
}
|
|
38
|
+
/** Parse a pax extended-header data block's `key=value` records, e.g. `path=...`. */
|
|
39
|
+
function parsePaxRecords(data) {
|
|
40
|
+
const text = Buffer.from(data).toString('utf8');
|
|
41
|
+
const out = new Map();
|
|
42
|
+
let i = 0;
|
|
43
|
+
while (i < text.length) {
|
|
44
|
+
// Each record is "<len> <key>=<value>\n", <len> counts the whole record
|
|
45
|
+
// (including the length prefix itself and the trailing newline).
|
|
46
|
+
const spaceIdx = text.indexOf(' ', i);
|
|
47
|
+
if (spaceIdx < 0)
|
|
48
|
+
break;
|
|
49
|
+
const len = parseInt(text.slice(i, spaceIdx), 10);
|
|
50
|
+
if (!Number.isFinite(len) || len <= 0)
|
|
51
|
+
break;
|
|
52
|
+
const record = text.slice(i, i + len);
|
|
53
|
+
const eq = record.indexOf('=', spaceIdx - i + 1);
|
|
54
|
+
if (eq >= 0) {
|
|
55
|
+
const key = record.slice(spaceIdx - i + 1, eq);
|
|
56
|
+
// record.slice(eq + 1) still has the trailing '\n' the pax format
|
|
57
|
+
// mandates — strip it to get the bare value.
|
|
58
|
+
const value = record.slice(eq + 1).replace(/\n$/, '');
|
|
59
|
+
out.set(key, value);
|
|
60
|
+
}
|
|
61
|
+
i += len;
|
|
62
|
+
}
|
|
63
|
+
return out;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Extract a gzipped tar archive into a flat map of in-archive path → file
|
|
67
|
+
* bytes. Directory entries are dropped; only regular files are kept. Pax
|
|
68
|
+
* extended headers (typeflag 'x') are honored for long paths; pax global
|
|
69
|
+
* headers (typeflag 'g') are skipped. Any other typeflag is skipped (its
|
|
70
|
+
* data block(s) consumed and discarded so the stream stays in sync).
|
|
71
|
+
*
|
|
72
|
+
* Returns keys as the raw in-archive paths, including the leading
|
|
73
|
+
* `<owner>-<repo>-<sha>/` root-dir component GitHub tarballs always have —
|
|
74
|
+
* callers strip that themselves.
|
|
75
|
+
*/
|
|
76
|
+
export function extractTarGz(bytes, limits = {}) {
|
|
77
|
+
const lim = { ...DEFAULT_TAR_LIMITS, ...limits };
|
|
78
|
+
if (bytes.length > lim.maxCompressedBytes) {
|
|
79
|
+
throw new Error(`compressed archive size ${bytes.length} exceeds limit of ${lim.maxCompressedBytes} bytes`);
|
|
80
|
+
}
|
|
81
|
+
let tar;
|
|
82
|
+
try {
|
|
83
|
+
// maxOutputLength caps the inflated size — a gzip bomb aborts here with
|
|
84
|
+
// ERR_BUFFER_TOO_LARGE rather than exhausting memory.
|
|
85
|
+
tar = gunzipSync(bytes, { maxOutputLength: lim.maxExpandedBytes });
|
|
86
|
+
}
|
|
87
|
+
catch (e) {
|
|
88
|
+
if (e.code === 'ERR_BUFFER_TOO_LARGE') {
|
|
89
|
+
throw new Error(`expanded archive size exceeds limit of ${lim.maxExpandedBytes} bytes`);
|
|
90
|
+
}
|
|
91
|
+
throw e;
|
|
92
|
+
}
|
|
93
|
+
const out = new Map();
|
|
94
|
+
let offset = 0;
|
|
95
|
+
let fileCount = 0;
|
|
96
|
+
let pendingLongPath;
|
|
97
|
+
while (offset + BLOCK <= tar.length) {
|
|
98
|
+
const header = tar.subarray(offset, offset + BLOCK);
|
|
99
|
+
// Two consecutive all-zero blocks mark the end of the archive.
|
|
100
|
+
if (header.every((b) => b === 0))
|
|
101
|
+
break;
|
|
102
|
+
const nameField = readCString(header.subarray(0, 100));
|
|
103
|
+
const sizeField = parseOctal(header.subarray(124, 136));
|
|
104
|
+
const typeflag = String.fromCharCode(header[156] ?? 0);
|
|
105
|
+
const prefixField = readCString(header.subarray(345, 500));
|
|
106
|
+
// parseOctal on a corrupt size field can yield NaN; a valid entry is a
|
|
107
|
+
// non-negative integer.
|
|
108
|
+
if (!Number.isInteger(sizeField) || sizeField < 0) {
|
|
109
|
+
throw new Error(`corrupt tar header: invalid size field (got ${sizeField})`);
|
|
110
|
+
}
|
|
111
|
+
offset += BLOCK;
|
|
112
|
+
const dataStart = offset;
|
|
113
|
+
const dataEnd = dataStart + sizeField;
|
|
114
|
+
// subarray silently clamps a past-the-end range, so a truncated archive
|
|
115
|
+
// would otherwise yield a short read instead of an error — guard it.
|
|
116
|
+
if (dataEnd > tar.length) {
|
|
117
|
+
throw new Error(`corrupt tar archive: entry data (${sizeField} bytes) extends past end of archive`);
|
|
118
|
+
}
|
|
119
|
+
const data = tar.subarray(dataStart, dataEnd);
|
|
120
|
+
// Data is padded up to the next 512-byte boundary.
|
|
121
|
+
offset += Math.ceil(sizeField / BLOCK) * BLOCK;
|
|
122
|
+
if (typeflag === '5') {
|
|
123
|
+
// directory — nothing to keep
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
if (typeflag === 'g') {
|
|
127
|
+
// pax global extended header — applies to the whole archive; we don't
|
|
128
|
+
// need any of its records (uid/gid/etc), just skip its data.
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (typeflag === 'x') {
|
|
132
|
+
// pax extended header for the NEXT entry — currently only 'path' matters.
|
|
133
|
+
const records = parsePaxRecords(data);
|
|
134
|
+
const p = records.get('path');
|
|
135
|
+
if (p !== undefined)
|
|
136
|
+
pendingLongPath = p;
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
if (typeflag !== '0' && typeflag !== '\0') {
|
|
140
|
+
// unsupported typeflag (symlink, hardlink, GNU longname, ...) — skip.
|
|
141
|
+
pendingLongPath = undefined;
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
// Regular file.
|
|
145
|
+
if (sizeField > lim.maxFileBytes) {
|
|
146
|
+
throw new Error(`archive entry exceeds per-file size limit of ${lim.maxFileBytes} bytes (${sizeField} bytes)`);
|
|
147
|
+
}
|
|
148
|
+
// Validate the FINAL, fully-resolved name (pax 'path' override or USTAR
|
|
149
|
+
// prefix+name), not the raw 100-byte field — that is what a caller joins.
|
|
150
|
+
const name = pendingLongPath ?? (prefixField ? `${prefixField}/${nameField}` : nameField);
|
|
151
|
+
pendingLongPath = undefined;
|
|
152
|
+
if (name) {
|
|
153
|
+
if (name.length > lim.maxPathLength) {
|
|
154
|
+
throw new Error(`archive entry path length ${name.length} exceeds limit of ${lim.maxPathLength} chars`);
|
|
155
|
+
}
|
|
156
|
+
fileCount += 1;
|
|
157
|
+
if (fileCount > lim.maxFileCount) {
|
|
158
|
+
throw new Error(`archive file count exceeds limit of ${lim.maxFileCount}`);
|
|
159
|
+
}
|
|
160
|
+
out.set(name, new Uint8Array(data));
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return out;
|
|
164
|
+
}
|
package/dist/util.d.ts
CHANGED
|
@@ -1,4 +1,80 @@
|
|
|
1
1
|
/** Small dependency-light helpers. */
|
|
2
|
+
/**
|
|
3
|
+
* `mkdir -p` for a project-state directory (e.g. `.owenloop`), refusing to
|
|
4
|
+
* create it through a symlinked component (SEC-3, parent-directory half).
|
|
5
|
+
*
|
|
6
|
+
* The destination-FILE symlink defense lives in `hub.ts`'s `writeFileAtomic`;
|
|
7
|
+
* this closes the sibling hole where the PARENT directory itself is a symlink.
|
|
8
|
+
* If `.owenloop` is a symlink to a directory elsewhere, `mkdirSync(dir, {
|
|
9
|
+
* recursive: true })` silently succeeds (the target already exists) and every
|
|
10
|
+
* subsequent write lands beneath the link target, outside the project checkout
|
|
11
|
+
* — a hostile checkout can ship `.owenloop -> /somewhere/else` to redirect
|
|
12
|
+
* writes that way.
|
|
13
|
+
*
|
|
14
|
+
* Lives in `util.ts` (engine core) rather than `hub.ts` so the core store
|
|
15
|
+
* factory (`factory.ts`) can share the one helper without core depending on a
|
|
16
|
+
* hub/CLI module (the `boundaries.test.ts` core→hub import boundary).
|
|
17
|
+
*
|
|
18
|
+
* Behavior:
|
|
19
|
+
* 1. `lstat` the dir. If it is a symbolic link (to a directory, to a file, or
|
|
20
|
+
* dangling), throw a clear error naming the path. A dangling symlink matters
|
|
21
|
+
* too: recursive `mkdir` would otherwise CREATE the link target.
|
|
22
|
+
* 2. If it exists and is not a directory (a plain file), throw a clear
|
|
23
|
+
* "not a directory" error rather than letting `mkdirSync` surface a raw
|
|
24
|
+
* EEXIST/ENOTDIR — mirroring `writeFileAtomic`'s directory-refusal courtesy.
|
|
25
|
+
* 3. Otherwise `mkdirSync(dir, { recursive: true })` — creates it fresh when
|
|
26
|
+
* missing, no-op when a real directory already exists.
|
|
27
|
+
*
|
|
28
|
+
* TOCTOU stance matches `writeFileAtomic`: the `lstat` immediately before the
|
|
29
|
+
* `mkdir` defends the STATIC hostile-checkout threat (the attacker controls
|
|
30
|
+
* repo contents at clone time). Unlike `rename`, `mkdir` does not neutralize a
|
|
31
|
+
* symlink raced into place between the `lstat` and the `mkdir` — a live local
|
|
32
|
+
* attacker racing the process is outside this threat model, the same caveat
|
|
33
|
+
* framing `writeFileAtomic` carries.
|
|
34
|
+
*/
|
|
35
|
+
export declare function mkdirRefusingSymlink(dir: string): void;
|
|
36
|
+
/**
|
|
37
|
+
* File-level half of SEC-3: refuse to open the DEFAULT `state.db` (and its
|
|
38
|
+
* SQLite sidecars) through a symlink. PR #47 / `mkdirRefusingSymlink` closed
|
|
39
|
+
* the PARENT-directory hole; this closes the sibling one where `.owenloop` is a
|
|
40
|
+
* REAL directory but `state.db` (or a sidecar) is a symlink to a file
|
|
41
|
+
* elsewhere. Opening the database follows that file symlink, redirecting every
|
|
42
|
+
* write outside the project checkout — a hostile checkout can ship
|
|
43
|
+
* `.owenloop/state.db -> /elsewhere` to do exactly that.
|
|
44
|
+
*
|
|
45
|
+
* Sidecars are guarded, not just the main file: `store.ts` runs
|
|
46
|
+
* `PRAGMA journal_mode = WAL`, so writes flow through `state.db-wal` and
|
|
47
|
+
* coordinate via the mmap'd `state.db-shm`. SQLite's unix VFS opens both with
|
|
48
|
+
* plain `open(2)`, which follows file symlinks unless `SQLITE_OPEN_NOFOLLOW` is
|
|
49
|
+
* passed — and `node:sqlite`'s `DatabaseSync` does not pass it. So a regular
|
|
50
|
+
* (or absent) `state.db` with `state.db-wal -> /elsewhere` still redirects
|
|
51
|
+
* writes; WAL mode does not make the sidecars moot. `state.db-journal` only
|
|
52
|
+
* matters in rollback-journal modes, but a pre-existing hostile db opens in its
|
|
53
|
+
* on-disk journal mode until the WAL switch runs, so its journal can be touched
|
|
54
|
+
* transiently during open — one more loop iteration guards it too.
|
|
55
|
+
*
|
|
56
|
+
* Lives in `util.ts` (engine core) alongside `mkdirRefusingSymlink` so both
|
|
57
|
+
* `factory.ts` (core) and `cli.ts` share the one helper without core depending
|
|
58
|
+
* on a hub/CLI module (the `boundaries.test.ts` core→hub import boundary).
|
|
59
|
+
*
|
|
60
|
+
* Behavior, for each of `<dbPath>`, `-wal`, `-shm`, `-journal`:
|
|
61
|
+
* 1. `lstat` it. Missing → fine (a fresh create).
|
|
62
|
+
* 2. A symbolic link (to a file, a directory, or dangling — `lstat` reports the
|
|
63
|
+
* link itself either way) → throw a clear error naming the path. A dangling
|
|
64
|
+
* link matters too: opening it would otherwise CREATE the link target.
|
|
65
|
+
* 3. Exists but is not a regular file (a directory, FIFO, …) → throw a clear
|
|
66
|
+
* "not a regular file" error rather than letting SQLite surface a rawer one
|
|
67
|
+
* — a symmetry courtesy mirroring `mkdirRefusingSymlink`'s "not a directory"
|
|
68
|
+
* branch.
|
|
69
|
+
* 4. A regular file → fine (a normal reopen).
|
|
70
|
+
*
|
|
71
|
+
* TOCTOU stance matches `writeFileAtomic`/`mkdirRefusingSymlink`: the `lstat`
|
|
72
|
+
* defends the STATIC hostile-checkout threat (the attacker controls repo
|
|
73
|
+
* contents at clone time). A live local attacker racing a symlink into place
|
|
74
|
+
* between the `lstat` and SQLite's open is outside this threat model, the same
|
|
75
|
+
* caveat those siblings carry.
|
|
76
|
+
*/
|
|
77
|
+
export declare function dbPathRefusingSymlink(dbPath: string): void;
|
|
2
78
|
/** Deterministic id from parts — same parts always yield the same id. */
|
|
3
79
|
export declare function detId(prefix: string, ...parts: string[]): string;
|
|
4
80
|
/** Random unique id — for instances (workflow/run) that must not collide. */
|
package/dist/util.js
CHANGED
|
@@ -1,5 +1,101 @@
|
|
|
1
1
|
/** Small dependency-light helpers. */
|
|
2
2
|
import { createHash, randomBytes } from 'node:crypto';
|
|
3
|
+
import { lstatSync, mkdirSync } from 'node:fs';
|
|
4
|
+
/**
|
|
5
|
+
* `mkdir -p` for a project-state directory (e.g. `.owenloop`), refusing to
|
|
6
|
+
* create it through a symlinked component (SEC-3, parent-directory half).
|
|
7
|
+
*
|
|
8
|
+
* The destination-FILE symlink defense lives in `hub.ts`'s `writeFileAtomic`;
|
|
9
|
+
* this closes the sibling hole where the PARENT directory itself is a symlink.
|
|
10
|
+
* If `.owenloop` is a symlink to a directory elsewhere, `mkdirSync(dir, {
|
|
11
|
+
* recursive: true })` silently succeeds (the target already exists) and every
|
|
12
|
+
* subsequent write lands beneath the link target, outside the project checkout
|
|
13
|
+
* — a hostile checkout can ship `.owenloop -> /somewhere/else` to redirect
|
|
14
|
+
* writes that way.
|
|
15
|
+
*
|
|
16
|
+
* Lives in `util.ts` (engine core) rather than `hub.ts` so the core store
|
|
17
|
+
* factory (`factory.ts`) can share the one helper without core depending on a
|
|
18
|
+
* hub/CLI module (the `boundaries.test.ts` core→hub import boundary).
|
|
19
|
+
*
|
|
20
|
+
* Behavior:
|
|
21
|
+
* 1. `lstat` the dir. If it is a symbolic link (to a directory, to a file, or
|
|
22
|
+
* dangling), throw a clear error naming the path. A dangling symlink matters
|
|
23
|
+
* too: recursive `mkdir` would otherwise CREATE the link target.
|
|
24
|
+
* 2. If it exists and is not a directory (a plain file), throw a clear
|
|
25
|
+
* "not a directory" error rather than letting `mkdirSync` surface a raw
|
|
26
|
+
* EEXIST/ENOTDIR — mirroring `writeFileAtomic`'s directory-refusal courtesy.
|
|
27
|
+
* 3. Otherwise `mkdirSync(dir, { recursive: true })` — creates it fresh when
|
|
28
|
+
* missing, no-op when a real directory already exists.
|
|
29
|
+
*
|
|
30
|
+
* TOCTOU stance matches `writeFileAtomic`: the `lstat` immediately before the
|
|
31
|
+
* `mkdir` defends the STATIC hostile-checkout threat (the attacker controls
|
|
32
|
+
* repo contents at clone time). Unlike `rename`, `mkdir` does not neutralize a
|
|
33
|
+
* symlink raced into place between the `lstat` and the `mkdir` — a live local
|
|
34
|
+
* attacker racing the process is outside this threat model, the same caveat
|
|
35
|
+
* framing `writeFileAtomic` carries.
|
|
36
|
+
*/
|
|
37
|
+
export function mkdirRefusingSymlink(dir) {
|
|
38
|
+
const existing = lstatSync(dir, { throwIfNoEntry: false });
|
|
39
|
+
if (existing?.isSymbolicLink()) {
|
|
40
|
+
throw new Error(`refusing to write under ${dir}: it is a symbolic link`);
|
|
41
|
+
}
|
|
42
|
+
if (existing && !existing.isDirectory()) {
|
|
43
|
+
throw new Error(`refusing to write under ${dir}: it is not a directory`);
|
|
44
|
+
}
|
|
45
|
+
mkdirSync(dir, { recursive: true });
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* File-level half of SEC-3: refuse to open the DEFAULT `state.db` (and its
|
|
49
|
+
* SQLite sidecars) through a symlink. PR #47 / `mkdirRefusingSymlink` closed
|
|
50
|
+
* the PARENT-directory hole; this closes the sibling one where `.owenloop` is a
|
|
51
|
+
* REAL directory but `state.db` (or a sidecar) is a symlink to a file
|
|
52
|
+
* elsewhere. Opening the database follows that file symlink, redirecting every
|
|
53
|
+
* write outside the project checkout — a hostile checkout can ship
|
|
54
|
+
* `.owenloop/state.db -> /elsewhere` to do exactly that.
|
|
55
|
+
*
|
|
56
|
+
* Sidecars are guarded, not just the main file: `store.ts` runs
|
|
57
|
+
* `PRAGMA journal_mode = WAL`, so writes flow through `state.db-wal` and
|
|
58
|
+
* coordinate via the mmap'd `state.db-shm`. SQLite's unix VFS opens both with
|
|
59
|
+
* plain `open(2)`, which follows file symlinks unless `SQLITE_OPEN_NOFOLLOW` is
|
|
60
|
+
* passed — and `node:sqlite`'s `DatabaseSync` does not pass it. So a regular
|
|
61
|
+
* (or absent) `state.db` with `state.db-wal -> /elsewhere` still redirects
|
|
62
|
+
* writes; WAL mode does not make the sidecars moot. `state.db-journal` only
|
|
63
|
+
* matters in rollback-journal modes, but a pre-existing hostile db opens in its
|
|
64
|
+
* on-disk journal mode until the WAL switch runs, so its journal can be touched
|
|
65
|
+
* transiently during open — one more loop iteration guards it too.
|
|
66
|
+
*
|
|
67
|
+
* Lives in `util.ts` (engine core) alongside `mkdirRefusingSymlink` so both
|
|
68
|
+
* `factory.ts` (core) and `cli.ts` share the one helper without core depending
|
|
69
|
+
* on a hub/CLI module (the `boundaries.test.ts` core→hub import boundary).
|
|
70
|
+
*
|
|
71
|
+
* Behavior, for each of `<dbPath>`, `-wal`, `-shm`, `-journal`:
|
|
72
|
+
* 1. `lstat` it. Missing → fine (a fresh create).
|
|
73
|
+
* 2. A symbolic link (to a file, a directory, or dangling — `lstat` reports the
|
|
74
|
+
* link itself either way) → throw a clear error naming the path. A dangling
|
|
75
|
+
* link matters too: opening it would otherwise CREATE the link target.
|
|
76
|
+
* 3. Exists but is not a regular file (a directory, FIFO, …) → throw a clear
|
|
77
|
+
* "not a regular file" error rather than letting SQLite surface a rawer one
|
|
78
|
+
* — a symmetry courtesy mirroring `mkdirRefusingSymlink`'s "not a directory"
|
|
79
|
+
* branch.
|
|
80
|
+
* 4. A regular file → fine (a normal reopen).
|
|
81
|
+
*
|
|
82
|
+
* TOCTOU stance matches `writeFileAtomic`/`mkdirRefusingSymlink`: the `lstat`
|
|
83
|
+
* defends the STATIC hostile-checkout threat (the attacker controls repo
|
|
84
|
+
* contents at clone time). A live local attacker racing a symlink into place
|
|
85
|
+
* between the `lstat` and SQLite's open is outside this threat model, the same
|
|
86
|
+
* caveat those siblings carry.
|
|
87
|
+
*/
|
|
88
|
+
export function dbPathRefusingSymlink(dbPath) {
|
|
89
|
+
for (const p of [dbPath, `${dbPath}-wal`, `${dbPath}-shm`, `${dbPath}-journal`]) {
|
|
90
|
+
const existing = lstatSync(p, { throwIfNoEntry: false });
|
|
91
|
+
if (existing?.isSymbolicLink()) {
|
|
92
|
+
throw new Error(`refusing to write to ${p}: it is a symbolic link`);
|
|
93
|
+
}
|
|
94
|
+
if (existing && !existing.isFile()) {
|
|
95
|
+
throw new Error(`refusing to write to ${p}: it is not a regular file`);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
3
99
|
/** Deterministic id from parts — same parts always yield the same id. */
|
|
4
100
|
export function detId(prefix, ...parts) {
|
|
5
101
|
const h = createHash('sha1').update(parts.join(' ')).digest('hex');
|
package/docs/authoring.md
CHANGED
|
@@ -43,7 +43,7 @@ steps:
|
|
|
43
43
|
# maxSchemaFailures: 1 # just for this produce — see design.md §6
|
|
44
44
|
body: | # the prompt; ${WORKFLOW} ${RUN} ${INDEX} are filled in
|
|
45
45
|
Read the proposal and produce a `plan`.
|
|
46
|
-
bodyFile: path/to.md # load body from a file, relative to this workflow's dir; mutually exclusive with body
|
|
46
|
+
bodyFile: path/to.md # load body from a file, relative to this workflow's dir (must resolve inside it); mutually exclusive with body
|
|
47
47
|
|
|
48
48
|
generates: # optional; outputs this step makes that NO step
|
|
49
49
|
- audit_log # consumes. Exempt from dead-end lint; otherwise
|
|
@@ -65,6 +65,14 @@ steps:
|
|
|
65
65
|
invalidates: [plan] # which input stems this step may invalidate
|
|
66
66
|
cadence: "0s" # min spacing between runs (e.g. "30m")
|
|
67
67
|
maxRunsPerDay: 1000
|
|
68
|
+
reapTtl: 30m # duration; per-step stranded-lease TTL (overrides
|
|
69
|
+
# engine reapTtlMs; default 2h). A beat within it
|
|
70
|
+
# keeps the lease fresh.
|
|
71
|
+
maxLease: 2h # duration; opt-in cap on TOTAL lease lifetime,
|
|
72
|
+
# enforced regardless of heartbeats. Omitted =
|
|
73
|
+
# no cap (heartbeats extend indefinitely). Set it
|
|
74
|
+
# only as a runaway backstop — it can reap a
|
|
75
|
+
# still-beating job.
|
|
68
76
|
model: standard # quality tier (fast | standard | strong |
|
|
69
77
|
# strongest) or a literal model id — opaque
|
|
70
78
|
# to the engine, passed through on the order
|
|
@@ -74,6 +82,8 @@ steps:
|
|
|
74
82
|
command: … # required when worker: command; opaque, never shelled out
|
|
75
83
|
spec: # optional opaque config map, shape-checked like x: (a plain map)
|
|
76
84
|
anything: goes # contents never read by the engine; see below
|
|
85
|
+
labels: [nightly, batch] # optional routing labels for peer-orchestrator
|
|
86
|
+
# claim filtering (tick --label); empty = absent; see below
|
|
77
87
|
x: # optional; opaque extension map, passed through
|
|
78
88
|
anything: goes # untouched onto the order (Order.x); see design.md §27.3
|
|
79
89
|
```
|
|
@@ -155,6 +165,35 @@ interpreted by the engine beyond that. All three (`worker`, `command`,
|
|
|
155
165
|
like `model` and `workdir`. See [`docs/design.md` §27.4](design.md) for the
|
|
156
166
|
full contract.
|
|
157
167
|
|
|
168
|
+
## `labels:` — routing a step to a particular tick caller
|
|
169
|
+
|
|
170
|
+
`labels:` is an optional list of opaque strings on a step. It exists so several
|
|
171
|
+
orchestrators ticking the *same* database can split the work between them: a
|
|
172
|
+
tick caller may pass one or more `--label <x>` filters (CLI) or
|
|
173
|
+
`engine.tick(wf, { labels: [...] })` (embedding), and a step is left for a
|
|
174
|
+
*different* caller only when **both** the caller's filter and the step's own
|
|
175
|
+
`labels:` are non-empty and share no value — that step's firings defer with
|
|
176
|
+
reason `label-mismatch` until a caller whose filter matches (or a caller with
|
|
177
|
+
no filter at all) ticks it. An empty `labels:` list normalizes to absent.
|
|
178
|
+
|
|
179
|
+
Two consequences are worth stating plainly:
|
|
180
|
+
|
|
181
|
+
- **Routing, not authorization.** A caller that passes *no* filter claims every
|
|
182
|
+
step, labeled or not — and any worker that can reach the database can tick
|
|
183
|
+
without a filter. Labels are a work-splitting convenience, never a security
|
|
184
|
+
boundary; never rely on them to keep a step away from a worker that shouldn't
|
|
185
|
+
run it.
|
|
186
|
+
- **Starvation hazard.** If every live caller ticks with a label filter and
|
|
187
|
+
none of them intersects a given step's labels, that step's orders sit
|
|
188
|
+
deferred forever with `label-mismatch` — no caller ever claims them.
|
|
189
|
+
Operators running labeled callers should monitor the deferred/blocked entries
|
|
190
|
+
(each folded deferral in a `TickResult` carries its `reason`) so a
|
|
191
|
+
permanently unclaimed step is caught rather than silently stuck.
|
|
192
|
+
|
|
193
|
+
`labels:` is distinct from [`worker:`](#worker--declaring-the-executor):
|
|
194
|
+
`worker:` says what *kind* of executor should run an order once it's claimed;
|
|
195
|
+
`labels:` says *which tick caller* is allowed to claim it in the first place.
|
|
196
|
+
|
|
158
197
|
## `produces:` vs `generates:`
|
|
159
198
|
|
|
160
199
|
A stem under `produces:` is expected to be consumed downstream — owenloop's lint warns
|
|
@@ -440,3 +479,11 @@ Don't put credentials or secrets in an artifact value. Values are persisted
|
|
|
440
479
|
as plaintext in the SQLite store (no encryption at rest) and are copied
|
|
441
480
|
verbatim into the prompt/context of every order that consumes them — anyone
|
|
442
481
|
who can read the database or a downstream job's prompt can read it.
|
|
482
|
+
|
|
483
|
+
Those values are also *retained*, not just transiently persisted: every
|
|
484
|
+
artifact version is kept (the store never overwrites a prior version in place),
|
|
485
|
+
and every run's issued order packet — prompt plus the input values it consumed
|
|
486
|
+
— is written to the SQLite `run` table at claim time and stays there after the
|
|
487
|
+
run closes and the workflow finishes. A secret that flowed through an input
|
|
488
|
+
therefore lives on in the database until the file itself is disposed of — see
|
|
489
|
+
[Storage](../README.md#storage) for the operator's disposal responsibility.
|