@niwa-labs/proc 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +45 -2
- package/package.json +18 -4
- package/src/artifact.ts +225 -0
- package/src/box-rules.ts +40 -0
- package/src/flow.ts +146 -0
- package/src/index.ts +316 -0
- package/src/records.ts +184 -0
- package/src/refusal.ts +17 -0
- package/src/supervise.ts +152 -0
- package/test/artifact.test.ts +214 -0
- package/test/budget.test.ts +55 -0
- package/test/cancel.test.ts +81 -0
- package/test/exec.test.ts +154 -0
- package/test/flow.test.ts +107 -0
- package/test/grant.test.ts +136 -0
- package/test/live.test.ts +178 -0
- package/test/records.test.ts +46 -0
- package/test/refusal.test.ts +113 -0
- package/test/run.test.ts +273 -0
- package/test/supervise.test.ts +127 -0
- package/test/support.ts +178 -0
- package/tsconfig.json +6 -0
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
// The artifact of an output: named by its content, kept by the host, bounded in the result, scrubbed of secrets.
|
|
2
|
+
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
import { existsSync, readdirSync } from "node:fs";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { expect, test } from "bun:test";
|
|
7
|
+
|
|
8
|
+
import type { ArtifactRef } from "@niwa-labs/core";
|
|
9
|
+
import { createNiwa, tempHome } from "@niwa-labs/testing";
|
|
10
|
+
|
|
11
|
+
import { OutputCapture, Scrubber } from "../src/artifact.ts";
|
|
12
|
+
|
|
13
|
+
const enc = (text: string) => new TextEncoder().encode(text);
|
|
14
|
+
|
|
15
|
+
/** A home with the artifact store of a host over it. */
|
|
16
|
+
async function storeHome() {
|
|
17
|
+
const home = await tempHome();
|
|
18
|
+
const artifacts = createNiwa({ home: home.path }).artifacts;
|
|
19
|
+
return {
|
|
20
|
+
path: home.path,
|
|
21
|
+
cleanup: home.cleanup,
|
|
22
|
+
artifacts,
|
|
23
|
+
dir: join(home.path, "artifacts"),
|
|
24
|
+
text: (ref: ArtifactRef) => new TextDecoder().decode(artifacts.read(ref)),
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async function capture(secrets: string[] = [], limit = 1 << 20, tailBytes = 16) {
|
|
29
|
+
const home = await storeHome();
|
|
30
|
+
return { home, capture: new OutputCapture({ artifacts: home.artifacts, secrets, limit, tailBytes }) };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
test("the artifact is the output, named by the digest of its bytes", async () => {
|
|
34
|
+
const { home, capture: out } = await capture();
|
|
35
|
+
try {
|
|
36
|
+
out.feed("out", enc("hello "));
|
|
37
|
+
out.feed("err", enc("world\n"));
|
|
38
|
+
const done = out.finish();
|
|
39
|
+
const sha = createHash("sha256").update("hello world\n").digest("hex");
|
|
40
|
+
expect(done.output).toEqual({ digest: `sha256:${sha}`, size: 12, type: "text/plain" });
|
|
41
|
+
expect(home.text(done.output)).toBe("hello world\n");
|
|
42
|
+
expect(readdirSync(home.dir)).toEqual([sha]);
|
|
43
|
+
expect(done.truncated).toBe(false);
|
|
44
|
+
} finally {
|
|
45
|
+
await home.cleanup();
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test("the same output is one artifact", async () => {
|
|
50
|
+
const home = await storeHome();
|
|
51
|
+
try {
|
|
52
|
+
const refs = [1, 2].map(() => {
|
|
53
|
+
const out = new OutputCapture({ artifacts: home.artifacts, secrets: [], limit: 100, tailBytes: 10 });
|
|
54
|
+
out.feed("out", enc("same"));
|
|
55
|
+
return out.finish().output;
|
|
56
|
+
});
|
|
57
|
+
expect(refs[0]).toEqual(refs[1]!);
|
|
58
|
+
expect(readdirSync(home.dir)).toHaveLength(1);
|
|
59
|
+
} finally {
|
|
60
|
+
await home.cleanup();
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("the result keeps only a bounded tail of a long output, and the artifact keeps all of it", async () => {
|
|
65
|
+
const { home, capture: out } = await capture([], 1 << 20, 8);
|
|
66
|
+
try {
|
|
67
|
+
out.feed("out", enc("0123456789"));
|
|
68
|
+
out.feed("out", enc("abcdefghij"));
|
|
69
|
+
const done = out.finish();
|
|
70
|
+
expect(done.tail).toBe("cdefghij");
|
|
71
|
+
expect(done.output.size).toBe(20);
|
|
72
|
+
expect(home.text(done.output)).toBe("0123456789abcdefghij");
|
|
73
|
+
} finally {
|
|
74
|
+
await home.cleanup();
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test("an output past the limit is cut in the artifact and says so; the tail is still its end", async () => {
|
|
79
|
+
const { home, capture: out } = await capture([], 10, 4);
|
|
80
|
+
try {
|
|
81
|
+
out.feed("out", enc("0123456789"));
|
|
82
|
+
out.feed("out", enc("ABCDEF"));
|
|
83
|
+
const done = out.finish();
|
|
84
|
+
expect(done.truncated).toBe(true);
|
|
85
|
+
expect(done.output.size).toBe(10);
|
|
86
|
+
expect(home.text(done.output)).toBe("0123456789");
|
|
87
|
+
expect(done.tail).toBe("CDEF");
|
|
88
|
+
} finally {
|
|
89
|
+
await home.cleanup();
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test("a secret is replaced in the artifact and in the tail, even when two chunks split it", async () => {
|
|
94
|
+
const { home, capture: out } = await capture(["hunter2-token"], 1 << 20, 64);
|
|
95
|
+
try {
|
|
96
|
+
out.feed("out", enc("key=hunter2"));
|
|
97
|
+
out.feed("out", enc("-token and again hunter2-token\n"));
|
|
98
|
+
const done = out.finish();
|
|
99
|
+
const stored = home.text(done.output);
|
|
100
|
+
expect(stored).toBe("key=*** and again ***\n");
|
|
101
|
+
expect(done.tail).toBe("key=*** and again ***\n");
|
|
102
|
+
expect(stored).not.toContain("hunter2");
|
|
103
|
+
} finally {
|
|
104
|
+
await home.cleanup();
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test("a secret at the very end of a stream is replaced, and a part of one that never completes is kept", () => {
|
|
109
|
+
const scrub = new Scrubber(["abcdef"]);
|
|
110
|
+
const first = scrub.push(enc("xx abc")).toString();
|
|
111
|
+
const second = scrub.push(enc("def")).toString();
|
|
112
|
+
expect(first + second + scrub.end().toString()).toBe("xx ***");
|
|
113
|
+
const lone = new Scrubber(["abcdef"]);
|
|
114
|
+
expect(lone.push(enc("abc")).toString() + lone.end().toString()).toBe("abc");
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
test("with a secret to hold back, the two streams stay in the order their chunks came", async () => {
|
|
118
|
+
const { home, capture: out } = await capture(["hunter2-token"], 1 << 20, 64);
|
|
119
|
+
try {
|
|
120
|
+
out.feed("err", enc("first\n"));
|
|
121
|
+
out.feed("out", enc("second\n"));
|
|
122
|
+
out.feed("err", enc("third\n"));
|
|
123
|
+
const done = out.finish();
|
|
124
|
+
expect(home.text(done.output)).toBe("first\nsecond\nthird\n");
|
|
125
|
+
expect(done.tail).toBe("first\nsecond\nthird\n");
|
|
126
|
+
} finally {
|
|
127
|
+
await home.cleanup();
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
test("a secret one stream split is replaced even when the other stream spoke in between", async () => {
|
|
132
|
+
const { home, capture: out } = await capture(["hunter2-token"], 1 << 20, 64);
|
|
133
|
+
try {
|
|
134
|
+
out.feed("out", enc("key=hunter2"));
|
|
135
|
+
out.feed("err", enc("E"));
|
|
136
|
+
out.feed("out", enc("-token\n"));
|
|
137
|
+
const done = out.finish();
|
|
138
|
+
const stored = home.text(done.output);
|
|
139
|
+
expect(stored).not.toContain("hunter2");
|
|
140
|
+
expect(stored).not.toContain("-token");
|
|
141
|
+
expect(stored).toBe("key=***E\n");
|
|
142
|
+
expect(done.tail).toBe(stored);
|
|
143
|
+
} finally {
|
|
144
|
+
await home.cleanup();
|
|
145
|
+
}
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test("a secret that two streams' ends make together is replaced where they meet", async () => {
|
|
149
|
+
const { home, capture: out } = await capture(["hunter2-token"], 1 << 20, 64);
|
|
150
|
+
try {
|
|
151
|
+
out.feed("out", enc("a hunter2"));
|
|
152
|
+
out.feed("err", enc("-token b"));
|
|
153
|
+
const done = out.finish();
|
|
154
|
+
const stored = home.text(done.output);
|
|
155
|
+
expect(stored).toBe("a *** b");
|
|
156
|
+
expect(done.tail).toBe("a *** b");
|
|
157
|
+
} finally {
|
|
158
|
+
await home.cleanup();
|
|
159
|
+
}
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
test("a stream that holds a part of a secret back cannot make the other pile up without end", async () => {
|
|
163
|
+
const { home, capture: out } = await capture(["hunter2-token"], 1 << 24, 16);
|
|
164
|
+
try {
|
|
165
|
+
out.feed("out", enc("hunter2"));
|
|
166
|
+
const block = Buffer.alloc(64 * 1024, "e");
|
|
167
|
+
for (let i = 0; i < 16; i += 1) out.feed("err", block);
|
|
168
|
+
expect(out.queued).toBeLessThanOrEqual(256 * 1024 + block.length);
|
|
169
|
+
out.feed("out", enc("-token"));
|
|
170
|
+
const done = out.finish();
|
|
171
|
+
const stored = home.text(done.output);
|
|
172
|
+
expect(stored.length).toBeGreaterThanOrEqual(16 * 64 * 1024);
|
|
173
|
+
expect(stored).not.toContain("hunter2-token");
|
|
174
|
+
} finally {
|
|
175
|
+
await home.cleanup();
|
|
176
|
+
}
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
test("however the chunks of two streams are cut, a secret never reaches the artifact and the rest of each stream is kept in order", async () => {
|
|
180
|
+
const secrets = ["hunter2-token", "abc"];
|
|
181
|
+
let seed = 7;
|
|
182
|
+
const random = (n: number) => {
|
|
183
|
+
seed = (seed * 1103515245 + 12345) % 2147483648;
|
|
184
|
+
return seed % n;
|
|
185
|
+
};
|
|
186
|
+
for (let round = 0; round < 40; round += 1) {
|
|
187
|
+
const pieces = Array.from({ length: 12 }, (_, i) => `[${i}]${["hunter2-token", "abc", "xyz", "hunter2", "-token", " "][random(6)]}`);
|
|
188
|
+
const { home, capture: out } = await capture(secrets, 1 << 20, 4096);
|
|
189
|
+
try {
|
|
190
|
+
// One stream carries everything, cut at random places: what is stored is the text with the secrets replaced.
|
|
191
|
+
const text = pieces.join("");
|
|
192
|
+
for (let at = 0; at < text.length; ) {
|
|
193
|
+
const next = Math.min(text.length, at + 1 + random(9));
|
|
194
|
+
out.feed("out", enc(text.slice(at, next)));
|
|
195
|
+
at = next;
|
|
196
|
+
}
|
|
197
|
+
const stored = home.text(out.finish().output);
|
|
198
|
+
expect(stored).toBe(text.replaceAll("hunter2-token", "***").replaceAll("abc", "***"));
|
|
199
|
+
} finally {
|
|
200
|
+
await home.cleanup();
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
test("a discarded capture leaves no file behind", async () => {
|
|
206
|
+
const { home, capture: out } = await capture();
|
|
207
|
+
try {
|
|
208
|
+
out.feed("out", enc("half"));
|
|
209
|
+
out.discard();
|
|
210
|
+
expect(existsSync(home.dir) ? readdirSync(home.dir) : []).toEqual([]);
|
|
211
|
+
} finally {
|
|
212
|
+
await home.cleanup();
|
|
213
|
+
}
|
|
214
|
+
});
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
// What the options of the module may be, and what a record may hold.
|
|
2
|
+
//
|
|
3
|
+
// What it proves:
|
|
4
|
+
// 1. an option that is not a usable number is refused when the module is made, not on a call
|
|
5
|
+
// 2. the tail of a result is cut to what a `proc.finished` record can hold, so a command that has
|
|
6
|
+
// run and stored its output still gives its result, however its tail is escaped in the journal
|
|
7
|
+
// 3. a working directory or a program too long for the record is refused before anything starts
|
|
8
|
+
|
|
9
|
+
import { afterEach, expect, test } from "bun:test";
|
|
10
|
+
|
|
11
|
+
import { RECORD_DATA_LIMIT } from "@niwa-labs/core";
|
|
12
|
+
|
|
13
|
+
import { MAX_TAIL_BYTES, procRun } from "../src/index.ts";
|
|
14
|
+
import { proc } from "../src/index.ts";
|
|
15
|
+
import { dirIn, outputOf, procHost, recordingSandbox } from "./support.ts";
|
|
16
|
+
|
|
17
|
+
type Host = Awaited<ReturnType<typeof procHost>>;
|
|
18
|
+
const hosts: Host[] = [];
|
|
19
|
+
afterEach(async () => {
|
|
20
|
+
for (const host of hosts.splice(0)) await host.stop();
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
test("an option that is not a usable number is refused when the module is made", () => {
|
|
24
|
+
expect(() => proc({ tailBytes: MAX_TAIL_BYTES + 1 })).toThrow(/tailBytes/);
|
|
25
|
+
expect(() => proc({ tailBytes: -1 })).toThrow(/tailBytes/);
|
|
26
|
+
expect(() => proc({ tailBytes: 1.5 })).toThrow(/tailBytes/);
|
|
27
|
+
expect(() => proc({ outputLimit: 0 })).toThrow(/outputLimit/);
|
|
28
|
+
expect(() => proc({ outputLimit: Number.NaN })).toThrow(/outputLimit/);
|
|
29
|
+
expect(() => proc({ killGraceMs: -5 })).toThrow(/killGraceMs/);
|
|
30
|
+
expect(() => proc({ tailBytes: MAX_TAIL_BYTES, outputLimit: 1, killGraceMs: 0 })).not.toThrow();
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test("a tail that escapes to more than a record can hold is cut, and the command's result is still returned", async () => {
|
|
34
|
+
const host = await procHost({ sandbox: recordingSandbox(), tailBytes: MAX_TAIL_BYTES });
|
|
35
|
+
hosts.push(host);
|
|
36
|
+
const cwd = dirIn(host.home, "work");
|
|
37
|
+
// Every byte of a control character is six bytes of JSON.
|
|
38
|
+
const result = await host.niwa.call(procRun, { cmd: ["/bin/sh", "-c", `head -c ${MAX_TAIL_BYTES} /dev/zero | tr '\\0' '\\1'`], cwd, timeout: 20_000 });
|
|
39
|
+
expect(result.exit).toBe(0);
|
|
40
|
+
expect(outputOf(host, result.output)).toHaveLength(MAX_TAIL_BYTES);
|
|
41
|
+
expect(result.tail.length).toBeGreaterThan(0);
|
|
42
|
+
expect(result.tail.length).toBeLessThan(MAX_TAIL_BYTES);
|
|
43
|
+
expect(Buffer.byteLength(JSON.stringify(result.tail))).toBeLessThan(RECORD_DATA_LIMIT);
|
|
44
|
+
const finished = (await host.niwa.records.read()).filter((row) => row.type === "proc.finished");
|
|
45
|
+
expect(finished).toHaveLength(1);
|
|
46
|
+
expect((finished[0]?.data as { tail: string }).tail).toBe(result.tail);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test("a working directory or a program too long for the record is refused before anything starts", async () => {
|
|
50
|
+
const host = await procHost({ sandbox: recordingSandbox() });
|
|
51
|
+
hosts.push(host);
|
|
52
|
+
const cwd = dirIn(host.home, "work");
|
|
53
|
+
await expect(host.niwa.call(procRun, { cmd: ["x".repeat(5000)], cwd, timeout: 1000 })).rejects.toThrow();
|
|
54
|
+
await expect(host.niwa.call(procRun, { cmd: ["/usr/bin/true"], cwd: `/${"d".repeat(5000)}`, timeout: 1000 })).rejects.toThrow();
|
|
55
|
+
});
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// A call cancelled while it runs: the command is stopped, with what it started, and the record says so.
|
|
2
|
+
//
|
|
3
|
+
// What it proves, for proc.run and proc.exec:
|
|
4
|
+
// 1. a call cancelled before the command starts, even while the sandbox is being probed, starts nothing
|
|
5
|
+
// 2. a call cancelled while the command runs stops it and its children, and `proc.finished` says
|
|
6
|
+
// the command was cancelled
|
|
7
|
+
|
|
8
|
+
import { existsSync, readFileSync, realpathSync } from "node:fs";
|
|
9
|
+
import { join } from "node:path";
|
|
10
|
+
import { afterEach, expect, test } from "bun:test";
|
|
11
|
+
|
|
12
|
+
import { eventually } from "@niwa-labs/testing";
|
|
13
|
+
|
|
14
|
+
import { absentSandbox, alive, dirIn, procActions, recordingSandbox } from "./support.ts";
|
|
15
|
+
|
|
16
|
+
type Bench = Awaited<ReturnType<typeof procActions>>;
|
|
17
|
+
const benches: Bench[] = [];
|
|
18
|
+
afterEach(async () => {
|
|
19
|
+
for (const bench of benches.splice(0)) await bench.stop();
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
async function start(options: Parameters<typeof procActions>[0] = {}): Promise<Bench> {
|
|
23
|
+
const bench = await procActions({ killGraceMs: 200, ...options });
|
|
24
|
+
benches.push(bench);
|
|
25
|
+
return bench;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const sh = (script: string) => ["/bin/sh", "-c", script];
|
|
29
|
+
|
|
30
|
+
test("a call cancelled before it starts starts nothing", async () => {
|
|
31
|
+
const sandbox = recordingSandbox();
|
|
32
|
+
const bench = await start({ sandbox });
|
|
33
|
+
const cwd = realpathSync(dirIn(bench.home, "work"));
|
|
34
|
+
const gone = new AbortController();
|
|
35
|
+
gone.abort("stopped by the owner");
|
|
36
|
+
for (const name of ["proc.run", "proc.exec"] as const) {
|
|
37
|
+
await expect(bench.call(name, { cmd: sh("touch ran"), cwd, timeout: 10_000 }, gone.signal)).rejects.toBeDefined();
|
|
38
|
+
}
|
|
39
|
+
await Bun.sleep(100);
|
|
40
|
+
expect(existsSync(join(cwd, "ran"))).toBe(false);
|
|
41
|
+
expect(sandbox.specs).toEqual([]);
|
|
42
|
+
expect(bench.emitted).toEqual([]);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test("a call cancelled while the sandbox is being probed does not start the command", async () => {
|
|
46
|
+
const gone = new AbortController();
|
|
47
|
+
const base = recordingSandbox();
|
|
48
|
+
const sandbox = {
|
|
49
|
+
...base,
|
|
50
|
+
available: async () => {
|
|
51
|
+
await Bun.sleep(50);
|
|
52
|
+
gone.abort("stopped by the owner");
|
|
53
|
+
return true;
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
const bench = await start({ sandbox });
|
|
57
|
+
const cwd = realpathSync(dirIn(bench.home, "work"));
|
|
58
|
+
await expect(bench.call("proc.run", { cmd: sh("touch ran"), cwd, timeout: 10_000 }, gone.signal)).rejects.toBeDefined();
|
|
59
|
+
await Bun.sleep(100);
|
|
60
|
+
expect(existsSync(join(cwd, "ran"))).toBe(false);
|
|
61
|
+
expect(base.specs).toEqual([]);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("a call cancelled while its command runs stops it with its children, and the record says it was cancelled", async () => {
|
|
65
|
+
for (const name of ["proc.run", "proc.exec"] as const) {
|
|
66
|
+
const bench = await start({ sandbox: name === "proc.run" ? recordingSandbox() : absentSandbox });
|
|
67
|
+
const cwd = realpathSync(dirIn(bench.home, `work-${name}`));
|
|
68
|
+
const gone = new AbortController();
|
|
69
|
+
const began = Date.now();
|
|
70
|
+
const running = bench.call(name, { cmd: sh("sleep 60 & echo $! > child.pid; wait"), cwd, timeout: 60_000 }, gone.signal);
|
|
71
|
+
await eventually(() => existsSync(join(cwd, "child.pid")), 5_000, "the command should have started");
|
|
72
|
+
gone.abort("stopped by the owner");
|
|
73
|
+
const result = await running;
|
|
74
|
+
expect(Date.now() - began).toBeLessThan(10_000);
|
|
75
|
+
expect(result.outcome).toBe("cancelled");
|
|
76
|
+
expect(result.exit).toBeNull();
|
|
77
|
+
const child = Number(readFileSync(join(cwd, "child.pid"), "utf8"));
|
|
78
|
+
await eventually(() => !alive(child), 5_000, "the child should be gone");
|
|
79
|
+
expect(bench.emitted.map((row) => [row.type, row.data["outcome"]])).toEqual([["proc.finished", "cancelled"]]);
|
|
80
|
+
}
|
|
81
|
+
});
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
// proc.exec and secrets: a command with no box, its environment, and a secret that is never written.
|
|
2
|
+
//
|
|
3
|
+
// What it proves:
|
|
4
|
+
// 1. exec runs with no box, even where none could run, and is not held to the cwd
|
|
5
|
+
// 2. its environment is PATH, HOME, what the call lists and the secrets it names: not the host's
|
|
6
|
+
// 3. a secret is resolved from a file of the home or from the host's environment, and reaches the process
|
|
7
|
+
// 4. the value of a secret is in no byte under the home (journal, its log, artifacts), nor in the
|
|
8
|
+
// result, nor in an error; the output has it replaced
|
|
9
|
+
// 5. a secret that is not there, or a file anyone may read, stops the call before a process exists
|
|
10
|
+
// 6. exec is stopped on its timeout with what it started, and says so
|
|
11
|
+
|
|
12
|
+
import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
13
|
+
import { join } from "node:path";
|
|
14
|
+
import { afterEach, expect, test } from "bun:test";
|
|
15
|
+
|
|
16
|
+
import { eventually } from "@niwa-labs/testing";
|
|
17
|
+
|
|
18
|
+
import { secret } from "@niwa-labs/core";
|
|
19
|
+
|
|
20
|
+
import { procExec } from "../src/index.ts";
|
|
21
|
+
import { absentSandbox, alive, dirIn, filesUnder, outputOf, procHost } from "./support.ts";
|
|
22
|
+
|
|
23
|
+
type Host = Awaited<ReturnType<typeof procHost>>;
|
|
24
|
+
const hosts: Host[] = [];
|
|
25
|
+
afterEach(async () => {
|
|
26
|
+
delete process.env["NIWA_SECRET_FROM_ENV"];
|
|
27
|
+
delete process.env["NIWA_ID"];
|
|
28
|
+
for (const host of hosts.splice(0)) await host.stop();
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
async function start(): Promise<Host> {
|
|
32
|
+
// No box can run: exec must not need one.
|
|
33
|
+
const host = await procHost({ sandbox: absentSandbox, killGraceMs: 200 });
|
|
34
|
+
hosts.push(host);
|
|
35
|
+
return host;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const sh = (script: string) => ["/bin/sh", "-c", script];
|
|
39
|
+
const VALUE = "s3cr3t-Value-0042";
|
|
40
|
+
|
|
41
|
+
function putSecret(host: Host, name: string, value: string, mode = 0o600): void {
|
|
42
|
+
const dir = host.niwa.secrets.dir;
|
|
43
|
+
mkdirSync(dir, { recursive: true });
|
|
44
|
+
writeFileSync(join(dir, name), `${value}\n`, { mode });
|
|
45
|
+
chmodSync(join(dir, name), mode);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
test("exec runs with no box and is not held to its cwd", async () => {
|
|
49
|
+
const host = await start();
|
|
50
|
+
const cwd = dirIn(host.home, "work");
|
|
51
|
+
const elsewhere = join(host.home, "elsewhere.txt");
|
|
52
|
+
const result = await host.niwa.call(procExec, { cmd: sh(`echo done > ${elsewhere}`), cwd, timeout: 10_000 });
|
|
53
|
+
expect(result.exit).toBe(0);
|
|
54
|
+
expect(readFileSync(elsewhere, "utf8")).toBe("done\n");
|
|
55
|
+
const finished = (await host.niwa.records.read()).filter((row) => row.type === "proc.finished");
|
|
56
|
+
expect(finished.map((row) => (row.data as { action: string; box?: unknown }).action)).toEqual(["proc.exec"]);
|
|
57
|
+
expect((finished[0]?.data as { box?: unknown }).box).toBeUndefined();
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test("the environment of exec is what the call lists, not the host's", async () => {
|
|
61
|
+
const host = await start();
|
|
62
|
+
const cwd = dirIn(host.home, "work");
|
|
63
|
+
process.env["NIWA_ID"] = "leaked-session";
|
|
64
|
+
const result = await host.niwa.call(procExec, { cmd: sh("env | sort"), cwd, timeout: 10_000, env: { MODE: "ci" } });
|
|
65
|
+
const names = outputOf(host, result.output)
|
|
66
|
+
.split("\n")
|
|
67
|
+
.filter((line) => line.includes("="))
|
|
68
|
+
.map((line) => line.split("=")[0])
|
|
69
|
+
.filter((name) => name !== "PWD" && name !== "SHLVL" && name !== "_" && name !== "OLDPWD");
|
|
70
|
+
expect(names).toEqual(["HOME", "MODE", "PATH"]);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("a secret from a file reaches the process, and the output has it replaced", async () => {
|
|
74
|
+
const host = await start();
|
|
75
|
+
const cwd = dirIn(host.home, "work");
|
|
76
|
+
putSecret(host, "DEPLOY", VALUE);
|
|
77
|
+
const result = await host.niwa.call(procExec, {
|
|
78
|
+
cmd: sh('printf "len=%s\\n" "${#TOKEN}"; printf "value=%s\\n" "$TOKEN"'),
|
|
79
|
+
cwd,
|
|
80
|
+
timeout: 10_000,
|
|
81
|
+
env: { TOKEN: secret("DEPLOY") },
|
|
82
|
+
});
|
|
83
|
+
expect(result.exit).toBe(0);
|
|
84
|
+
expect(result.tail).toBe(`len=${VALUE.length}\nvalue=***\n`);
|
|
85
|
+
expect(outputOf(host, result.output)).toBe(`len=${VALUE.length}\nvalue=***\n`);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test("the secrets a call names are given as variables of their own names", async () => {
|
|
89
|
+
const host = await start();
|
|
90
|
+
const cwd = dirIn(host.home, "work");
|
|
91
|
+
process.env["NIWA_SECRET_FROM_ENV"] = VALUE;
|
|
92
|
+
const result = await host.niwa.call(procExec, { cmd: sh('printf "%s" "${#FROM_ENV}"'), cwd, timeout: 10_000, secrets: ["FROM_ENV"] });
|
|
93
|
+
expect(result.tail).toBe(String(VALUE.length));
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
test("the value of a secret is in no byte under the home, and not in the result", async () => {
|
|
97
|
+
const host = await start();
|
|
98
|
+
const cwd = dirIn(host.home, "work");
|
|
99
|
+
process.env["NIWA_SECRET_FROM_ENV"] = VALUE;
|
|
100
|
+
const result = await host.niwa.call(procExec, {
|
|
101
|
+
cmd: sh('echo "token is $TOKEN"; echo "again $TOKEN" >&2'),
|
|
102
|
+
cwd,
|
|
103
|
+
timeout: 10_000,
|
|
104
|
+
env: { TOKEN: secret("FROM_ENV") },
|
|
105
|
+
});
|
|
106
|
+
expect(JSON.stringify(result)).not.toContain(VALUE);
|
|
107
|
+
expect(result.tail.split("\n").sort()).toEqual(["", "again ***", "token is ***"]);
|
|
108
|
+
const files = filesUnder(host.home);
|
|
109
|
+
expect(files.some((file) => file.path.includes("artifacts"))).toBe(true);
|
|
110
|
+
// The scan reaches the journal: the record of this very command is in its bytes.
|
|
111
|
+
expect(files.some((file) => file.path.includes("journal.db") && file.bytes.includes(Buffer.from("proc.finished")))).toBe(true);
|
|
112
|
+
for (const file of files) expect(file.bytes.includes(Buffer.from(VALUE))).toBe(false);
|
|
113
|
+
const finished = (await host.niwa.records.read()).filter((row) => row.type === "proc.finished");
|
|
114
|
+
expect(JSON.stringify(finished)).not.toContain(VALUE);
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
test("a secret that is not there stops the call before a process exists, and the error names no value", async () => {
|
|
118
|
+
const host = await start();
|
|
119
|
+
const cwd = dirIn(host.home, "work");
|
|
120
|
+
const marker = join(cwd, "ran");
|
|
121
|
+
const call = host.niwa.call(procExec, { cmd: sh(`touch ${marker}`), cwd, timeout: 10_000, env: { TOKEN: secret("MISSING") } });
|
|
122
|
+
await expect(call).rejects.toThrow(/no secret "MISSING"/);
|
|
123
|
+
await Bun.sleep(100);
|
|
124
|
+
expect(existsSync(marker)).toBe(false);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("a secret file that others may read is refused, naming the secret and not its value", async () => {
|
|
128
|
+
const host = await start();
|
|
129
|
+
const cwd = dirIn(host.home, "work");
|
|
130
|
+
putSecret(host, "OPEN", VALUE, 0o644);
|
|
131
|
+
const error = await host.niwa.call(procExec, { cmd: ["/usr/bin/true"], cwd, timeout: 10_000, env: { TOKEN: secret("OPEN") } }).then(
|
|
132
|
+
() => undefined,
|
|
133
|
+
(thrown: unknown) => thrown as Error,
|
|
134
|
+
);
|
|
135
|
+
expect(error?.message).toMatch(/"OPEN".*0600/);
|
|
136
|
+
expect(error?.message).not.toContain(VALUE);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test("exec that outlasts its timeout is stopped with the child it started", async () => {
|
|
140
|
+
const host = await start();
|
|
141
|
+
const cwd = dirIn(host.home, "work");
|
|
142
|
+
const result = await host.niwa.call(procExec, { cmd: sh("sleep 60 & echo $! > child.pid; wait"), cwd, timeout: 400 });
|
|
143
|
+
expect(result.outcome).toBe("timed-out");
|
|
144
|
+
expect(result.exit).toBeNull();
|
|
145
|
+
const child = Number(readFileSync(join(cwd, "child.pid"), "utf8"));
|
|
146
|
+
await eventually(() => !alive(child), 5_000, "the child should be gone");
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test("a program that is not there is refused, and nothing is recorded", async () => {
|
|
150
|
+
const host = await start();
|
|
151
|
+
const cwd = dirIn(host.home, "work");
|
|
152
|
+
await expect(host.niwa.call(procExec, { cmd: ["/no/such/program"], cwd, timeout: 1000 })).rejects.toThrow(/cannot start/);
|
|
153
|
+
expect((await host.niwa.records.read()).filter((row) => row.type === "proc.finished")).toEqual([]);
|
|
154
|
+
});
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
// runCheck in a real script: the check runs with proc.run, passes on a zero exit, and reads the
|
|
2
|
+
// report the command printed at the end of its output. A script names the rules of the box in its
|
|
3
|
+
// `uses` like any action, and a run's box is widened only within them.
|
|
4
|
+
|
|
5
|
+
import { afterEach, expect, test } from "bun:test";
|
|
6
|
+
import { realpathSync } from "node:fs";
|
|
7
|
+
import { mkdir, symlink, writeFile } from "node:fs/promises";
|
|
8
|
+
import { join } from "node:path";
|
|
9
|
+
|
|
10
|
+
import { createNiwa, requests, tempHome, timers, workflow } from "@niwa-labs/testing";
|
|
11
|
+
import type { Host } from "@niwa-labs/testing";
|
|
12
|
+
|
|
13
|
+
import { boxRules, proc } from "../src/index.ts";
|
|
14
|
+
import { reportOf } from "../src/flow.ts";
|
|
15
|
+
import { procOptions, recordingSandbox } from "./support.ts";
|
|
16
|
+
|
|
17
|
+
const cleanups: Array<() => Promise<void>> = [];
|
|
18
|
+
afterEach(async () => {
|
|
19
|
+
for (const cleanup of cleanups.splice(0)) await cleanup();
|
|
20
|
+
}, 30_000);
|
|
21
|
+
|
|
22
|
+
/** Run a script that imports `@niwa-labs/proc/flow`, on a host with the proc module, and answer its result. */
|
|
23
|
+
async function runScript(source: string, rules = false): Promise<{ result: unknown; sandbox: ReturnType<typeof recordingSandbox>; work: string }> {
|
|
24
|
+
const temp = await tempHome();
|
|
25
|
+
const home = realpathSync(temp.path);
|
|
26
|
+
const sandbox = recordingSandbox();
|
|
27
|
+
const host: Host = createNiwa({ home });
|
|
28
|
+
host.use(requests(), timers([]), proc(procOptions({ sandbox })), ...(rules ? [boxRules()] : []), workflow());
|
|
29
|
+
await host.serve();
|
|
30
|
+
cleanups.push(async () => {
|
|
31
|
+
await host.close().catch(() => {});
|
|
32
|
+
await temp.cleanup();
|
|
33
|
+
});
|
|
34
|
+
const src = join(home, "src");
|
|
35
|
+
const work = join(home, "work");
|
|
36
|
+
await mkdir(join(src, "node_modules", "@niwa-labs"), { recursive: true });
|
|
37
|
+
await mkdir(work, { recursive: true });
|
|
38
|
+
await symlink(realpathSync(join(import.meta.dir, "..")), join(src, "node_modules", "@niwa-labs", "proc"), "dir");
|
|
39
|
+
await writeFile(join(src, "check.ts"), source.replaceAll("$WORK", work));
|
|
40
|
+
const ended = (await host.call("workflow.start" as never, { file: join(src, "check.ts"), wait: true, timeoutMs: 30_000 } as never)) as {
|
|
41
|
+
state: string;
|
|
42
|
+
outcome?: { status: string; result?: unknown };
|
|
43
|
+
};
|
|
44
|
+
if (ended.outcome?.status !== "done") throw new Error(`the run did not finish: ${JSON.stringify(ended).slice(0, 1500)}`);
|
|
45
|
+
return { result: ended.outcome.result, sandbox, work };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
test("a check passes on a zero exit, fails on another, and gives the report it printed last", async () => {
|
|
49
|
+
const { result, sandbox, work } = await runScript(`
|
|
50
|
+
import { runCheck } from "@niwa-labs/proc/flow";
|
|
51
|
+
export default workflow({ output: z.json() }, async (): Promise<any> => {
|
|
52
|
+
const good = await runCheck("good", { cmd: ["sh", "-c", "echo testing; echo '{\\"coverage\\": 0.8}'"], cwd: "$WORK", timeout: "1m", report: "json" });
|
|
53
|
+
const bad = await runCheck("bad", { cmd: ["sh", "-c", "echo it broke; exit 3"], cwd: "$WORK" });
|
|
54
|
+
return { good: { ok: good.ok, exit: good.exit, report: good.report, artifact: typeof good.artifact.digest }, bad: { ok: bad.ok, exit: bad.exit, summary: bad.summary } };
|
|
55
|
+
});`);
|
|
56
|
+
expect(result).toEqual({
|
|
57
|
+
good: { ok: true, exit: 0, report: { coverage: 0.8 }, artifact: "string" },
|
|
58
|
+
bad: { ok: false, exit: 3, summary: "exited with 3\nit broke" },
|
|
59
|
+
});
|
|
60
|
+
expect(sandbox.specs.map((spec) => spec.cwd)).toEqual([work, work]);
|
|
61
|
+
}, 30_000);
|
|
62
|
+
|
|
63
|
+
test("a report is read from the end of the output: JSON on the last line, lcov totals, JUnit suite counts", () => {
|
|
64
|
+
expect(reportOf("json", "noise\n{\"total\": 3}\n")).toEqual({ total: 3 });
|
|
65
|
+
expect(reportOf("json", "no report here")).toBeUndefined();
|
|
66
|
+
expect(reportOf("lcov", "SF:a.ts\nLF:10\nLH:5\nend_of_record\nSF:b.ts\nLF:10\nLH:10\nend_of_record\n")).toEqual({ found: 20, hit: 15, coverage: 0.75 });
|
|
67
|
+
expect(reportOf("junit", '<testsuite name="a" tests="4" failures="1" errors="0"></testsuite><testsuite tests="2" failures="0" errors="1" skipped="1">')).toEqual({
|
|
68
|
+
tests: 6,
|
|
69
|
+
failures: 1,
|
|
70
|
+
errors: 1,
|
|
71
|
+
skipped: 1,
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test("a script names the rules of the box in its uses, and its runs are widened only within them", async () => {
|
|
76
|
+
const script = (uses: string) => `
|
|
77
|
+
export default workflow({ uses: [${uses}], output: z.json() }, async (): Promise<any> => {
|
|
78
|
+
try {
|
|
79
|
+
const run = await call("proc.run", { cmd: ["true"], cwd: "$WORK", timeout: 10000, box: { allow: ["bun"] } }) as any;
|
|
80
|
+
return { rules: run.box.rules };
|
|
81
|
+
} catch (error) {
|
|
82
|
+
return { refused: String(error) };
|
|
83
|
+
}
|
|
84
|
+
});`;
|
|
85
|
+
const granted = await runScript(script(`"proc.run@$WORK", "box.use@bun"`), true);
|
|
86
|
+
expect(granted.result).toEqual({ rules: ["box.use@bun"] });
|
|
87
|
+
const narrow = await runScript(script(`"proc.run@$WORK"`), true);
|
|
88
|
+
expect(String((narrow.result as { refused?: string }).refused)).toContain("box.use@bun");
|
|
89
|
+
}, 60_000);
|
|
90
|
+
|
|
91
|
+
test("a check asks its box for what it needs, and a check that holds a secret takes no box", async () => {
|
|
92
|
+
const { result, sandbox } = await runScript(`
|
|
93
|
+
import { runCheck } from "@niwa-labs/proc/flow";
|
|
94
|
+
export default workflow({ output: z.json() }, async (): Promise<any> => {
|
|
95
|
+
const built = await runCheck("build", { cmd: ["true"], cwd: "$WORK", box: { allow: ["bun"] } });
|
|
96
|
+
let refused = "";
|
|
97
|
+
try {
|
|
98
|
+
await runCheck("secret", { cmd: ["true"], cwd: "$WORK", env: { TOKEN: { secret: "t" } }, box: { only: [] } });
|
|
99
|
+
} catch (error) {
|
|
100
|
+
refused = String(error);
|
|
101
|
+
}
|
|
102
|
+
return { ok: built.ok, refused };
|
|
103
|
+
});`);
|
|
104
|
+
expect((result as { ok: boolean }).ok).toBe(true);
|
|
105
|
+
expect((result as { refused: string }).refused).toContain("has no box");
|
|
106
|
+
expect(sandbox.specs[0]?.write.map(String).some((path) => path.endsWith("/install/cache"))).toBe(true);
|
|
107
|
+
}, 30_000);
|