@celilo/cli 5.2.1 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/api/serve-inprocess.test.ts +103 -0
- package/src/api/serve.ts +132 -1
- package/src/module/packaging/artifact-bytes.test.ts +91 -0
- package/src/module/packaging/artifact-bytes.ts +135 -0
- package/src/module/packaging/build.test.ts +55 -2
- package/src/module/packaging/build.ts +73 -4
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@celilo/cli",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.3.0",
|
|
4
4
|
"description": "Celilo — home lab orchestration CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -61,7 +61,7 @@
|
|
|
61
61
|
"@aws-sdk/lib-storage": "^3.1101.0",
|
|
62
62
|
"@celilo/capabilities": "^6.1.0",
|
|
63
63
|
"@celilo/cli-display": "^0.2.0",
|
|
64
|
-
"@celilo/core": "^0.
|
|
64
|
+
"@celilo/core": "^0.16.0",
|
|
65
65
|
"@celilo/event-bus": "^0.7.0",
|
|
66
66
|
"ajv": "^8.18.0",
|
|
67
67
|
"drizzle-orm": "^0.36.4",
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A read runs in THIS process; everything else still gets a child.
|
|
3
|
+
*
|
|
4
|
+
* Every command used to be a fresh `celilo` child, which costs a full CLI boot —
|
|
5
|
+
* ~1.1s on celilo-mgr, whose root is an SD card — paid per command even on a
|
|
6
|
+
* reused connection. With connection reuse landed, every console endpoint still
|
|
7
|
+
* measured ~1.0-1.5s regardless of what it did, because the boot dominated the
|
|
8
|
+
* work. A read cannot park, so it has nothing to outlive the transport and needs
|
|
9
|
+
* no child (celilo#1457).
|
|
10
|
+
*
|
|
11
|
+
* These pin the two things that make in-process execution safe. Both are
|
|
12
|
+
* invisible in normal operation and catastrophic when wrong, which is exactly
|
|
13
|
+
* the shape that needs a test rather than a comment.
|
|
14
|
+
*/
|
|
15
|
+
import { describe, expect, test } from 'bun:test';
|
|
16
|
+
import { isReadOnlyPath } from '@celilo/core';
|
|
17
|
+
|
|
18
|
+
describe('which commands may run in-process', () => {
|
|
19
|
+
test('the console’s reads are classified read-only', () => {
|
|
20
|
+
// If any of these stopped being a read, it would start paying a CLI boot
|
|
21
|
+
// again and the console would silently regress to ~1s per call.
|
|
22
|
+
for (const argv of [
|
|
23
|
+
['console', 'status', '--json'],
|
|
24
|
+
['alerts', 'list', '--json'],
|
|
25
|
+
['backup', 'list', '--json'],
|
|
26
|
+
['person', 'list', '--json'],
|
|
27
|
+
['module', 'list', '--json'],
|
|
28
|
+
]) {
|
|
29
|
+
expect(isReadOnlyPath(argv)).toBe(true);
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test('a mutating command is NOT, so it still gets a child', () => {
|
|
34
|
+
// The child is what lets a parked command outlive the ssh transport. A
|
|
35
|
+
// write running in-process would lose that, and a park would die with the
|
|
36
|
+
// connection.
|
|
37
|
+
for (const argv of [
|
|
38
|
+
['module', 'deploy', 'caddy'],
|
|
39
|
+
['alerts', 'ack', 'some-id'],
|
|
40
|
+
['module', 'update'],
|
|
41
|
+
['system', 'init'],
|
|
42
|
+
]) {
|
|
43
|
+
expect(isReadOnlyPath(argv)).toBe(false);
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test('an unknown verb is not treated as a read', () => {
|
|
48
|
+
// Fail closed: an unrecognised path must take the child, not the fast path.
|
|
49
|
+
expect(isReadOnlyPath(['not-a-command'])).toBe(false);
|
|
50
|
+
expect(isReadOnlyPath([])).toBe(false);
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
describe('the guards in-process execution depends on', () => {
|
|
55
|
+
test('process.exit can be intercepted and restored', () => {
|
|
56
|
+
// The CLI calls process.exit on some paths and inside runCli
|
|
57
|
+
// (cli/index.ts:1582). For a CHILD that is an ordinary exit; in-process it
|
|
58
|
+
// would take the whole api-serve down mid-session, dropping the client's
|
|
59
|
+
// connection and any parked session it was holding.
|
|
60
|
+
const real = process.exit;
|
|
61
|
+
class ExitCalled extends Error {
|
|
62
|
+
constructor(readonly code: number) {
|
|
63
|
+
super(`exit ${code}`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
let caught: number | null = null;
|
|
67
|
+
try {
|
|
68
|
+
(process as { exit: unknown }).exit = (code?: number): never => {
|
|
69
|
+
throw new ExitCalled(code ?? 0);
|
|
70
|
+
};
|
|
71
|
+
try {
|
|
72
|
+
process.exit(3);
|
|
73
|
+
} catch (e) {
|
|
74
|
+
if (e instanceof ExitCalled) caught = e.code;
|
|
75
|
+
}
|
|
76
|
+
} finally {
|
|
77
|
+
(process as { exit: unknown }).exit = real;
|
|
78
|
+
}
|
|
79
|
+
expect(caught).toBe(3);
|
|
80
|
+
expect(process.exit).toBe(real);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test('a stdout reference bound BEFORE a redirect survives it', () => {
|
|
84
|
+
// `send` writes the protocol through a reference bound at module load. If
|
|
85
|
+
// it used process.stdout.write directly, the capture would swallow the
|
|
86
|
+
// framing and the client would receive api-serve's own protocol as if it
|
|
87
|
+
// were command output — unparseable, and three layers from the cause.
|
|
88
|
+
const real = process.stdout.write.bind(process.stdout);
|
|
89
|
+
const bound = real;
|
|
90
|
+
const captured: string[] = [];
|
|
91
|
+
try {
|
|
92
|
+
(process.stdout as { write: unknown }).write = (c: unknown): boolean => {
|
|
93
|
+
captured.push(String(c));
|
|
94
|
+
return true;
|
|
95
|
+
};
|
|
96
|
+
process.stdout.write('this is command output\n');
|
|
97
|
+
bound(''); // the protocol path: must NOT land in `captured`
|
|
98
|
+
} finally {
|
|
99
|
+
(process.stdout as { write: unknown }).write = real;
|
|
100
|
+
}
|
|
101
|
+
expect(captured).toEqual(['this is command output\n']);
|
|
102
|
+
});
|
|
103
|
+
});
|
package/src/api/serve.ts
CHANGED
|
@@ -25,8 +25,10 @@ import {
|
|
|
25
25
|
ClientMessageSchema,
|
|
26
26
|
type ServerMessage,
|
|
27
27
|
ServerMessageSchema,
|
|
28
|
+
isReadOnlyPath,
|
|
28
29
|
translateOutputLine,
|
|
29
30
|
} from '@celilo/core';
|
|
31
|
+
import { runCli } from '../cli/index';
|
|
30
32
|
import { parseArguments } from '../cli/parser';
|
|
31
33
|
import { getDataDir, getEventBusPath } from '../config/paths';
|
|
32
34
|
import { getDb } from '../db/client';
|
|
@@ -51,8 +53,17 @@ const EXIT_PERMISSION_DENIED = 126;
|
|
|
51
53
|
/** How often an attached client polls a session's buffer for new output. */
|
|
52
54
|
const ATTACH_POLL_MS = 250;
|
|
53
55
|
|
|
56
|
+
/**
|
|
57
|
+
* The real stdout, captured at module load.
|
|
58
|
+
*
|
|
59
|
+
* `runCommandInProcess` redirects `process.stdout.write` to capture a command's
|
|
60
|
+
* output. Without this reference `send` would write the PROTOCOL through that
|
|
61
|
+
* redirect and the client would receive its own framing as command output.
|
|
62
|
+
*/
|
|
63
|
+
const writeProtocol = process.stdout.write.bind(process.stdout);
|
|
64
|
+
|
|
54
65
|
function send(msg: ServerMessage): void {
|
|
55
|
-
|
|
66
|
+
writeProtocol(`${JSON.stringify(msg)}\n`);
|
|
56
67
|
}
|
|
57
68
|
|
|
58
69
|
/**
|
|
@@ -132,8 +143,128 @@ async function pumpLines(
|
|
|
132
143
|
}
|
|
133
144
|
}
|
|
134
145
|
|
|
146
|
+
/**
|
|
147
|
+
* Run a READ-ONLY command in this process instead of spawning a child.
|
|
148
|
+
*
|
|
149
|
+
* Every command used to be a fresh `celilo` child, which costs a full CLI boot —
|
|
150
|
+
* ~1.1s on celilo-mgr, whose root is an SD card. That is paid per command even
|
|
151
|
+
* on a reused connection, and it was the floor under the web console: with
|
|
152
|
+
* connection reuse landed, every endpoint still measured ~1.0-1.5s regardless of
|
|
153
|
+
* what it actually did, because the boot dominated the work (celilo#1457).
|
|
154
|
+
*
|
|
155
|
+
* WHY ONLY READS. The child exists so a command that PARKS can outlive the ssh
|
|
156
|
+
* transport: the session keeps it alive and buffers its output. A read cannot
|
|
157
|
+
* park — it raises no interview — so it has nothing to outlive and needs no
|
|
158
|
+
* child. `isReadOnlyPath` is the same predicate the API's authz uses, so this
|
|
159
|
+
* cannot widen what runs in-process without also widening what counts as a read.
|
|
160
|
+
*
|
|
161
|
+
* TWO THINGS MAKE IT SAFE, and neither is optional:
|
|
162
|
+
*
|
|
163
|
+
* - Output is captured by redirecting `process.stdout/stderr.write`, so `send`
|
|
164
|
+
* must not use them. It writes through `writeProtocol`, bound before any
|
|
165
|
+
* redirect exists. Commands are serialized (one `commandRunning` slot), so
|
|
166
|
+
* the redirect can never straddle two commands.
|
|
167
|
+
* - `process.exit` is intercepted. The CLI calls it on some paths and inside
|
|
168
|
+
* `runCli` (cli/index.ts:1582), which for a CHILD is an ordinary exit and
|
|
169
|
+
* in-process would take the whole server down mid-session. It is replaced
|
|
170
|
+
* with a throw that carries the code, and restored in `finally`.
|
|
171
|
+
*/
|
|
172
|
+
async function runCommandInProcess(
|
|
173
|
+
argv: string[],
|
|
174
|
+
forward: (msg: ServerMessage) => void,
|
|
175
|
+
): Promise<number> {
|
|
176
|
+
const realStdout = process.stdout.write.bind(process.stdout);
|
|
177
|
+
const realStderr = process.stderr.write.bind(process.stderr);
|
|
178
|
+
const realExit = process.exit.bind(process);
|
|
179
|
+
|
|
180
|
+
const buffers: Record<'stdout' | 'stderr', string> = { stdout: '', stderr: '' };
|
|
181
|
+
const capture =
|
|
182
|
+
(stream: 'stdout' | 'stderr') =>
|
|
183
|
+
(chunk: unknown): boolean => {
|
|
184
|
+
const text =
|
|
185
|
+
typeof chunk === 'string' ? chunk : new TextDecoder().decode(chunk as Uint8Array);
|
|
186
|
+
buffers[stream] += text;
|
|
187
|
+
let nl = buffers[stream].indexOf('\n');
|
|
188
|
+
while (nl >= 0) {
|
|
189
|
+
const line = buffers[stream].slice(0, nl);
|
|
190
|
+
buffers[stream] = buffers[stream].slice(nl + 1);
|
|
191
|
+
const msg = translateOutputLine(line);
|
|
192
|
+
forward(msg.type === 'log' ? { ...msg, stream } : msg);
|
|
193
|
+
nl = buffers[stream].indexOf('\n');
|
|
194
|
+
}
|
|
195
|
+
return true;
|
|
196
|
+
};
|
|
197
|
+
|
|
198
|
+
const flush = (): void => {
|
|
199
|
+
for (const stream of ['stdout', 'stderr'] as const) {
|
|
200
|
+
if (buffers[stream].length > 0) {
|
|
201
|
+
const msg = translateOutputLine(buffers[stream]);
|
|
202
|
+
forward(msg.type === 'log' ? { ...msg, stream } : msg);
|
|
203
|
+
buffers[stream] = '';
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
class ExitCalled extends Error {
|
|
209
|
+
constructor(readonly code: number) {
|
|
210
|
+
super(`process.exit(${code})`);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
let exitCode: number;
|
|
215
|
+
try {
|
|
216
|
+
(process.stdout as { write: unknown }).write = capture('stdout');
|
|
217
|
+
(process.stderr as { write: unknown }).write = capture('stderr');
|
|
218
|
+
(process as { exit: unknown }).exit = (code?: number): never => {
|
|
219
|
+
throw new ExitCalled(code ?? 0);
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
// `parseArguments` does `argv.slice(2)`, so runCli takes process.argv SHAPE —
|
|
223
|
+
// executable, script, then the command. The child path gets this right by
|
|
224
|
+
// construction (`Bun.spawn([process.execPath, Bun.main, ...argv])`); passing
|
|
225
|
+
// bare argv here silently dropped the first two words, so `service list
|
|
226
|
+
// --json` parsed as `--json` and the command failed. Mirror the child.
|
|
227
|
+
const result = await runCli([process.execPath, Bun.main, ...argv]);
|
|
228
|
+
// `runCli` RETURNS the terminal message rather than printing it, so it never
|
|
229
|
+
// passed through the capture above. A child printed it on the way out; in
|
|
230
|
+
// process it has to be forwarded explicitly or the client sees an empty
|
|
231
|
+
// payload for a command that succeeded.
|
|
232
|
+
if (result.success) {
|
|
233
|
+
exitCode = 0;
|
|
234
|
+
if (result.message) {
|
|
235
|
+
forward({ type: 'log', message: result.message, stream: 'stdout' });
|
|
236
|
+
}
|
|
237
|
+
} else {
|
|
238
|
+
exitCode = 1;
|
|
239
|
+
forward({ type: 'log', message: result.error, stream: 'stderr' });
|
|
240
|
+
}
|
|
241
|
+
} catch (error) {
|
|
242
|
+
if (error instanceof ExitCalled) {
|
|
243
|
+
exitCode = error.code;
|
|
244
|
+
} else {
|
|
245
|
+
exitCode = 1;
|
|
246
|
+
forward({
|
|
247
|
+
type: 'log',
|
|
248
|
+
message: error instanceof Error ? error.message : String(error),
|
|
249
|
+
stream: 'stderr',
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
} finally {
|
|
253
|
+
(process.stdout as { write: unknown }).write = realStdout;
|
|
254
|
+
(process.stderr as { write: unknown }).write = realStderr;
|
|
255
|
+
(process as { exit: unknown }).exit = realExit;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
flush();
|
|
259
|
+
forward(resultMessage(exitCode === 0, exitCode));
|
|
260
|
+
return exitCode;
|
|
261
|
+
}
|
|
262
|
+
|
|
135
263
|
/** Run an authorized command as a child; returns its exit code. */
|
|
136
264
|
async function runCommand(argv: string[], forward: (msg: ServerMessage) => void): Promise<number> {
|
|
265
|
+
// A read cannot park, so it needs no child and should not pay a CLI boot.
|
|
266
|
+
if (isReadOnlyPath(argv)) return runCommandInProcess(argv, forward);
|
|
267
|
+
|
|
137
268
|
// Re-invoke this same CLI as a child. The child is non-TTY (piped stdout) so
|
|
138
269
|
// ProgressDisplay resolves to protocol mode and emits `[progress:*]` markers.
|
|
139
270
|
const child = Bun.spawn([process.execPath, Bun.main, ...argv], {
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { chmodSync, mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { inspectArtifactFile, verifyBuildArtifacts } from './artifact-bytes';
|
|
6
|
+
|
|
7
|
+
const ELF = Buffer.from([0x7f, 0x45, 0x4c, 0x46]);
|
|
8
|
+
|
|
9
|
+
function dir(): string {
|
|
10
|
+
return mkdtempSync(join(tmpdir(), 'artifact-bytes-'));
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function write(base: string, rel: string, body: Buffer, exec = false): string {
|
|
14
|
+
const full = join(base, rel);
|
|
15
|
+
mkdirSync(join(full, '..'), { recursive: true });
|
|
16
|
+
writeFileSync(full, body);
|
|
17
|
+
if (exec) chmodSync(full, 0o755);
|
|
18
|
+
return full;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
describe('inspectArtifactFile', () => {
|
|
22
|
+
test('rejects a right-sized file of pure NUL — the celilo#1463 shape', () => {
|
|
23
|
+
const d = dir();
|
|
24
|
+
const p = write(d, 'server-linux-x86_64', Buffer.alloc(4096), true);
|
|
25
|
+
expect(inspectArtifactFile(p)).toMatch(/4096 bytes of pure NUL/);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test('rejects an empty file', () => {
|
|
29
|
+
const d = dir();
|
|
30
|
+
expect(inspectArtifactFile(write(d, 'bin', Buffer.alloc(0), true))).toMatch(/empty/);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test('accepts a real ELF', () => {
|
|
34
|
+
const d = dir();
|
|
35
|
+
const p = write(d, 'bin', Buffer.concat([ELF, Buffer.alloc(1024, 0x41)]), true);
|
|
36
|
+
expect(inspectArtifactFile(p)).toBeNull();
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test('rejects an executable carrying no executable magic', () => {
|
|
40
|
+
const d = dir();
|
|
41
|
+
const p = write(d, 'bin', Buffer.from('error: build failed\n'), true);
|
|
42
|
+
expect(inspectArtifactFile(p)).toMatch(/no known executable magic/);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test('does not apply the magic check to non-executable artifacts', () => {
|
|
46
|
+
// Modules declare .xz tarballs, .sha256 sidecars and index.html as
|
|
47
|
+
// artifacts. None is a program and none may be rejected as one.
|
|
48
|
+
const d = dir();
|
|
49
|
+
expect(inspectArtifactFile(write(d, 'forgejo.xz.sha256', Buffer.from('abc f\n')))).toBeNull();
|
|
50
|
+
expect(inspectArtifactFile(write(d, 'index.html', Buffer.from('<!doctype html>')))).toBeNull();
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test('a NUL file is rejected even when it is not executable', () => {
|
|
54
|
+
const d = dir();
|
|
55
|
+
expect(inspectArtifactFile(write(d, 'forgejo.xz', Buffer.alloc(64)))).toMatch(/pure NUL/);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test('a file whose NUL run ends late is still accepted', () => {
|
|
59
|
+
// Guards the streaming scan: content past the first chunk must be seen.
|
|
60
|
+
const d = dir();
|
|
61
|
+
const body = Buffer.alloc((1 << 20) + 16);
|
|
62
|
+
body[body.length - 1] = 0x41;
|
|
63
|
+
expect(inspectArtifactFile(write(d, 'blob', body))).toBeNull();
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
describe('verifyBuildArtifacts', () => {
|
|
68
|
+
test('names the module, the artifact and its size', () => {
|
|
69
|
+
const d = dir();
|
|
70
|
+
write(d, 'ansible/roles/r/files/srv-linux-x86_64', Buffer.alloc(106129743 % 8192), true);
|
|
71
|
+
const reason = verifyBuildArtifacts(d, ['ansible/roles/r/files/srv-linux-x86_64'], 'dj-live');
|
|
72
|
+
expect(reason).toContain('dj-live');
|
|
73
|
+
expect(reason).toContain('ansible/roles/r/files/srv-linux-x86_64');
|
|
74
|
+
expect(reason).toMatch(/bytes of pure NUL/);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test('passes a sound set', () => {
|
|
78
|
+
const d = dir();
|
|
79
|
+
write(d, 'files/a', Buffer.concat([ELF, Buffer.alloc(16, 1)]), true);
|
|
80
|
+
write(d, 'files/b.sha256', Buffer.from('deadbeef\n'));
|
|
81
|
+
expect(verifyBuildArtifacts(d, ['files/a', 'files/b.sha256'], 'm')).toBeNull();
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
test('walks a declared directory artifact', () => {
|
|
85
|
+
const d = dir();
|
|
86
|
+
write(d, 'spa/index.html', Buffer.from('<!doctype html>'));
|
|
87
|
+
write(d, 'spa/assets/app.js', Buffer.alloc(32));
|
|
88
|
+
const reason = verifyBuildArtifacts(d, ['spa'], 'console');
|
|
89
|
+
expect(reason).toContain('assets/app.js');
|
|
90
|
+
});
|
|
91
|
+
});
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { closeSync, openSync, readSync, readdirSync, statSync } from 'node:fs';
|
|
2
|
+
import { join, relative } from 'node:path';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Executable magics, for a file the build marked executable.
|
|
6
|
+
*
|
|
7
|
+
* Only consulted when the artifact carries an exec bit, because
|
|
8
|
+
* `build.artifacts` is not a list of programs — modules declare `.xz`
|
|
9
|
+
* tarballs, a `.sha256` sidecar, an `index.html` and a whole `spa/`
|
|
10
|
+
* directory. A magic check applied to all of them would fail every module in
|
|
11
|
+
* the fleet; applied to the files the build itself chmod'd +x it is exactly
|
|
12
|
+
* the "is this runnable" question.
|
|
13
|
+
*/
|
|
14
|
+
const EXEC_MAGICS: readonly (readonly number[])[] = [
|
|
15
|
+
[0x7f, 0x45, 0x4c, 0x46], // ELF
|
|
16
|
+
[0xfe, 0xed, 0xfa, 0xce], // Mach-O 32
|
|
17
|
+
[0xfe, 0xed, 0xfa, 0xcf], // Mach-O 64
|
|
18
|
+
[0xce, 0xfa, 0xed, 0xfe], // Mach-O 32, byte-swapped
|
|
19
|
+
[0xcf, 0xfa, 0xed, 0xfe], // Mach-O 64, byte-swapped
|
|
20
|
+
[0xca, 0xfe, 0xba, 0xbe], // Mach-O universal
|
|
21
|
+
[0x4d, 0x5a], // PE/COFF "MZ"
|
|
22
|
+
[0x23, 0x21], // "#!" script
|
|
23
|
+
];
|
|
24
|
+
|
|
25
|
+
const CHUNK = 1 << 20;
|
|
26
|
+
|
|
27
|
+
/** Read up to `n` leading bytes without loading the file. */
|
|
28
|
+
function head(path: string, n: number): Buffer {
|
|
29
|
+
const fd = openSync(path, 'r');
|
|
30
|
+
try {
|
|
31
|
+
const buf = Buffer.alloc(n);
|
|
32
|
+
const read = readSync(fd, buf, 0, n, 0);
|
|
33
|
+
return buf.subarray(0, read);
|
|
34
|
+
} finally {
|
|
35
|
+
closeSync(fd);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* True when every byte of the file is NUL.
|
|
41
|
+
*
|
|
42
|
+
* Streams, and stops at the first non-zero byte — so a healthy artifact costs
|
|
43
|
+
* one 1 MiB read (its first byte is non-zero) and only a genuinely corrupt one
|
|
44
|
+
* is read through.
|
|
45
|
+
*/
|
|
46
|
+
function isAllNul(path: string, size: number): boolean {
|
|
47
|
+
const fd = openSync(path, 'r');
|
|
48
|
+
try {
|
|
49
|
+
const buf = Buffer.alloc(CHUNK);
|
|
50
|
+
for (let off = 0; off < size; ) {
|
|
51
|
+
const read = readSync(fd, buf, 0, CHUNK, off);
|
|
52
|
+
if (read === 0) break;
|
|
53
|
+
for (let i = 0; i < read; i++) {
|
|
54
|
+
if (buf[i] !== 0) return false;
|
|
55
|
+
}
|
|
56
|
+
off += read;
|
|
57
|
+
}
|
|
58
|
+
return true;
|
|
59
|
+
} finally {
|
|
60
|
+
closeSync(fd);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Inspect one artifact file's own bytes. Returns a reason, or null when sound.
|
|
66
|
+
*
|
|
67
|
+
* Takes the verdict from the bytes rather than from the exit code of whatever
|
|
68
|
+
* wrote them. celilo#1463: two 100 MiB module binaries arrived as right-sized
|
|
69
|
+
* files of pure NUL, and every stage downstream reported success — because
|
|
70
|
+
* every one of those stages hashes, signs and verifies the artifact it is
|
|
71
|
+
* given. A checksum computed over corruption certifies corruption. The first
|
|
72
|
+
* thing that noticed was systemd, on a production host, with `Exec format
|
|
73
|
+
* error` and a restart counter.
|
|
74
|
+
*/
|
|
75
|
+
export function inspectArtifactFile(path: string): string | null {
|
|
76
|
+
const size = statSync(path).size;
|
|
77
|
+
if (size === 0) return 'file is empty (0 bytes)';
|
|
78
|
+
if (isAllNul(path, size)) return `file is ${size} bytes of pure NUL — nothing was written`;
|
|
79
|
+
|
|
80
|
+
// Only a file the build marked runnable has to look runnable.
|
|
81
|
+
const mode = statSync(path).mode;
|
|
82
|
+
if (!(mode & 0o111)) return null;
|
|
83
|
+
|
|
84
|
+
const magic = head(path, 4);
|
|
85
|
+
const known = EXEC_MAGICS.some((m) => m.every((b, i) => magic[i] === b));
|
|
86
|
+
if (!known) {
|
|
87
|
+
return `file is executable but carries no known executable magic (starts ${magic.toString('hex')}) — not a runnable program`;
|
|
88
|
+
}
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Gate every declared build artifact on its own bytes.
|
|
94
|
+
*
|
|
95
|
+
* Returns a human-readable failure naming module, artifact and size, or null
|
|
96
|
+
* when all artifacts are sound. A declared directory is walked, because a
|
|
97
|
+
* module can declare a built tree (`spa/`) and a NUL file inside one is as
|
|
98
|
+
* invisible as a NUL file beside it.
|
|
99
|
+
*/
|
|
100
|
+
export function verifyBuildArtifacts(
|
|
101
|
+
buildDir: string,
|
|
102
|
+
artifacts: readonly string[],
|
|
103
|
+
moduleId: string,
|
|
104
|
+
): string | null {
|
|
105
|
+
const failures: string[] = [];
|
|
106
|
+
|
|
107
|
+
const check = (fullPath: string, label: string): void => {
|
|
108
|
+
const reason = inspectArtifactFile(fullPath);
|
|
109
|
+
if (reason) failures.push(` ${label}: ${reason}`);
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
// `root` stays the artifact's own directory so a nested file reports its
|
|
113
|
+
// whole path under the artifact, not just its basename.
|
|
114
|
+
const walk = (dir: string, root: string, artifact: string): void => {
|
|
115
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
116
|
+
const full = join(dir, entry.name);
|
|
117
|
+
if (entry.isDirectory()) walk(full, root, artifact);
|
|
118
|
+
else if (entry.isFile()) check(full, join(artifact, relative(root, full)));
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
for (const artifact of artifacts) {
|
|
123
|
+
const full = join(buildDir, artifact);
|
|
124
|
+
if (statSync(full).isDirectory()) walk(full, full, artifact);
|
|
125
|
+
else check(full, artifact);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if (failures.length === 0) return null;
|
|
129
|
+
return [
|
|
130
|
+
`Refusing to package ${moduleId}: the build reported success but produced an unusable artifact.`,
|
|
131
|
+
...failures,
|
|
132
|
+
'',
|
|
133
|
+
"The verdict is taken from the artifact's own bytes, not from the build command's exit code. Every check downstream of here — checksums, signature, the registry sha256 — would hash this faithfully and certify it (celilo#1463).",
|
|
134
|
+
].join('\n');
|
|
135
|
+
}
|
|
@@ -8,10 +8,10 @@
|
|
|
8
8
|
|
|
9
9
|
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
10
10
|
import { execFileSync } from 'node:child_process';
|
|
11
|
-
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
11
|
+
import { mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
12
12
|
import { tmpdir } from 'node:os';
|
|
13
13
|
import { join } from 'node:path';
|
|
14
|
-
import { buildModule, computeChecksums } from './build';
|
|
14
|
+
import { buildModule, computeChecksums, stagingParent, workspaceRoot } from './build';
|
|
15
15
|
|
|
16
16
|
describe('computeChecksums — scripts/node_modules bundling (ISS-0046)', () => {
|
|
17
17
|
let dir: string;
|
|
@@ -143,3 +143,56 @@ describe('buildModule — a root package.json must not strip the hook runtime',
|
|
|
143
143
|
expect(listed).not.toContain('scripts/node_modules/.bin/tldts');
|
|
144
144
|
});
|
|
145
145
|
});
|
|
146
|
+
|
|
147
|
+
describe('stagingParent — cross-device staging (celilo#1463)', () => {
|
|
148
|
+
it('stages under tmpdir when it shares a device with the source', () => {
|
|
149
|
+
expect(stagingParent('/src/mod', () => 1)).toBe(tmpdir());
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
it('stages beside the source when tmpdir is a different mount', () => {
|
|
153
|
+
// The CI runner's PrivateTmp=yes case: a rename from the build cwd into
|
|
154
|
+
// tmpdir() would be cross-device, and bun ships an all-NUL binary.
|
|
155
|
+
const devs: Record<string, number> = { [tmpdir()]: 2 };
|
|
156
|
+
expect(stagingParent('/src/mod', (p) => devs[p] ?? 1)).toBe('/src');
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it('stages beside the source when a device cannot be read', () => {
|
|
160
|
+
expect(
|
|
161
|
+
stagingParent('/src/mod', () => {
|
|
162
|
+
throw new Error('EACCES');
|
|
163
|
+
}),
|
|
164
|
+
).toBe('/src');
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
it('stages at the workspace root, never inside it (ce-eogp)', () => {
|
|
168
|
+
// Staging at modules/celilo-package-XXX puts a second copy of
|
|
169
|
+
// modules/<id>/site under the root's `modules/*/site` workspace glob, and
|
|
170
|
+
// `bun install` then aborts with `Workspace name ... already exists`.
|
|
171
|
+
const parent = stagingParent(
|
|
172
|
+
'/repo/modules/celilo-website',
|
|
173
|
+
() => {
|
|
174
|
+
throw new Error('EXDEV');
|
|
175
|
+
},
|
|
176
|
+
() => '/repo',
|
|
177
|
+
);
|
|
178
|
+
expect(parent).toBe('/repo');
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
it('finds the nearest ancestor package.json declaring workspaces', () => {
|
|
182
|
+
const files: Record<string, string> = {
|
|
183
|
+
'/repo/package.json': '{"workspaces":["modules/*/site"]}',
|
|
184
|
+
'/repo/modules/m/package.json': '{"name":"m"}',
|
|
185
|
+
};
|
|
186
|
+
expect(workspaceRoot('/repo/modules/m', (f) => files[f] ?? null)).toBe('/repo');
|
|
187
|
+
expect(workspaceRoot('/elsewhere/m', (f) => files[f] ?? null)).toBeNull();
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
it('always picks a parent on the source device, for real paths', () => {
|
|
191
|
+
const src = mkdtempSync(join(tmpdir(), 'celilo-staging-parent-'));
|
|
192
|
+
try {
|
|
193
|
+
expect(statSync(stagingParent(src)).dev).toBe(statSync(src).dev);
|
|
194
|
+
} finally {
|
|
195
|
+
rmSync(src, { recursive: true, force: true });
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
});
|
|
@@ -1,20 +1,78 @@
|
|
|
1
1
|
import { execFileSync, execSync } from 'node:child_process';
|
|
2
|
-
import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
2
|
+
import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs';
|
|
3
3
|
import { readFile, readdir, writeFile } from 'node:fs/promises';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
|
-
import { basename, join, relative } from 'node:path';
|
|
5
|
+
import { basename, dirname, join, relative } from 'node:path';
|
|
6
6
|
import { gunzipSync } from 'node:zlib';
|
|
7
7
|
import { create as tarCreate } from 'tar';
|
|
8
8
|
import { parse as parseYaml } from 'yaml';
|
|
9
9
|
import { log } from '../../cli/prompts';
|
|
10
10
|
import { formatViolations, scanModuleDirectory } from '../../policy/module-script-scan';
|
|
11
11
|
import { validateModuleDirectory } from '../import';
|
|
12
|
+
import { verifyBuildArtifacts } from './artifact-bytes';
|
|
12
13
|
import { resolveBuildCommandPaths } from './build-paths';
|
|
13
14
|
import { computeFileChecksum } from './checksum';
|
|
14
15
|
import { classifyModulePath, includeNodeModulesPath } from './package-rules';
|
|
15
16
|
import { signChecksums } from './signature';
|
|
16
17
|
import { rewriteWorkspaceDeps } from './workspace-deps';
|
|
17
18
|
|
|
19
|
+
/**
|
|
20
|
+
* Pick the parent directory the build staging dir is created under.
|
|
21
|
+
*
|
|
22
|
+
* `bun build --compile --outfile=X` writes its staging temp file in the
|
|
23
|
+
* process CWD and `renameat()`s it onto X. Across a mount boundary that
|
|
24
|
+
* rename returns EXDEV, and bun's fallback fallocates the right size then
|
|
25
|
+
* `copy_file_range()`s nothing (it never rewinds the source fd, and never
|
|
26
|
+
* checks the return) — producing a right-sized, entirely-NUL binary and a
|
|
27
|
+
* zero exit. The CI runner's systemd unit sets `PrivateTmp=yes`, which makes
|
|
28
|
+
* tmpdir() a different mount from the checkout, so every celilo module build
|
|
29
|
+
* that renamed an artifact into the stage hit it (celilo#1463).
|
|
30
|
+
*
|
|
31
|
+
* So: stage under tmpdir() when it shares a device with the module source,
|
|
32
|
+
* and on the source's own device otherwise. That keeps the rename intra-device
|
|
33
|
+
* for the whole class of builds, not just bun's.
|
|
34
|
+
*
|
|
35
|
+
* "Beside the source" is NOT safe for a module inside a bun workspace: the
|
|
36
|
+
* staged copy of `modules/<id>` lands at `modules/celilo-package-XXXXXX`, whose
|
|
37
|
+
* `site/`, `server/` and `e2e/` subdirs match the root package.json's
|
|
38
|
+
* `modules/*\/site` globs, so `bun install` during the module build aborts with
|
|
39
|
+
* `Workspace name "<x>" already exists` (ce-eogp, regression from celilo#1463).
|
|
40
|
+
* So when the source sits inside a workspace, stage at the workspace ROOT,
|
|
41
|
+
* which is on the same device and matched by no workspace glob.
|
|
42
|
+
*/
|
|
43
|
+
export function workspaceRoot(
|
|
44
|
+
dir: string,
|
|
45
|
+
readPkg: (p: string) => string | null = safeRead,
|
|
46
|
+
): string | null {
|
|
47
|
+
for (let d = dir; ; d = dirname(d)) {
|
|
48
|
+
const raw = readPkg(join(d, 'package.json'));
|
|
49
|
+
if (raw?.includes('"workspaces"')) return d;
|
|
50
|
+
if (dirname(d) === d) return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function safeRead(p: string): string | null {
|
|
55
|
+
try {
|
|
56
|
+
return readFileSync(p, 'utf-8');
|
|
57
|
+
} catch {
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function stagingParent(
|
|
63
|
+
sourceDir: string,
|
|
64
|
+
deviceOf: (p: string) => number = (p) => statSync(p).dev,
|
|
65
|
+
findRoot: (d: string) => string | null = workspaceRoot,
|
|
66
|
+
): string {
|
|
67
|
+
try {
|
|
68
|
+
if (deviceOf(tmpdir()) === deviceOf(sourceDir)) return tmpdir();
|
|
69
|
+
} catch {
|
|
70
|
+
// Either path unreadable — fall through to a same-device parent, which is
|
|
71
|
+
// the safe side: same device by construction.
|
|
72
|
+
}
|
|
73
|
+
return findRoot(sourceDir) ?? dirname(sourceDir);
|
|
74
|
+
}
|
|
75
|
+
|
|
18
76
|
/**
|
|
19
77
|
* Checksums data structure
|
|
20
78
|
*/
|
|
@@ -229,7 +287,7 @@ export async function buildModule(options: ModuleBuildOptions): Promise<ModuleBu
|
|
|
229
287
|
// Avoids copying node_modules, build artifacts, git history.
|
|
230
288
|
// - If no package.json (simple modules, test fixtures), fall back to
|
|
231
289
|
// recursive copy with EXCLUDE_PATTERNS filter.
|
|
232
|
-
const buildDir = mkdtempSync(join(
|
|
290
|
+
const buildDir = mkdtempSync(join(stagingParent(sourceDir), 'celilo-package-'));
|
|
233
291
|
const hasPackageJson = existsSync(join(sourceDir, 'package.json'));
|
|
234
292
|
|
|
235
293
|
try {
|
|
@@ -385,7 +443,14 @@ export async function buildModule(options: ModuleBuildOptions): Promise<ModuleBu
|
|
|
385
443
|
return { success: false, error: `Auto-build failed: Build exited with code ${msg}` };
|
|
386
444
|
}
|
|
387
445
|
|
|
388
|
-
// Verify artifacts exist
|
|
446
|
+
// Verify artifacts exist, and that they are actually usable.
|
|
447
|
+
//
|
|
448
|
+
// Existence was the whole check, and existence is exactly what a
|
|
449
|
+
// right-sized file of pure NUL has. celilo#1463: two ~100 MiB module
|
|
450
|
+
// binaries were packaged, signed, published, downloaded, verified and
|
|
451
|
+
// installed as all-zero files, and every stage reported success, because
|
|
452
|
+
// this is the last point in the pipeline that sees the artifact before it
|
|
453
|
+
// becomes the thing all those checks are computed OVER.
|
|
389
454
|
if (manifest.build.artifacts) {
|
|
390
455
|
const missing = manifest.build.artifacts.filter(
|
|
391
456
|
(a: string) => !existsSync(join(buildDir, a)),
|
|
@@ -396,6 +461,10 @@ export async function buildModule(options: ModuleBuildOptions): Promise<ModuleBu
|
|
|
396
461
|
error: `Build succeeded but artifacts missing: ${missing.join(', ')}`,
|
|
397
462
|
};
|
|
398
463
|
}
|
|
464
|
+
const unusable = verifyBuildArtifacts(buildDir, manifest.build.artifacts, moduleId);
|
|
465
|
+
if (unusable) {
|
|
466
|
+
return { success: false, error: unusable };
|
|
467
|
+
}
|
|
399
468
|
}
|
|
400
469
|
}
|
|
401
470
|
|