@celilo/cli 5.2.2 → 5.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/cli",
-  "version": "5.2.2",
+  "version": "5.3.0",
   "description": "Celilo — home lab orchestration CLI",
   "type": "module",
   "bin": {
@@ -61,7 +61,7 @@
     "@aws-sdk/lib-storage": "^3.1101.0",
     "@celilo/capabilities": "^6.1.0",
     "@celilo/cli-display": "^0.2.0",
-    "@celilo/core": "^0.15.0",
+    "@celilo/core": "^0.16.0",
     "@celilo/event-bus": "^0.7.0",
     "ajv": "^8.18.0",
     "drizzle-orm": "^0.36.4",

package/src/api/serve-inprocess.test.ts

@@ -0,0 +1,103 @@
+/**
+ * A read runs in THIS process; everything else still gets a child.
+ *
+ * Every command used to be a fresh `celilo` child, which costs a full CLI boot —
+ * ~1.1s on celilo-mgr, whose root is an SD card — paid per command even on a
+ * reused connection. With connection reuse landed, every console endpoint still
+ * measured ~1.0-1.5s regardless of what it did, because the boot dominated the
+ * work. A read cannot park, so it has nothing to outlive the transport and needs
+ * no child (celilo#1457).
+ *
+ * These pin the two things that make in-process execution safe. Both are
+ * invisible in normal operation and catastrophic when wrong, which is exactly
+ * the shape that needs a test rather than a comment.
+ */
+import { describe, expect, test } from 'bun:test';
+import { isReadOnlyPath } from '@celilo/core';
+
+describe('which commands may run in-process', () => {
+  test('the console’s reads are classified read-only', () => {
+    // If any of these stopped being a read, it would start paying a CLI boot
+    // again and the console would silently regress to ~1s per call.
+    for (const argv of [
+      ['console', 'status', '--json'],
+      ['alerts', 'list', '--json'],
+      ['backup', 'list', '--json'],
+      ['person', 'list', '--json'],
+      ['module', 'list', '--json'],
+    ]) {
+      expect(isReadOnlyPath(argv)).toBe(true);
+    }
+  });
+
+  test('a mutating command is NOT, so it still gets a child', () => {
+    // The child is what lets a parked command outlive the ssh transport. A
+    // write running in-process would lose that, and a park would die with the
+    // connection.
+    for (const argv of [
+      ['module', 'deploy', 'caddy'],
+      ['alerts', 'ack', 'some-id'],
+      ['module', 'update'],
+      ['system', 'init'],
+    ]) {
+      expect(isReadOnlyPath(argv)).toBe(false);
+    }
+  });
+
+  test('an unknown verb is not treated as a read', () => {
+    // Fail closed: an unrecognised path must take the child, not the fast path.
+    expect(isReadOnlyPath(['not-a-command'])).toBe(false);
+    expect(isReadOnlyPath([])).toBe(false);
+  });
+});
+
+describe('the guards in-process execution depends on', () => {
+  test('process.exit can be intercepted and restored', () => {
+    // The CLI calls process.exit on some paths and inside runCli
+    // (cli/index.ts:1582). For a CHILD that is an ordinary exit; in-process it
+    // would take the whole api-serve down mid-session, dropping the client's
+    // connection and any parked session it was holding.
+    const real = process.exit;
+    class ExitCalled extends Error {
+      constructor(readonly code: number) {
+        super(`exit ${code}`);
+      }
+    }
+    let caught: number | null = null;
+    try {
+      (process as { exit: unknown }).exit = (code?: number): never => {
+        throw new ExitCalled(code ?? 0);
+      };
+      try {
+        process.exit(3);
+      } catch (e) {
+        if (e instanceof ExitCalled) caught = e.code;
+      }
+    } finally {
+      (process as { exit: unknown }).exit = real;
+    }
+    expect(caught).toBe(3);
+    expect(process.exit).toBe(real);
+  });
+
+  test('a stdout reference bound BEFORE a redirect survives it', () => {
+    // `send` writes the protocol through a reference bound at module load. If
+    // it used process.stdout.write directly, the capture would swallow the
+    // framing and the client would receive api-serve's own protocol as if it
+    // were command output — unparseable, and three layers from the cause.
+    const real = process.stdout.write.bind(process.stdout);
+    const bound = real;
+    const captured: string[] = [];
+    try {
+      (process.stdout as { write: unknown }).write = (c: unknown): boolean => {
+        captured.push(String(c));
+        return true;
+      };
+      process.stdout.write('this is command output\n');
+      bound(''); // the protocol path: must NOT land in `captured`
+    } finally {
+      (process.stdout as { write: unknown }).write = real;
+    }
+    expect(captured).toEqual(['this is command output\n']);
+  });
+});

package/src/api/serve.ts

@@ -25,8 +25,10 @@ import {
   ClientMessageSchema,
   type ServerMessage,
   ServerMessageSchema,
+  isReadOnlyPath,
   translateOutputLine,
 } from '@celilo/core';
+import { runCli } from '../cli/index';
 import { parseArguments } from '../cli/parser';
 import { getDataDir, getEventBusPath } from '../config/paths';
 import { getDb } from '../db/client';
@@ -51,8 +53,17 @@ const EXIT_PERMISSION_DENIED = 126;
 /** How often an attached client polls a session's buffer for new output. */
 const ATTACH_POLL_MS = 250;
 
+/**
+ * The real stdout, captured at module load.
+ *
+ * `runCommandInProcess` redirects `process.stdout.write` to capture a command's
+ * output. Without this reference `send` would write the PROTOCOL through that
+ * redirect and the client would receive its own framing as command output.
+ */
+const writeProtocol = process.stdout.write.bind(process.stdout);
+
 function send(msg: ServerMessage): void {
-  process.stdout.write(`${JSON.stringify(msg)}\n`);
+  writeProtocol(`${JSON.stringify(msg)}\n`);
 }
 
 /**
@@ -132,8 +143,128 @@ async function pumpLines(
   }
 }
 
+/**
+ * Run a READ-ONLY command in this process instead of spawning a child.
+ *
+ * Every command used to be a fresh `celilo` child, which costs a full CLI boot —
+ * ~1.1s on celilo-mgr, whose root is an SD card. That is paid per command even
+ * on a reused connection, and it was the floor under the web console: with
+ * connection reuse landed, every endpoint still measured ~1.0-1.5s regardless of
+ * what it actually did, because the boot dominated the work (celilo#1457).
+ *
+ * WHY ONLY READS. The child exists so a command that PARKS can outlive the ssh
+ * transport: the session keeps it alive and buffers its output. A read cannot
+ * park — it raises no interview — so it has nothing to outlive and needs no
+ * child. `isReadOnlyPath` is the same predicate the API's authz uses, so this
+ * cannot widen what runs in-process without also widening what counts as a read.
+ *
+ * TWO THINGS MAKE IT SAFE, and neither is optional:
+ *
+ *  - Output is captured by redirecting `process.stdout/stderr.write`, so `send`
+ *    must not use them. It writes through `writeProtocol`, bound before any
+ *    redirect exists. Commands are serialized (one `commandRunning` slot), so
+ *    the redirect can never straddle two commands.
+ *  - `process.exit` is intercepted. The CLI calls it on some paths and inside
+ *    `runCli` (cli/index.ts:1582), which for a CHILD is an ordinary exit and
+ *    in-process would take the whole server down mid-session. It is replaced
+ *    with a throw that carries the code, and restored in `finally`.
+ */
+async function runCommandInProcess(
+  argv: string[],
+  forward: (msg: ServerMessage) => void,
+): Promise<number> {
+  const realStdout = process.stdout.write.bind(process.stdout);
+  const realStderr = process.stderr.write.bind(process.stderr);
+  const realExit = process.exit.bind(process);
+
+  const buffers: Record<'stdout' | 'stderr', string> = { stdout: '', stderr: '' };
+  const capture =
+    (stream: 'stdout' | 'stderr') =>
+    (chunk: unknown): boolean => {
+      const text =
+        typeof chunk === 'string' ? chunk : new TextDecoder().decode(chunk as Uint8Array);
+      buffers[stream] += text;
+      let nl = buffers[stream].indexOf('\n');
+      while (nl >= 0) {
+        const line = buffers[stream].slice(0, nl);
+        buffers[stream] = buffers[stream].slice(nl + 1);
+        const msg = translateOutputLine(line);
+        forward(msg.type === 'log' ? { ...msg, stream } : msg);
+        nl = buffers[stream].indexOf('\n');
+      }
+      return true;
+    };
+
+  const flush = (): void => {
+    for (const stream of ['stdout', 'stderr'] as const) {
+      if (buffers[stream].length > 0) {
+        const msg = translateOutputLine(buffers[stream]);
+        forward(msg.type === 'log' ? { ...msg, stream } : msg);
+        buffers[stream] = '';
+      }
+    }
+  };
+
+  class ExitCalled extends Error {
+    constructor(readonly code: number) {
+      super(`process.exit(${code})`);
+    }
+  }
+
+  let exitCode: number;
+  try {
+    (process.stdout as { write: unknown }).write = capture('stdout');
+    (process.stderr as { write: unknown }).write = capture('stderr');
+    (process as { exit: unknown }).exit = (code?: number): never => {
+      throw new ExitCalled(code ?? 0);
+    };
+
+    // `parseArguments` does `argv.slice(2)`, so runCli takes process.argv SHAPE —
+    // executable, script, then the command. The child path gets this right by
+    // construction (`Bun.spawn([process.execPath, Bun.main, ...argv])`); passing
+    // bare argv here silently dropped the first two words, so `service list
+    // --json` parsed as `--json` and the command failed. Mirror the child.
+    const result = await runCli([process.execPath, Bun.main, ...argv]);
+    // `runCli` RETURNS the terminal message rather than printing it, so it never
+    // passed through the capture above. A child printed it on the way out; in
+    // process it has to be forwarded explicitly or the client sees an empty
+    // payload for a command that succeeded.
+    if (result.success) {
+      exitCode = 0;
+      if (result.message) {
+        forward({ type: 'log', message: result.message, stream: 'stdout' });
+      }
+    } else {
+      exitCode = 1;
+      forward({ type: 'log', message: result.error, stream: 'stderr' });
+    }
+  } catch (error) {
+    if (error instanceof ExitCalled) {
+      exitCode = error.code;
+    } else {
+      exitCode = 1;
+      forward({
+        type: 'log',
+        message: error instanceof Error ? error.message : String(error),
+        stream: 'stderr',
+      });
+    }
+  } finally {
+    (process.stdout as { write: unknown }).write = realStdout;
+    (process.stderr as { write: unknown }).write = realStderr;
+    (process as { exit: unknown }).exit = realExit;
+  }
+
+  flush();
+  forward(resultMessage(exitCode === 0, exitCode));
+  return exitCode;
+}
+
 /** Run an authorized command as a child; returns its exit code. */
 async function runCommand(argv: string[], forward: (msg: ServerMessage) => void): Promise<number> {
+  // A read cannot park, so it needs no child and should not pay a CLI boot.
+  if (isReadOnlyPath(argv)) return runCommandInProcess(argv, forward);
+
   // Re-invoke this same CLI as a child. The child is non-TTY (piped stdout) so
   // ProgressDisplay resolves to protocol mode and emits `[progress:*]` markers.
   const child = Bun.spawn([process.execPath, Bun.main, ...argv], {

package/src/module/packaging/build.test.ts

@@ -8,10 +8,10 @@
 
 import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
 import { execFileSync } from 'node:child_process';
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
-import { buildModule, computeChecksums } from './build';
+import { buildModule, computeChecksums, stagingParent, workspaceRoot } from './build';
 
 describe('computeChecksums — scripts/node_modules bundling (ISS-0046)', () => {
   let dir: string;
@@ -143,3 +143,56 @@ describe('buildModule — a root package.json must not strip the hook runtime',
     expect(listed).not.toContain('scripts/node_modules/.bin/tldts');
   });
 });
+
+describe('stagingParent — cross-device staging (celilo#1463)', () => {
+  it('stages under tmpdir when it shares a device with the source', () => {
+    expect(stagingParent('/src/mod', () => 1)).toBe(tmpdir());
+  });
+
+  it('stages beside the source when tmpdir is a different mount', () => {
+    // The CI runner's PrivateTmp=yes case: a rename from the build cwd into
+    // tmpdir() would be cross-device, and bun ships an all-NUL binary.
+    const devs: Record<string, number> = { [tmpdir()]: 2 };
+    expect(stagingParent('/src/mod', (p) => devs[p] ?? 1)).toBe('/src');
+  });
+
+  it('stages beside the source when a device cannot be read', () => {
+    expect(
+      stagingParent('/src/mod', () => {
+        throw new Error('EACCES');
+      }),
+    ).toBe('/src');
+  });
+
+  it('stages at the workspace root, never inside it (ce-eogp)', () => {
+    // Staging at modules/celilo-package-XXX puts a second copy of
+    // modules/<id>/site under the root's `modules/*/site` workspace glob, and
+    // `bun install` then aborts with `Workspace name ... already exists`.
+    const parent = stagingParent(
+      '/repo/modules/celilo-website',
+      () => {
+        throw new Error('EXDEV');
+      },
+      () => '/repo',
+    );
+    expect(parent).toBe('/repo');
+  });
+
+  it('finds the nearest ancestor package.json declaring workspaces', () => {
+    const files: Record<string, string> = {
+      '/repo/package.json': '{"workspaces":["modules/*/site"]}',
+      '/repo/modules/m/package.json': '{"name":"m"}',
+    };
+    expect(workspaceRoot('/repo/modules/m', (f) => files[f] ?? null)).toBe('/repo');
+    expect(workspaceRoot('/elsewhere/m', (f) => files[f] ?? null)).toBeNull();
+  });
+
+  it('always picks a parent on the source device, for real paths', () => {
+    const src = mkdtempSync(join(tmpdir(), 'celilo-staging-parent-'));
+    try {
+      expect(statSync(stagingParent(src)).dev).toBe(statSync(src).dev);
+    } finally {
+      rmSync(src, { recursive: true, force: true });
+    }
+  });
+});

package/src/module/packaging/build.ts

@@ -1,8 +1,8 @@
 import { execFileSync, execSync } from 'node:child_process';
-import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs';
 import { readFile, readdir, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
-import { basename, join, relative } from 'node:path';
+import { basename, dirname, join, relative } from 'node:path';
 import { gunzipSync } from 'node:zlib';
 import { create as tarCreate } from 'tar';
 import { parse as parseYaml } from 'yaml';
@@ -16,6 +16,63 @@ import { classifyModulePath, includeNodeModulesPath } from './package-rules';
 import { signChecksums } from './signature';
 import { rewriteWorkspaceDeps } from './workspace-deps';
 
+/**
+ * Pick the parent directory the build staging dir is created under.
+ *
+ * `bun build --compile --outfile=X` writes its staging temp file in the
+ * process CWD and `renameat()`s it onto X. Across a mount boundary that
+ * rename returns EXDEV, and bun's fallback fallocates the right size then
+ * `copy_file_range()`s nothing (it never rewinds the source fd, and never
+ * checks the return) — producing a right-sized, entirely-NUL binary and a
+ * zero exit. The CI runner's systemd unit sets `PrivateTmp=yes`, which makes
+ * tmpdir() a different mount from the checkout, so every celilo module build
+ * that renamed an artifact into the stage hit it (celilo#1463).
+ *
+ * So: stage under tmpdir() when it shares a device with the module source,
+ * and on the source's own device otherwise. That keeps the rename intra-device
+ * for the whole class of builds, not just bun's.
+ *
+ * "Beside the source" is NOT safe for a module inside a bun workspace: the
+ * staged copy of `modules/<id>` lands at `modules/celilo-package-XXXXXX`, whose
+ * `site/`, `server/` and `e2e/` subdirs match the root package.json's
+ * `modules/*\/site` globs, so `bun install` during the module build aborts with
+ * `Workspace name "<x>" already exists` (ce-eogp, regression from celilo#1463).
+ * So when the source sits inside a workspace, stage at the workspace ROOT,
+ * which is on the same device and matched by no workspace glob.
+ */
+export function workspaceRoot(
+  dir: string,
+  readPkg: (p: string) => string | null = safeRead,
+): string | null {
+  for (let d = dir; ; d = dirname(d)) {
+    const raw = readPkg(join(d, 'package.json'));
+    if (raw?.includes('"workspaces"')) return d;
+    if (dirname(d) === d) return null;
+  }
+}
+
+function safeRead(p: string): string | null {
+  try {
+    return readFileSync(p, 'utf-8');
+  } catch {
+    return null;
+  }
+}
+
+export function stagingParent(
+  sourceDir: string,
+  deviceOf: (p: string) => number = (p) => statSync(p).dev,
+  findRoot: (d: string) => string | null = workspaceRoot,
+): string {
+  try {
+    if (deviceOf(tmpdir()) === deviceOf(sourceDir)) return tmpdir();
+  } catch {
+    // Either path unreadable — fall through to a same-device parent, which is
+    // the safe side: same device by construction.
+  }
+  return findRoot(sourceDir) ?? dirname(sourceDir);
+}
+
 /**
  * Checksums data structure
  */
@@ -230,7 +287,7 @@ export async function buildModule(options: ModuleBuildOptions): Promise<ModuleBu
   //   Avoids copying node_modules, build artifacts, git history.
   // - If no package.json (simple modules, test fixtures), fall back to
   //   recursive copy with EXCLUDE_PATTERNS filter.
-  const buildDir = mkdtempSync(join(tmpdir(), 'celilo-package-'));
+  const buildDir = mkdtempSync(join(stagingParent(sourceDir), 'celilo-package-'));
   const hasPackageJson = existsSync(join(sourceDir, 'package.json'));
 
   try {