@celilo/e2e 0.9.3 → 0.10.2

package/simulators/signal-cli/server.ts

@@ -0,0 +1,328 @@
+/**
+ * signal-cli JSON-RPC daemon simulator.
+ *
+ * Mimics `signal-cli daemon --http <addr>`: JSON-RPC 2.0 over POST, with the
+ * behaviours that make the real thing awkward, because those are the ones
+ * celilo has to survive:
+ *
+ *   - errors arrive as a JSON-RPC `error` member with HTTP **200**, so code
+ *     that checks `response.ok` believes a rejected send succeeded;
+ *   - an unlinked/revoked account answers HTTP perfectly and can send nothing;
+ *   - `receive` returns receipts and typing indicators alongside real
+ *     messages, and those are not replies;
+ *   - `receive` is REFUSED unless the daemon was started with
+ *     `--receive-mode=manual`, which is the real default and the reason the
+ *     ack path shipped broken.
+ *
+ * It pushes back rather than saying yes. Sending to a number that has not been
+ * registered with the simulator fails the way Signal fails.
+ *
+ * ── FIDELITY ───────────────────────────────────────────────────────────────
+ * Every shape below is verified against signal-cli 0.14.6 running in a
+ * container, plus its published JSON schemas — not against recollection. The
+ * contract test in e2e re-checks the real daemon so this file cannot silently
+ * drift from it. See design.md D16 for the verified table.
+ *
+ * Still true: this cannot prove that a LINKED account delivers to a phone.
+ * That needs Signal's real network, and CDSI/SVR2 attestation makes a local
+ * substitute impossible. Everything short of delivery is verified.
+ * ───────────────────────────────────────────────────────────────────────────
+ *
+ * Control surface (simulator-only, not part of signal-cli):
+ *   POST /_control/inbound   {from, body}  queue a reply as if a human sent it
+ *   POST /_control/soft-fail {recipient}   make a recipient fail INSIDE a success
+ *   POST /_control/unlink                  revoke the device link
+ *   POST /_control/relink                  restore it
+ *   GET  /_control/sent                    every message sent, for assertions
+ */
+
+const ACCOUNT = process.env.SIGNAL_ACCOUNT ?? '+15551234567';
+const PORT = Number(process.env.SIGNAL_RPC_PORT ?? 8080);
+
+/**
+ * Mirrors `signal-cli daemon --receive-mode=<mode>`, and DEFAULTS TO THE REAL
+ * DEFAULT (`on-start`) rather than to the convenient one.
+ *
+ * In on-start the daemon registers a strong receive handler, drains every
+ * inbound message into its own SSE stream, and REFUSES the `receive` JSON-RPC
+ * call outright. A simulator that answers `receive` regardless is a placebo
+ * for the whole ack path: that is precisely how the deployed module shipped
+ * with an unreadable transport while this suite stayed green. Only `manual`
+ * lets `receive` work — so a caller has to be started the way it must be
+ * deployed.
+ *
+ * Verified in signal-cli 0.14.6: DaemonCommand.java:78, :114;
+ * ManagerImpl.java:1494-1501, :1605; ReceiveCommand.java:125.
+ */
+const RECEIVE_MODE = process.env.SIGNAL_RECEIVE_MODE ?? 'on-start';
+
+/** Numbers the simulator will accept as recipients. Anything else is rejected. */
+const KNOWN_RECIPIENTS = new Set(
+  (process.env.SIGNAL_KNOWN_RECIPIENTS ?? '')
+    .split(',')
+    .map((n) => n.trim())
+    .filter(Boolean),
+);
+
+interface SentMessage {
+  recipient: string;
+  message: string;
+  timestamp: number;
+}
+
+interface QueuedInbound {
+  from: string;
+  body: string;
+  timestamp: number;
+}
+
+const sent: SentMessage[] = [];
+/**
+ * Recipients that fail INSIDE a successful response, with a non-SUCCESS
+ * `type`. signal-cli only raises a JSON-RPC error when nothing succeeded, so
+ * this is how a partial failure looks — and how a client that checks only
+ * `error` ends up recording an undelivered page as sent.
+ */
+const unregisteredRecipients = new Set<string>();
+const inboundQueue: QueuedInbound[] = [];
+let linked = true;
+let clock = 1_722_200_000_000;
+
+const nextTimestamp = () => ++clock;
+
+function rpcResult(id: unknown, result: unknown): Response {
+  return Response.json({ jsonrpc: '2.0', id, result });
+}
+
+/**
+ * A JSON-RPC error, returned with HTTP 200 exactly as signal-cli does. This is
+ * the single most important fidelity detail in the simulator.
+ */
+function rpcError(id: unknown, code: number, message: string): Response {
+  return Response.json({ jsonrpc: '2.0', id, error: { code, message } }, { status: 200 });
+}
+
+function handleSend(id: unknown, params: Record<string, unknown>): Response {
+  if (!linked) {
+    return rpcError(id, -32000, 'Account is not registered or the device link was removed');
+  }
+
+  const recipients = Array.isArray(params.recipient) ? (params.recipient as string[]) : [];
+  const message = typeof params.message === 'string' ? params.message : '';
+
+  if (recipients.length === 0) return rpcError(id, -32602, 'No recipient given');
+
+  for (const recipient of recipients) {
+    // Push back like Signal: an unregistered number is a hard failure, not a
+    // silent no-op.
+    if (KNOWN_RECIPIENTS.size > 0 && !KNOWN_RECIPIENTS.has(recipient)) {
+      return rpcError(id, -32000, `Unregistered user: ${recipient}`);
+    }
+  }
+
+  const timestamp = nextTimestamp();
+  for (const recipient of recipients) {
+    sent.push({ recipient, message, timestamp });
+    console.log(`[signal-sim] -> ${recipient}: ${message.split('\n')[0]}`);
+  }
+
+  // Verified shape: { timestamp, results: [ { type, recipientAddress } ] }.
+  // The per-recipient `type` is the THIRD failure mode — a call can succeed
+  // at the JSON-RPC level and still not have delivered.
+  return rpcResult(id, {
+    timestamp,
+    results: recipients.map((recipient) => ({
+      type: unregisteredRecipients.has(recipient) ? 'UNREGISTERED_FAILURE' : 'SUCCESS',
+      recipientAddress: { number: recipient },
+    })),
+  });
+}
+
+/**
+ * Drain the inbound queue.
+ *
+ * Interleaves a delivery receipt and a typing indicator with real messages —
+ * the real daemon does, and a consumer that treats every envelope as a reply
+ * will misbehave on them.
+ */
+function handleReceive(id: unknown): Response {
+  // The refusal comes BEFORE the link check, as it does in the real daemon:
+  // the receive thread is started per account at daemon start, so an
+  // already-receiving manager rejects the call without ever looking at what
+  // is queued. Verbatim message from ReceiveCommand.java:125.
+  if (RECEIVE_MODE !== 'manual') {
+    return rpcError(
+      id,
+      -32001,
+      'Receive command cannot be used if messages are already being received.',
+    );
+  }
+
+  if (!linked) {
+    return rpcError(id, -32000, 'Account is not registered or the device link was removed');
+  }
+
+  const envelopes: unknown[] = [
+    {
+      envelope: {
+        sourceNumber: ACCOUNT,
+        timestamp: nextTimestamp(),
+        receiptMessage: { isDelivery: true },
+      },
+    },
+  ];
+
+  while (inboundQueue.length > 0) {
+    const item = inboundQueue.shift() as QueuedInbound;
+    // A reply from the linked account itself — the default single-operator
+    // setup, where the route points at the number the transport is a
+    // secondary device of — is a NOTE-TO-SELF, and the real daemon delivers
+    // it as a sync transcript, never as a dataMessage. Emitting it as a
+    // dataMessage would let a client that only reads dataMessage look
+    // correct here while dropping every reply in production (#460).
+    envelopes.push(
+      item.from === ACCOUNT
+        ? {
+            envelope: {
+              sourceNumber: ACCOUNT,
+              timestamp: nextTimestamp(),
+              syncMessage: {
+                sentMessage: {
+                  destinationNumber: ACCOUNT,
+                  timestamp: item.timestamp,
+                  message: item.body,
+                },
+              },
+            },
+          }
+        : {
+            envelope: {
+              sourceNumber: item.from,
+              timestamp: item.timestamp,
+              dataMessage: { message: item.body },
+            },
+          },
+    );
+    console.log(`[signal-sim] <- ${item.from}: ${item.body}`);
+  }
+
+  envelopes.push({
+    envelope: {
+      sourceNumber: ACCOUNT,
+      timestamp: nextTimestamp(),
+      typingMessage: { action: 'STARTED' },
+    },
+  });
+
+  return rpcResult(id, envelopes);
+}
+
+function handleListAccounts(id: unknown): Response {
+  // A revoked link answers happily with an empty account list — reachability
+  // is not health.
+  return rpcResult(id, linked ? [{ number: ACCOUNT }] : []);
+}
+
+async function handleControl(url: URL, req: Request): Promise<Response> {
+  switch (url.pathname) {
+    case '/_control/inbound': {
+      const body = (await req.json()) as { from?: string; body?: string };
+      if (!body.from || !body.body) {
+        return Response.json({ error: 'from and body required' }, { status: 400 });
+      }
+      inboundQueue.push({ from: body.from, body: body.body, timestamp: nextTimestamp() });
+      return Response.json({ queued: inboundQueue.length });
+    }
+    case '/_control/soft-fail': {
+      const body = (await req.json()) as { recipient?: string };
+      if (!body.recipient) {
+        return Response.json({ error: 'recipient required' }, { status: 400 });
+      }
+      unregisteredRecipients.add(body.recipient);
+      return Response.json({ softFailing: [...unregisteredRecipients] });
+    }
+    case '/_control/unlink':
+      linked = false;
+      console.log('[signal-sim] device link revoked');
+      return Response.json({ linked });
+    case '/_control/relink':
+      linked = true;
+      console.log('[signal-sim] device link restored');
+      return Response.json({ linked });
+    case '/_control/sent':
+      return Response.json({ sent });
+    case '/_control/reset':
+      sent.length = 0;
+      inboundQueue.length = 0;
+      unregisteredRecipients.clear();
+      linked = true;
+      return Response.json({ ok: true });
+    default:
+      return new Response('Not found', { status: 404 });
+  }
+}
+
+Bun.serve({
+  port: PORT,
+  hostname: '0.0.0.0',
+  async fetch(req) {
+    const url = new URL(req.url);
+
+    if (url.pathname.startsWith('/_control/')) return handleControl(url, req);
+
+    // Verified endpoints on the real daemon.
+    if (url.pathname === '/api/v1/check') {
+      return req.method === 'GET' ? new Response(null, { status: 200 }) : new Response(null, { status: 405 });
+    }
+    if (url.pathname === '/api/v1/events') {
+      return new Response('', { status: 200, headers: { 'content-type': 'text/event-stream' } });
+    }
+    if (url.pathname !== '/api/v1/rpc') return new Response('Not found', { status: 404 });
+
+    // Verified: the real daemon returns 415 when Content-Type is not JSON.
+    const contentType = req.headers.get('content-type');
+    if (!contentType?.startsWith('application/json')) {
+      return new Response(null, { status: 415 });
+    }
+
+    if (req.method !== 'POST') return new Response('Method not allowed', { status: 405 });
+
+    let payload: { id?: unknown; method?: string; params?: Record<string, unknown> };
+    try {
+      payload = (await req.json()) as typeof payload;
+    } catch {
+      return Response.json(
+        { jsonrpc: '2.0', id: null, error: { code: -32700, message: 'Parse error' } },
+        { status: 200 },
+      );
+    }
+
+    const { id = null, method, params = {} } = payload;
+
+    switch (method) {
+      case 'send':
+        return handleSend(id, params);
+      case 'receive':
+        return handleReceive(id);
+      case 'listAccounts':
+        return handleListAccounts(id);
+      case 'version':
+        // Verified: { "version": "0.14.6" }
+        return rpcResult(id, { version: '0.14.6' });
+      default:
+        return rpcError(id, -32601, `Method not found: ${method}`);
+    }
+  },
+});
+
+console.log(
+  `[signal-sim] signal-cli JSON-RPC simulator on :${PORT} as ${ACCOUNT} (receive-mode=${RECEIVE_MODE})`,
+);
+if (KNOWN_RECIPIENTS.size > 0) {
+  console.log(`[signal-sim] known recipients: ${[...KNOWN_RECIPIENTS].join(', ')}`);
+}
+
+// Marks this file a module so its top-level constants get their own scope.
+// Without it, `PORT` here collides with the same name in the isitup simulator,
+// which shares a tsconfig.
+export {};

package/src/cli/build.ts

@@ -23,6 +23,7 @@ import { cpSync, existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } fro
 import { basename, join, resolve } from 'node:path';
 import { gunzipSync } from 'node:zlib';
 import { stageAptRepo } from '../../scripts/stage-apt-repo';
+import { stageLibsignal } from '../../scripts/stage-libsignal';
 import { ensureRegistryServerBundle } from '../registry-bundle';
 
 /**
@@ -176,6 +177,12 @@ async function stageSimulatorInputs(pkgDir: string): Promise<void> {
   // time, so this must run before buildDockerImages().
   stageAptRepo(repoRoot, pkgDir);
 
+  // (d) Build + stage the libsignal aarch64 native as a .deb, into the SAME
+  // pool stageAptRepo just populated (so it must run after it — that function
+  // recreates the pool empty). Compiled here on the host's own network rather
+  // than inside the sealed e2e network; see stage-libsignal.ts.
+  stageLibsignal(repoRoot, pkgDir);
+
   console.log('');
 }
 

package/src/cli/index.ts

@@ -61,6 +61,12 @@ function detectTestDir(): { stateDir: string; testsPath: string } {
     }
   }
   if (existsSync(join(cwd, 'tests'))) return { stateDir: cwd, testsPath: join(cwd, 'tests') };
+  // The repo's own top-level suite lives at e2e/tests/, which is what `list`
+  // enumerates. Without this, `cele2e list` shows a test that `cele2e run
+  // <name>` then cannot find — the run/list disagreement in #412.
+  if (existsSync(join(cwd, 'e2e', 'tests'))) {
+    return { stateDir: join(cwd, 'e2e'), testsPath: join(cwd, 'e2e', 'tests') };
+  }
   return { stateDir: cwd, testsPath: join(cwd, 'tests') };
 }
 

package/src/container-manager.ts

@@ -4,7 +4,11 @@ import { tmpdir } from 'node:os';
 import { basename, join, resolve } from 'node:path';
 import { startBrowser } from './browser';
 import { MIN_CLI_VERSION, checkCliVersion } from './cli-version-contract';
-import { SHARED_PROJECT_NAME, generateTestComposeYaml } from './docker-compose-generator';
+import {
+  SHARED_PROJECT_NAME,
+  generateTestComposeYaml,
+  getAllMachines,
+} from './docker-compose-generator';
 import { ensureSharedInfra } from './shared-infra';
 import { SIMULATOR_IPS } from './simulator-ips';
 import { startSocksProxy } from './socks-proxy';
@@ -67,8 +71,16 @@ const activeProjects: Set<string> = new Set();
  *
  * NOTE: only tears down per-test projects. Shared infra is managed
  * separately (by the test runner or explicit stopSharedInfra call).
+ *
+ * MUST stay fully synchronous: it runs from an `exit` handler, where the
+ * event loop is already closed and any async work is silently dropped.
  */
 function cleanupOnExit() {
+  // `--keep` / `--reuse` mean the operator wants the stack to survive for
+  // debugging — and they want it MOST after a failure, which is exactly the
+  // path this handler covers. Tearing down here would delete the evidence.
+  if (process.env.CELILO_E2E_KEEP === '1' || process.env.CELILO_E2E_REUSE === '1') return;
+
   for (const project of activeProjects) {
     try {
       execSync(`docker compose -p ${project} down --volumes --remove-orphans`, {
@@ -77,6 +89,7 @@ function cleanupOnExit() {
       });
     } catch {}
   }
+  activeProjects.clear();
   try {
     execSync('docker network prune -f', { timeout: 10_000, stdio: 'pipe' });
   } catch {}
@@ -90,7 +103,25 @@ process.on('SIGINT', () => {
   cleanupOnExit();
   process.exit(1);
 });
-process.on('beforeExit', () => {
+// `exit`, NOT `beforeExit`: the runner always terminates via `process.exit()`
+// (runner.ts ends with `process.exit(failed > 0 ? 1 : 0)`), and `beforeExit`
+// does not fire for an explicit exit — nor after an uncaught throw. So on the
+// FAILURE path, where the test never reaches `handle.stop()`, teardown was
+// never running and the per-test networks leaked.
+//
+// That leak is not confined to the run that caused it. The sim's networks use
+// the same subnets as a real celilo fleet (dmz 10.0.10.0/24, app 10.0.20.0/24,
+// …), so on a host that lives in those subnets — celilo's own forgejo-builder
+// does — a leaked bridge leaves a DUPLICATE route for a production prefix.
+// Traffic from any later container to the real host at that address is routed
+// into the dead simulated network and blackholed. That took out three
+// consecutive release runs, each dying in `git fetch` after a ~132s connect
+// timeout, long after the e2e run that stranded the network had ended.
+//
+// The start-of-run prefix sweep stays as the crash-recovery backstop (it is
+// the only thing that can clean up after SIGKILL); this makes the common case
+// clean up after itself instead of poisoning the next job. See #508.
+process.on('exit', () => {
   cleanupOnExit();
 });
 
@@ -829,13 +860,11 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
     );
   }
 
-  // Wait for all target machines to complete setup (systemd boot + routing)
-  const allMachines = [
-    ...config.dmzMachines,
-    ...config.appMachines,
-    ...config.secureMachines,
-    ...config.internalMachines,
-  ];
+  // Wait for all target machines to complete setup (systemd boot + routing).
+  // Via getAllMachines so a newly-supported zone cannot be silently skipped —
+  // this list was hand-maintained and omitted `secure-mgmt`, so the first
+  // machine placed there raced `machine add` and failed on SSH intermittently.
+  const allMachines = getAllMachines(config);
   if (allMachines.length > 0) {
     console.log('[progress:start] waiting for target machines | target machines ready');
   }
@@ -935,7 +964,15 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
       `celilo system init --accept-defaults \
         network.internal.subnet=${ZONE_SUBNETS.internal} \
         network.internal.gateway=${ZONE_GATEWAYS.internal} \
-        dns.primary=100.100.0.1 \
+        ${
+          // The control plane needs its subnet declared if celilo-mgr lives
+          // there OR if a machine does — infrastructure selection for a module
+          // declaring `zone: secure-mgmt` reads it either way (#436).
+          config.managementZone === 'secure-mgmt' ||
+          (config.secureMgmtMachines ?? []).length > 0
+            ? `network.secure-mgmt.subnet=${ZONE_SUBNETS['secure-mgmt']} network.secure-mgmt.gateway=${ZONE_GATEWAYS['secure-mgmt']} `
+            : ''
+        }dns.primary=100.100.0.1 \
         dns.fallback=1.0.0.1,8.8.8.8`,
     );
     if (initResult.exitCode !== 0) {