@serfab/cadre-host 0.13.0 → 1.1.0

package/src/__tests__/orchestrator.test.ts

@@ -14,9 +14,8 @@ import { dirname, join, resolve as resolvePath } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 
 import { HostProcessOrchestrator } from '../orchestrator/host-process-orchestrator.js';
-import { allocateNodePorts, PortAllocator } from '../orchestrator/port-allocator.js';
 import { rotateOnDisk } from '../orchestrator/log-rotator.js';
-import { decodeDockerId, encodeDockerId } from '../orchestrator/types.js';
+import { decodeDockerId, encodeDockerId, type NodePorts } from '../orchestrator/types.js';
 import { StateStore } from '../orchestrator/state-store.js';
 import { isPidAlive } from '../orchestrator/pid-liveness.js';
 import { loadIdentity } from '../installer/identity.js';
@@ -165,7 +164,7 @@ describe('HostProcessOrchestrator re-spawn of the same containerId', () => {
   // The donated-node respawn path calls createContainer again with the id of a
   // container the host still holds a (now dead) handle for. Handles are keyed by
   // the per-spawn dockerId, so without an explicit drop the old one would linger
-  // forever and its four ports would never come back.
+  // forever and its ports would never come back.
   it('drops the stale handle, releases its ports, and keeps the identity', async () => {
     const orch = makeOrchestrator();
     const rootDir = (orch as unknown as { rootDir: string }).rootDir;
@@ -197,6 +196,58 @@ describe('HostProcessOrchestrator re-spawn of the same containerId', () => {
     // same node rather than a new one the borrower's cadre has never approved.
     expect(loadIdentity(identityPath).peerId).toBe(peerBefore);
   });
+
+  // A requester that dials the node itself (a phone) knows it only by the address it
+  // was given, so a re-spawn must come back on the SAME ports even when the allocator
+  // has lower ones free. The test above cannot tell reuse from lowest-free: there the
+  // old ports are also the lowest free ones.
+  it('comes back on its own ports even when lower ones are free', async () => {
+    const orch = makeOrchestrator();
+
+    const lower = await orch.createContainer(makeRequest('c0'));
+    const first = await orch.createContainer(makeRequest('c1'));
+    await waitFor(() => orch.isRunning(first.dockerId));
+    const portsBefore = orch.getNode('c1')!.ports;
+
+    await orch.removeContainer(lower.dockerId);
+    const { pid } = decodeDockerId(first.dockerId);
+    process.kill(pid, 'SIGKILL');
+    await waitFor(() => !isPidAlive(pid));
+
+    await orch.createContainer(makeRequest('c1'));
+    expect(orch.getNode('c1')!.ports).toEqual(portsBefore);
+  });
+
+  // A handle written to state.json before the WebSocket port existed has no `ws`.
+  // Rehydrating it must hold the ports it does have, and its re-spawn must keep those
+  // and gain a WebSocket port rather than trip over the missing key.
+  it('re-spawns a handle persisted without a ws port on its old ports plus a fresh ws', async () => {
+    const rootDir = join(tmpRoot, 'legacy');
+    const legacyPorts = { health: 18010, metrics: 18011, p2p: 18012, admin: 18013 };
+    new StateStore(rootDir).save({
+      version: 1,
+      handles: [{
+        containerId: 'c1',
+        dockerId: encodeDockerId(999999, 'deadbeef'),
+        pid: 999999,
+        startupToken: 'deadbeef',
+        workdir: join(rootDir, 'c1'),
+        ports: legacyPorts as NodePorts,
+        spawnedAt: new Date(0).toISOString(),
+        partyId: 'party-c1',
+        profile: 'transaction',
+      }],
+    });
+    const orch = makeOrchestrator({ rootDir });
+    await orch.init();
+
+    await orch.createContainer(makeRequest('c1'));
+
+    const ports = orch.getNode('c1')!.ports;
+    expect(ports).toMatchObject(legacyPorts);
+    // The lowest free port: nothing below 18010 was held, `undefined` included.
+    expect(ports.ws).toBe(18000);
+  });
 });
 
 /**
@@ -261,7 +312,7 @@ describe('HostProcessOrchestrator failed launch', () => {
     rmSync(storage, { force: true });
     const second = await orch.createContainer(makeRequest('c1'));
 
-    // The restored handle held its four ports across the failure, and the
+    // The restored handle held its ports across the failure, and the
     // allocator hands back the lowest free port — so identical ports prove the
     // failed attempt neither leaked nor stole them.
     expect(second.p2pPort).toBe(first.p2pPort);
@@ -279,16 +330,16 @@ describe('HostProcessOrchestrator failed launch', () => {
     const rootDir = (orch as unknown as { rootDir: string }).rootDir;
 
     await orch.createContainer(makeRequest('c1'));
-    expect(orch.getNode('c1')!.ports).toEqual({ health: 12000, metrics: 12001, p2p: 12002, admin: 12003 });
+    expect(orch.getNode('c1')!.ports).toEqual({ health: 12000, metrics: 12001, p2p: 12002, admin: 12003, ws: 12004 });
 
-    // No prior handle to restore here — the four ports are simply released.
+    // No prior handle to restore here — the five ports are simply released.
     mkdirSync(join(rootDir, 'c2'), { recursive: true });
     writeFileSync(join(rootDir, 'c2', 'storage'), 'not-a-directory', 'utf8');
     await expect(orch.createContainer(makeRequest('c2'))).rejects.toThrow();
 
     await orch.createContainer(makeRequest('c3'));
-    // With the leak these would start at 12008.
-    expect(orch.getNode('c3')!.ports).toEqual({ health: 12004, metrics: 12005, p2p: 12006, admin: 12007 });
+    // With the leak these would start at 12010.
+    expect(orch.getNode('c3')!.ports).toEqual({ health: 12005, metrics: 12006, p2p: 12007, admin: 12008, ws: 12009 });
   });
 
   it('leaves the owner node addressable on its original ports', async () => {
@@ -348,7 +399,7 @@ describe('HostProcessOrchestrator failed launch', () => {
     // persist must not reach the unwind that would delete it.
     expect(existsSync(join(rootDir, 'c1', 'identity.key'))).toBe(true);
 
-    // Its ports are still held: a fresh container gets four different ones.
+    // Its ports are still held: a fresh container gets different ones.
     const other = await orch.createContainer(makeRequest('c2'));
     expect(other.p2pPort).not.toBe(second.p2pPort);
     expect(orch.getNode('c2')!.ports).not.toEqual(orch.getNode('c1')!.ports);
@@ -361,8 +412,8 @@ describe('HostProcessOrchestrator failed launch', () => {
   // unwind then correctly declines to delete it. Exhausting the port range
   // fails the very next step instead, with the workdir genuinely fresh.
   it('discards the workdir a failed first spawn created', async () => {
-    // Exactly four ports — enough for one node and no more.
-    const orch = makeOrchestrator({ portRange: { start: 13000, end: 13003 } });
+    // Exactly five ports — enough for one node and no more.
+    const orch = makeOrchestrator({ portRange: { start: 13000, end: 13004 } });
     const rootDir = (orch as unknown as { rootDir: string }).rootDir;
 
     await orch.createContainer(makeRequest('c1'));
@@ -494,7 +545,7 @@ describe('HostProcessOrchestrator.reclaimWorkdir', () => {
 });
 
 describe('HostProcessOrchestrator.isRunning', () => {
-  it('is true after spawn, false after kill, false on tampered token', async () => {
+  it('is true after spawn and false after kill for a child this orchestrator spawned', async () => {
     const orch = makeOrchestrator();
     const { dockerId } = await orch.createContainer(makeRequest('c1'));
     const { pid } = decodeDockerId(dockerId);
@@ -502,20 +553,137 @@ describe('HostProcessOrchestrator.isRunning', () => {
     await waitFor(() => orch.isRunning(dockerId));
     expect(await orch.isRunning(dockerId)).toBe(true);
 
-    // Tamper with the token file
+    // The token file is not consulted for a child this process spawned — its
+    // ChildProcess is exact, and the file has a start-up window it does not.
     const workdir = join((orch as unknown as { rootDir: string }).rootDir, 'c1');
     writeFileSync(join(workdir, '.startup-token'), 'tampered', 'utf8');
-    expect(await orch.isRunning(dockerId)).toBe(false);
-
-    // Restore token, kill the process
-    const persisted = new StateStore((orch as unknown as { rootDir: string }).rootDir).load();
-    writeFileSync(join(workdir, '.startup-token'), persisted.handles[0]!.startupToken, 'utf8');
     expect(await orch.isRunning(dockerId)).toBe(true);
 
     process.kill(pid, 'SIGKILL');
     await waitFor(() => !isPidAlive(pid));
     expect(await orch.isRunning(dockerId)).toBe(false);
   });
+
+  it('checks a re-attached handle by pid plus startup token', async () => {
+    const rootDir = join(tmpRoot, 'reattach');
+    const a = makeOrchestrator({ rootDir });
+    const { dockerId } = await a.createContainer(makeRequest('c1'));
+    const { pid } = decodeDockerId(dockerId);
+    await waitFor(() => existsSync(join(rootDir, 'c1', '.startup-token')));
+
+    // A new orchestrator has no ChildProcess for it — the host-restart case.
+    const b = makeOrchestrator({ rootDir });
+    await b.init();
+    expect(await b.isRunning(dockerId)).toBe(true);
+
+    // A live pid whose token does not match may be an unrelated process that
+    // inherited the pid, so it does not count as ours.
+    const tokenPath = join(rootDir, 'c1', '.startup-token');
+    const token = readFileSync(tokenPath, 'utf8');
+    writeFileSync(tokenPath, 'tampered', 'utf8');
+    expect(await b.isRunning(dockerId)).toBe(false);
+
+    writeFileSync(tokenPath, token, 'utf8');
+    expect(await b.isRunning(dockerId)).toBe(true);
+
+    process.kill(pid, 'SIGKILL');
+    await waitFor(() => !isPidAlive(pid));
+    expect(await b.isRunning(dockerId)).toBe(false);
+  });
+});
+
+/**
+ * A child that binds its health port at once but writes its startup token only
+ * after `TOKEN_DELAY_MS` — the shape of a real `cadre-cli start` built before the
+ * token write moved to the front, and of any child that is slow to reach it. Dies
+ * on a port clash the way `cadre-cli start` does.
+ *
+ * The incident this guards against: the donation supervisor asked `isRunning`
+ * during that window, got `false`, and re-spawned the container. The re-spawn
+ * released and reused the same ports, rotated the seed token, and returned
+ * success; the second child then died on the port the first still held, leaving
+ * the host's records naming a dead child and the live one refusing every seed.
+ */
+describe('HostProcessOrchestrator re-spawn over a child that is still starting', () => {
+  const TOKEN_DELAY_MS = 3000;
+  const SLOW_TOKEN_CHILD = `
+import fs from 'node:fs';
+import net from 'node:net';
+const args = process.argv.slice(2);
+const tokenPath = args[args.indexOf('--startup-token-file') + 1];
+const server = net.createServer();
+server.on('error', (err) => { console.error('Failed to start cadre node: ' + err.message); process.exit(1); });
+server.listen(Number(process.env.CADRE_HEALTH_PORT), '0.0.0.0', () => {
+  console.log('listening');
+  setTimeout(() => fs.writeFileSync(tokenPath, process.env.CADRE_STARTUP_TOKEN), ${TOKEN_DELAY_MS});
+});
+process.on('SIGTERM', () => process.exit(0));
+setInterval(() => {}, 1 << 30);
+`;
+  const TEST_TIMEOUT_MS = 20_000;
+
+  let slowChildPath: string;
+  beforeEach(() => {
+    slowChildPath = join(tmpRoot, 'slow-token-child.mjs');
+    writeFileSync(slowChildPath, SLOW_TOKEN_CHILD, 'utf8');
+  });
+
+  it('reports the child running before its token is written, and refuses to launch over it', async () => {
+    const orch = makeOrchestrator({ spawn: { entrypoint: slowChildPath } });
+    const rootDir = (orch as unknown as { rootDir: string }).rootDir;
+    const tokenPath = join(rootDir, 'c1', '.startup-token');
+
+    const first = await orch.createContainer(makeRequest('c1'));
+    await waitFor(async () => (await orch.getLogs(first.dockerId)).includes('listening'));
+    const portsBefore = orch.getNode('c1')!.ports;
+
+    // Inside the start-up window: port bound, token not yet written.
+    expect(existsSync(tokenPath)).toBe(false);
+    expect(await orch.isRunning(first.dockerId)).toBe(true);
+
+    // Exactly what `DonationService.respawn` would do next.
+    await expect(orch.createContainer(makeRequest('c1'))).rejects.toThrow(/c1 is still running/);
+
+    // Nothing was dropped, released or rotated.
+    expect(orch.listNodes()).toHaveLength(1);
+    expect(orch.getNode('c1')?.dockerId).toBe(first.dockerId);
+    expect(orch.getNode('c1')!.ports).toEqual(portsBefore);
+    const other = await orch.createContainer(makeRequest('c2'));
+    expect(other.p2pPort).not.toBe(first.p2pPort);
+    expect(other.healthEndpoint).not.toBe(first.healthEndpoint);
+
+    // The first child finishes starting undisturbed, with the token it was given.
+    const { pid, token } = decodeDockerId(first.dockerId);
+    await waitFor(() => existsSync(tokenPath), TOKEN_DELAY_MS + 5000);
+    expect(readFileSync(tokenPath, 'utf8')).toBe(token);
+    expect(isPidAlive(pid)).toBe(true);
+    expect(await orch.getLogs(first.dockerId)).not.toContain('EADDRINUSE');
+  }, TEST_TIMEOUT_MS);
+
+  it('refuses to launch over a re-attached child whose ports are still bound', async () => {
+    const rootDir = join(tmpRoot, 'reattach-starting');
+    const a = makeOrchestrator({ rootDir, spawn: { entrypoint: slowChildPath } });
+    const first = await a.createContainer(makeRequest('c1'));
+    await waitFor(async () => (await a.getLogs(first.dockerId)).includes('listening'));
+
+    // A host restart while the child is mid-start: no ChildProcess, no token yet.
+    const b = makeOrchestrator({ rootDir, spawn: { entrypoint: slowChildPath } });
+    await b.init();
+    expect(await b.isRunning(first.dockerId)).toBe(false);
+
+    await expect(b.createContainer(makeRequest('c1')))
+      .rejects.toThrow(/c1 may still be running: port \d+ on 0\.0\.0\.0 is unavailable/);
+
+    // Nothing dropped: the record's dockerId still resolves on its ports, and
+    // the live child was not killed on unverified evidence.
+    expect(b.listNodes()).toHaveLength(1);
+    expect(b.getNode('c1')?.dockerId).toBe(first.dockerId);
+    const { pid } = decodeDockerId(first.dockerId);
+    expect(isPidAlive(pid)).toBe(true);
+
+    // Once it has written its token it is recognised as running.
+    await waitFor(() => b.isRunning(first.dockerId), TOKEN_DELAY_MS + 5000);
+  }, TEST_TIMEOUT_MS);
 });
 
 describe('HostProcessOrchestrator.stopContainer', () => {
@@ -704,73 +872,16 @@ describe('rotateOnDisk', () => {
   });
 });
 
-describe('PortAllocator', () => {
-  it('allocates sequentially, releases, and reuses', () => {
-    const a = new PortAllocator(100, 102);
-    expect(a.allocate()).toBe(100);
-    expect(a.allocate()).toBe(101);
-    expect(a.allocate()).toBe(102);
-    expect(() => a.allocate()).toThrow(/No available ports/);
-    a.release(101);
-    expect(a.allocate()).toBe(101);
-  });
-
-  it('markUsed reserves ports without allocating', () => {
-    const a = new PortAllocator(100, 102);
-    a.markUsed(101);
-    expect(a.allocate()).toBe(100);
-    expect(a.allocate()).toBe(102);
-    expect(() => a.allocate()).toThrow();
-  });
-});
-
-describe('allocateNodePorts', () => {
-  it('allocates the four ports in a fixed order', () => {
-    const a = new PortAllocator(10000, 10010);
-    expect(allocateNodePorts(a)).toEqual({ health: 10000, metrics: 10001, p2p: 10002, admin: 10003 });
-  });
-
-  // A partial set left reserved would leak from a bounded range on every failed
-  // provision — the whole reason this helper exists rather than four allocates.
-  it('is all-or-nothing: an exhausted range releases everything it took', () => {
-    const a = new PortAllocator(13000, 13002);
-    expect(() => allocateNodePorts(a)).toThrow(/No available ports/);
-    expect(a.has(13000)).toBe(false);
-    expect(a.has(13001)).toBe(false);
-    expect(a.has(13002)).toBe(false);
-  });
-
-  // The owner node's p2p port is pinned by the NAT mapping, not allocated.
-  it('honours an override and reserves it', () => {
-    const a = new PortAllocator(10000, 10010);
-    expect(allocateNodePorts(a, { p2p: 10005 })).toEqual({
-      health: 10000,
-      metrics: 10001,
-      p2p: 10005,
-      admin: 10002,
-    });
-    // Reserved, so no later allocation can hand it out again.
-    expect(a.has(10005)).toBe(true);
-  });
-
-  // An out-of-range override is the production case for the owner node: its
-  // libp2p port is chosen by NAT config, not by the allocator, so reserving it
-  // is a documented no-op rather than an error.
-  it('accepts an override outside the managed range without reserving it', () => {
-    const a = new PortAllocator(10000, 10010);
-    expect(allocateNodePorts(a, { p2p: 40000 })).toMatchObject({ p2p: 40000, health: 10000 });
-    expect(a.has(40000)).toBe(false);
-  });
-});
-
 describe('Restart recovery (init)', () => {
   it('re-attaches to surviving children and reports dead ones', async () => {
     const rootDir = join(tmpRoot, 'restart');
     const a = makeOrchestrator({ rootDir });
     const r1 = await a.createContainer(makeRequest('c1'));
     const r2 = await a.createContainer(makeRequest('c2'));
-    await waitFor(() => a.isRunning(r1.dockerId));
-    await waitFor(() => a.isRunning(r2.dockerId));
+    // Not `a.isRunning`: that answers from the ChildProcess at once, but `b`
+    // below can only recognise c1 once c1 has written its startup token.
+    await waitFor(() => existsSync(join(rootDir, 'c1', '.startup-token')));
+    await waitFor(() => existsSync(join(rootDir, 'c2', '.startup-token')));
 
     // Kill c2 externally
     const { pid: pid2 } = decodeDockerId(r2.dockerId);

package/src/bin/host.ts

@@ -102,6 +102,7 @@ program
   .option('--own-cadre', 'Also run the host\'s own personal cadre here (founder persona; default: donor-only)')
   .option('--system', 'System-wide install (not yet supported in v1)')
   .option('--node-path <path>', 'Override the node binary embedded in the service unit')
+  .option('--no-service', 'Write the data dir only; register no OS service (run the host with `cadre-host start`)')
   .action(async (opts: {
     nonInteractive?: boolean;
     dataDir?: string;
@@ -113,6 +114,7 @@ program
     ownCadre?: boolean;
     system?: boolean;
     nodePath?: string;
+    service?: boolean;
   }) => {
     const installer = new Installer();
     try {
@@ -127,11 +129,17 @@ program
         ownCadre: opts.ownCadre === true,
         system: Boolean(opts.system),
         ...(opts.nodePath ? { nodePath: opts.nodePath } : {}),
+        noService: opts.service === false,
       });
       console.log(`cadre-host installed.`);
       console.log(`  Data dir:     ${result.dataDir}`);
       console.log(`  UI:           ${result.uiUrl}`);
-      console.log(`  Service:      ${result.serviceName}`);
+      if (result.serviceName) {
+        console.log(`  Service:      ${result.serviceName}`);
+      } else {
+        console.log('  Service:      not registered (--no-service)');
+        console.log(`  Run the host: cadre-host start --data-dir "${result.dataDir}"`);
+      }
       if (result.enrollmentInvite) {
         printEnrollmentInvite(result.enrollmentInvite);
       }