@volter/world-runtime 2.0.0

package/src/reflect.ts

@@ -0,0 +1,440 @@
+import { serveHttp, type HttpServer } from '@volter/world-core';
+import { websocketRelay } from './websocket-relay.ts';
+import { sessionTrustEnv } from './ca-trust.ts';
+// The reflect attachment (docs/guides/route-a-cli-through-the-world.md): attach an UNMODIFIED consumer —
+// any language, any binary, no proxy env, no injector — by answering routed
+// vendor hosts' DNS with the world's own TLS front. The consumer's ordinary
+// `https://api.stripe.com` call roundtrips into the front, which terminates
+// TLS with the session CA's per-host leaf (exactly the redirect-proxy's
+// machinery) and forwards to the twin. Hosts that are not routed resolve real
+// and never touch the world: passthrough is the default.
+//
+// Two small servers, both world-side ("the door belongs to the world"):
+//   • the FRONT — a TCP listener that peeks the TLS ClientHello SNI and pipes
+//     the raw bytes to a per-host leaf-cert https server (the same
+//     one-server-per-host trick redirect-proxy uses for CONNECT, reused here
+//     because Bun does not honor SNICallback).
+//   • the RESOLVER — a UDP DNS server ("the ear", pointed at by the attacher's
+//     environment, e.g. `docker run --dns`): A-answers routed hosts with the
+//     front's address (TTL 0 — a route flip lands on the next lookup),
+//     empty-answers AAAA for routed hosts (else real IPv6 wins), and forwards
+//     everything else to the real upstream untouched.
+//
+// ROUTES are attachment-scoped state (docs/guides/route-a-cli-through-the-world.md): one host per line in
+// the instance's reflect-routes file, re-read live. The world stays ignorant
+// of who is attached; routes only select interception. v1 routes are exact
+// lowercase hostnames (wildcard families like *.atlassian.net: route the
+// specific hosts the consumer uses).
+//
+// Per CLAUDE.md the runtime stays a lifecycle primitive: `reflect` runs in the
+// foreground and supervises nothing; composition (containers, --dns wiring)
+// lives in the cookbook recipe.
+import dgram from 'node:dgram';
+import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
+import { connect as netConnect, createServer as createNetServer, type Server as NetServer, type Socket } from 'node:net';
+import { join } from 'node:path';
+import { activeVendorMap, ensureCa, leafCertFor, noTwinMessage, proxyTargetFor, twinUnreachableMessage } from './redirect-proxy.ts';
+import { instanceDir } from './runtime.ts';
+
+// ---------------------------------------------------------------------------
+// Routes — attachment-scoped, file-backed, re-read live by the resolver.
+// ---------------------------------------------------------------------------
+
+export function reflectRoutesPath(root: string, name: string): string {
+  return join(instanceDir(root, name), 'reflect-routes');
+}
+
+export function readReflectRoutes(path: string): Set<string> {
+  if (!existsSync(path)) return new Set();
+  return new Set(
+    readFileSync(path, 'utf8')
+      .split('\n')
+      .map((line) => line.trim().toLowerCase())
+      .filter((line) => line.length > 0 && !line.startsWith('#')),
+  );
+}
+
+export function writeReflectRoutes(path: string, routes: Set<string>): void {
+  writeFileSync(path, `${[...routes].sort().join('\n')}\n`);
+}
+
+// ---------------------------------------------------------------------------
+// SNI peek — minimal ClientHello parser (records buffered until complete).
+// ---------------------------------------------------------------------------
+
+/** Extract the SNI server name from a buffered TLS ClientHello, `undefined` if
+ * the buffer is not yet a complete handshake record or carries no SNI. */
+export function parseSni(buffer: Buffer): string | undefined {
+  if (buffer.length < 5 || buffer[0] !== 0x16) return undefined; // not a TLS handshake record
+  const recordLength = buffer.readUInt16BE(3);
+  if (buffer.length < 5 + recordLength) return undefined; // incomplete — keep buffering
+  let offset = 5;
+  if (buffer[offset] !== 0x01) return undefined; // not a ClientHello
+  offset += 4; // handshake type + 24-bit length
+  offset += 2 + 32; // client version + random
+  const sessionIdLength = buffer[offset]!;
+  offset += 1 + sessionIdLength;
+  const cipherLength = buffer.readUInt16BE(offset);
+  offset += 2 + cipherLength;
+  const compressionLength = buffer[offset]!;
+  offset += 1 + compressionLength;
+  if (offset + 2 > buffer.length) return undefined;
+  const extensionsEnd = offset + 2 + buffer.readUInt16BE(offset);
+  offset += 2;
+  while (offset + 4 <= extensionsEnd && offset + 4 <= buffer.length) {
+    const extensionType = buffer.readUInt16BE(offset);
+    const extensionLength = buffer.readUInt16BE(offset + 2);
+    offset += 4;
+    if (extensionType === 0x0000) {
+      // server_name: list length (2), entry type (1, 0 = hostname), name length (2), name
+      const nameLength = buffer.readUInt16BE(offset + 3);
+      return buffer.subarray(offset + 5, offset + 5 + nameLength).toString('ascii').toLowerCase();
+    }
+    offset += extensionLength;
+  }
+  return undefined;
+}
+
+// ---------------------------------------------------------------------------
+// The front.
+// ---------------------------------------------------------------------------
+
+export interface ReflectFrontOptions {
+  /** live env source — the world's twin map, exactly like the redirect proxy */
+  envLoader: () => Record<string, string | undefined>;
+  tlsDir: string;
+  host?: string;
+  port?: number;
+  /** additional SNI names to terminate (a served world's ADVERTISED hostname):
+   * the per-host handler already routes by Host header, so env-attached
+   * traffic arriving with SNI=<advertised> and Host=<vendor> lands in the
+   * right twin through the one door (docs/guides/route-a-cli-through-the-world.md, remote worlds). */
+  extraHosts?: string[];
+}
+
+export interface ReflectFrontHandle {
+  host: string;
+  port: number;
+  caCertPath: string;
+  close(): Promise<void>;
+}
+
+export async function startReflectFront(options: ReflectFrontOptions): Promise<ReflectFrontHandle> {
+  const host = options.host ?? '0.0.0.0';
+  const ca = ensureCa(options.tlsDir);
+  const openSockets = new Set<Socket>();
+  const hostServers = new Map<string, HttpServer>();
+  const hostPorts = new Map<string, number>();
+
+  // One TLS server per vendor host (the leaf cert names it), behind the SNI door. Bun's own server: it
+  // terminates TLS, forwards each request to the twin, and relays a websocket upgrade (a bot's gateway,
+  // a streaming API) to the twin's websocket — under Bun, node:http's 'upgrade' socket cannot write back
+  // to the client, so the relay is message-level, both ways.
+  const ensureHostServer = async (vendorHost: string): Promise<number> => {
+    const existing = hostPorts.get(vendorHost);
+    if (existing) return existing;
+    const leaf = leafCertFor(vendorHost, ca);
+    const resolveTwin = (req: Request): { origin: string; path: string; vendor: string } | { refuse: Response } => {
+      const url = new URL(req.url);
+      const hostHeader = (req.headers.get('host') ?? vendorHost).split(':')[0] ?? vendorHost;
+      const env = options.envLoader();
+      const prefixMatch = url.pathname.match(/^\/__vendor\/([a-z0-9-]+)(\/.*)?$/);
+      if (prefixMatch) {
+        const origin = activeVendorMap(env)[prefixMatch[1]!];
+        if (origin === undefined) return { refuse: new Response('no twin for vendor', { status: 502 }) };
+        return { origin, path: `${prefixMatch[2] ?? '/'}${url.search}`, vendor: prefixMatch[1]! };
+      }
+      const twin = proxyTargetFor(hostHeader, env, url.pathname) ?? proxyTargetFor(vendorHost, env, url.pathname);
+      if (!twin) return { refuse: new Response(noTwinMessage(hostHeader, url.pathname), { status: 502, headers: { 'content-type': 'text/plain; charset=utf-8' } }) };
+      return { origin: twin.origin, path: `${url.pathname}${url.search}`, vendor: twin.vendor };
+    };
+    const hostServer = await serveHttp({
+      hostname: '127.0.0.1',
+      port: 0,
+      tls: { key: leaf.key, cert: leaf.cert },
+      idleTimeout: 255,
+      async fetch(req) {
+        const target = resolveTwin(req);
+        if ('refuse' in target) return target.refuse;
+        let origin: URL;
+        try { origin = new URL(target.origin); } catch { return new Response('bad twin origin', { status: 502 }); }
+
+        const headers = new Headers(req.headers);
+        headers.delete('proxy-connection');
+        headers.delete('accept-encoding');
+        // the consumer spoke TLS to the front; a twin that answers with an absolute URL (Slack's Socket Mode link)
+        // gives the scheme the consumer can reach again, as a vendor behind a TLS-terminating edge does
+        headers.set('x-forwarded-proto', 'https');
+        try {
+          const upstream = await fetch(`${origin.origin}${target.path}`, { method: req.method, headers, body: req.body, ...(req.body ? { duplex: 'half' as const } : {}), redirect: 'manual' });
+          const out = new Headers(upstream.headers);
+          out.delete('content-encoding');
+          out.delete('content-length');
+          out.delete('transfer-encoding');
+          out.delete('connection');
+          out.delete('keep-alive');
+          return new Response(upstream.body, { status: upstream.status, headers: out });
+        } catch (error) {
+          let world: string | undefined;
+          try { world = options.envLoader().VOLTER_WORLD; } catch { /* the answer still goes out */ }
+          return new Response(twinUnreachableMessage(target.vendor, origin.origin, error instanceof Error ? error : new Error(String(error)), world), { status: 502 });
+        }
+      },
+      upgrade: websocketRelay(req => {
+        const target = resolveTwin(req);
+        if ('refuse' in target) return undefined;
+        const origin = new URL(target.origin);
+        return `${origin.protocol === 'https:' ? 'wss' : 'ws'}://${origin.host}${target.path}`;
+      }),
+    });
+    const hostPort = Number(hostServer.port);
+    hostServers.set(vendorHost, hostServer);
+    hostPorts.set(vendorHost, hostPort);
+    return hostPort;
+  };
+
+  const server: NetServer = createNetServer((clientSocket: Socket) => {
+    openSockets.add(clientSocket);
+    clientSocket.on('close', () => openSockets.delete(clientSocket));
+    clientSocket.on('error', () => clientSocket.destroy());
+    let buffered = Buffer.alloc(0);
+    const onData = (chunk: Buffer) => {
+      buffered = Buffer.concat([buffered, chunk]);
+      const sni = parseSni(buffered);
+      if (sni === undefined) {
+        if (buffered.length > 65_536) clientSocket.destroy(); // not a ClientHello — give up
+        return;
+      }
+      clientSocket.removeListener('data', onData);
+      const twin = proxyTargetFor(sni, options.envLoader());
+      if (!twin && !(options.extraHosts ?? []).includes(sni)) {
+        // only routed hosts (or the advertised name) are pointed here by the
+        // resolver/manifest; anything else is refused, never blind-forwarded
+        // (the front is a door to twins, not a general proxy)
+        clientSocket.destroy();
+        return;
+      }
+      ensureHostServer(sni)
+        .then((port) => {
+          const upstream: Socket = netConnect(port, '127.0.0.1');
+          openSockets.add(upstream);
+          upstream.on('close', () => openSockets.delete(upstream));
+          upstream.on('error', () => clientSocket.destroy());
+          clientSocket.on('error', () => upstream.destroy());
+          upstream.write(buffered);
+          upstream.pipe(clientSocket);
+          clientSocket.pipe(upstream);
+        })
+        .catch(() => clientSocket.destroy());
+    };
+    clientSocket.on('data', onData);
+  });
+
+  await new Promise<void>((resolveListen, reject) => {
+    server.once('error', reject);
+    server.listen(options.port ?? 0, host, () => resolveListen());
+  });
+  const address = server.address();
+  const port = typeof address === 'object' && address ? address.port : (options.port ?? 0);
+
+  return {
+    host,
+    port,
+    caCertPath: ca.caCert,
+    async close() {
+      for (const socket of openSockets) socket.destroy();
+      await new Promise<void>((resolveClose) => server.close(() => resolveClose()));
+      for (const hostServer of hostServers.values()) hostServer.stop(true);
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// The resolver.
+// ---------------------------------------------------------------------------
+
+export interface ReflectResolverOptions {
+  /** live route set — re-read per query so a route flip lands immediately */
+  routesLoader: () => Set<string>;
+  /** the address routed hosts resolve to (the front's reachable IP) */
+  targetIp: string;
+  upstream?: string;
+  /** upstream port — default 53; tests point at a fake upstream on a high port */
+  upstreamPort?: number;
+  host?: string;
+  port?: number;
+}
+
+export interface ReflectResolverHandle {
+  port: number;
+  close(): Promise<void>;
+}
+
+function dnsQuestion(message: Buffer): { name: string; qtype: number; questionEnd: number } | undefined {
+  if (message.length < 17) return undefined;
+  const labels: string[] = [];
+  let offset = 12;
+  while (offset < message.length) {
+    const length = message[offset]!;
+    if (length === 0) {
+      offset += 1;
+      break;
+    }
+    labels.push(message.subarray(offset + 1, offset + 1 + length).toString('ascii'));
+    offset += 1 + length;
+  }
+  if (offset + 4 > message.length) return undefined;
+  return { name: labels.join('.').toLowerCase(), qtype: message.readUInt16BE(offset), questionEnd: offset + 4 };
+}
+
+function dnsHeader(query: Buffer, answerCount: number): Buffer {
+  const header = Buffer.alloc(12);
+  query.copy(header, 0, 0, 2); // ID
+  header.writeUInt16BE(0x8180, 2); // response, RD|RA, NOERROR
+  header.writeUInt16BE(1, 4); // QDCOUNT
+  header.writeUInt16BE(answerCount, 6); // ANCOUNT
+  return header;
+}
+
+function dnsAnswerA(query: Buffer, questionEnd: number, ip: string): Buffer {
+  const answer = Buffer.alloc(16);
+  answer.writeUInt16BE(0xc00c, 0); // name: pointer to the question
+  answer.writeUInt16BE(1, 2); // TYPE A
+  answer.writeUInt16BE(1, 4); // CLASS IN
+  answer.writeUInt32BE(0, 6); // TTL 0 — activation flips on the next lookup
+  answer.writeUInt16BE(4, 10);
+  const parts = ip.split('.').map(Number);
+  for (let index = 0; index < 4; index++) answer[12 + index] = parts[index]!;
+  return Buffer.concat([dnsHeader(query, 1), query.subarray(12, questionEnd), answer]);
+}
+
+function dnsAnswerEmpty(query: Buffer, questionEnd: number): Buffer {
+  return Buffer.concat([dnsHeader(query, 0), query.subarray(12, questionEnd)]);
+}
+
+export async function startReflectResolver(options: ReflectResolverOptions): Promise<ReflectResolverHandle> {
+  const upstream = options.upstream ?? '8.8.8.8';
+  const server = dgram.createSocket('udp4');
+  server.on('message', (message, rinfo) => {
+    const question = dnsQuestion(message);
+    if (!question) return;
+    if (options.routesLoader().has(question.name)) {
+      const reply =
+        question.qtype === 1
+          ? dnsAnswerA(message, question.questionEnd, options.targetIp)
+          : dnsAnswerEmpty(message, question.questionEnd); // AAAA etc.: empty, so the reflected A wins
+      server.send(reply, rinfo.port, rinfo.address);
+      return;
+    }
+    // passthrough: relay to the real upstream untouched
+    const forwarder = dgram.createSocket('udp4');
+    const timer = setTimeout(() => forwarder.close(), 3_000);
+    forwarder.on('message', (response) => {
+      clearTimeout(timer);
+      server.send(response, rinfo.port, rinfo.address);
+      forwarder.close();
+    });
+    forwarder.send(message, options.upstreamPort ?? 53, upstream);
+  });
+  await new Promise<void>((resolveListen, reject) => {
+    server.once('error', reject);
+    server.bind(options.port ?? 0, options.host ?? '0.0.0.0', () => resolveListen());
+  });
+  return {
+    port: server.address().port,
+    async close() {
+      await new Promise<void>((resolveClose) => server.close(() => resolveClose()));
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// The reflect manifest — what a running `volter-world reflect` listens on, published for attachers
+// (`attach --via reflect`). Written by the verb on start, removed on exit; absent = no front running.
+// ---------------------------------------------------------------------------
+export interface ReflectManifest {
+  /** the address routed hosts resolve to — where attachers reach the front (the verb's --target-ip) */
+  targetIp: string;
+  /** the address attachers reach the resolver at (--resolver-ip; default the target): on some Docker hosts
+   * the gateway address answers DNS itself (colima's VM does), so the resolver needs another host address */
+  resolverIp: string;
+  frontPort: number;
+  resolverPort: number;
+  caCertPath: string;
+}
+
+export function reflectManifestPath(root: string, name: string): string {
+  return join(instanceDir(root, name), 'reflect.json');
+}
+
+export function writeReflectManifest(root: string, name: string, manifest: ReflectManifest): void {
+  writeFileSync(reflectManifestPath(root, name), `${JSON.stringify(manifest, null, 2)}\n`);
+}
+
+export function readReflectManifest(root: string, name: string): ReflectManifest | undefined {
+  const path = reflectManifestPath(root, name);
+  if (!existsSync(path)) return undefined;
+  const parsed = JSON.parse(readFileSync(path, 'utf8')) as Partial<ReflectManifest>;
+  if (typeof parsed.targetIp !== 'string' || typeof parsed.frontPort !== 'number' || typeof parsed.resolverPort !== 'number' || typeof parsed.caCertPath !== 'string') return undefined;
+  return { ...parsed, resolverIp: typeof parsed.resolverIp === 'string' ? parsed.resolverIp : parsed.targetIp } as ReflectManifest;
+}
+
+export function clearReflectManifest(root: string, name: string): void {
+  const path = reflectManifestPath(root, name);
+  if (existsSync(path)) unlinkSync(path);
+}
+
+/** Where the CA is mounted inside an attached container, and the env that makes its clients trust it. */
+export const ATTACHED_CA_PATH = '/etc/volter-world/ca.pem';
+export const CA_TRUST_ENV = ['SSL_CERT_FILE', 'REQUESTS_CA_BUNDLE', 'CURL_CA_BUNDLE', 'NODE_EXTRA_CA_CERTS', 'GIT_SSL_CAINFO'] as const;
+
+/**
+ * A docker compose override that attaches every named service through reflect: its DNS is the resolver
+ * (so routed vendor hosts resolve to the front), `host.docker.internal` stays resolvable (a service-level
+ * `dns` bypasses Docker's own resolver, which is what served that name), the session CA is mounted and
+ * named by the trust variables the common clients read. Requires the resolver on port 53: Docker takes a
+ * DNS address, never a port. Compose merges this after the consumer's own files, touching nothing else.
+ */
+export function composeOverrideForReflect(manifest: ReflectManifest, services: string[], worldName: string): string {
+  if (manifest.resolverPort !== 53) throw new Error(`reflect attach: containers take a DNS address, not a port — run the resolver on :53 (it is on :${manifest.resolverPort})`);
+  if (manifest.frontPort !== 443) throw new Error(`reflect attach: a consumer reaches https on :443 — run the front on :443 (it is on :${manifest.frontPort})`);
+  const trust = sessionTrustEnv(manifest.caCertPath);
+  const lines = ['# generated by `volter-world attach --via reflect` — never edit; merged after the consumer\'s own files', 'services:'];
+  for (const service of services) {
+    lines.push(`  ${service}:`);
+    lines.push('    dns:', `      - ${manifest.resolverIp}`);
+    lines.push('    extra_hosts:', '      - host.docker.internal:host-gateway');
+    lines.push('    environment:', `      - VOLTER_WORLD=${worldName}`);
+    for (const key of CA_TRUST_ENV) lines.push(`      - ${key}=${ATTACHED_CA_PATH}.${key}`);
+    lines.push('    volumes:', ...CA_TRUST_ENV.map((key) => `      - ${trust[key]}:${ATTACHED_CA_PATH}.${key}:ro`));
+  }
+  return `${lines.join('\n')}\n`;
+}
+
+// docker compose's global options that take a value — the override's -f must land after the consumer's own
+// files and before the subcommand, so these are stepped over to find it.
+const COMPOSE_VALUED_OPTIONS = new Set(['-f', '--file', '-p', '--project-name', '--profile', '--env-file', '--project-directory', '--progress', '--parallel', '--ansi']);
+
+/** Split `docker [docker flags] compose [compose flags] <subcommand…>`; undefined when it is not a compose command. */
+export function splitDockerComposeArgs(command: string[]): { head: string[]; composeFlags: string[]; tail: string[] } | undefined {
+  if (command[0] !== 'docker') return undefined;
+  const composeAt = command.indexOf('compose');
+  if (composeAt < 0) return undefined;
+  const head = command.slice(0, composeAt + 1);
+  const composeFlags: string[] = [];
+  let i = composeAt + 1;
+  while (i < command.length) {
+    const arg = command[i]!;
+    if (!arg.startsWith('-')) break;
+    if (COMPOSE_VALUED_OPTIONS.has(arg)) { composeFlags.push(arg, command[i + 1] ?? ''); i += 2; continue; }
+    composeFlags.push(arg); i += 1; // --flag=value or a boolean flag
+  }
+  return { head, composeFlags, tail: command.slice(i) };
+}
+
+/** The same compose command with the override merged last. */
+export function dockerComposeWithOverride(command: string[], overridePath: string): string[] {
+  const parts = splitDockerComposeArgs(command);
+  if (!parts) return command;
+  return [...parts.head, ...parts.composeFlags, '-f', overridePath, ...parts.tail];
+}

package/src/resources.ts

@@ -0,0 +1,22 @@
+// Actual host capacity and lifecycle inventory. Declarations never reduce reported free space.
+import { statfsSync } from 'node:fs';
+import { totalmem } from 'node:os';
+import { resolve } from 'node:path';
+import { inspectWorldLifecycles } from './lifecycle-record.ts';
+import { storagePoolResolver } from './storage-capacity.ts';
+
+export function worldResourceReport(root = process.cwd()) {
+  const inventory = inspectWorldLifecycles();
+  const measuredAt = new Date().toISOString();
+  const path = resolve(root);
+  let filesystem: { path: string; availableBytes: number | null; totalBytes: number | null; pool: string | null; source: string; error?: string };
+  try {
+    const fs = statfsSync(path);
+    filesystem = { path, availableBytes: Number(fs.bavail) * Number(fs.bsize), totalBytes: Number(fs.blocks) * Number(fs.bsize),
+      pool: storagePoolResolver()(path) ?? null, source: 'statfs' };
+  } catch (error) {
+    filesystem = { path, availableBytes: null, totalBytes: null, pool: null, source: 'statfs', error: (error as NodeJS.ErrnoException).code ?? 'unavailable' };
+  }
+  return { schemaVersion: 2, accounting: 'observed' as const, measuredAt,
+    host: { physicalMemoryBytes: totalmem(), measuredMemoryUsage: false }, filesystem, ...inventory };
+}