@tonalisk-labs/kit 1.0.0 → 1.0.6

package/testing/src/stack/browsers.ts

@@ -4,19 +4,38 @@ import { availableParallelism } from 'node:os';
 import path from 'node:path';
 
 import { isObject } from '../unknown.ts';
-import { container } from './containers.ts';
+import { container, exec, networkOf } from './containers.ts';
 import type { Service } from './process.ts';
 import type { ContainerSpec } from './runtime.ts';
+import { type Socks, startSocks } from './socks.ts';
 import { LOCAL_CA_DIR, localCaCertificate } from './tls.ts';
 
+/** The engines the image serves, one browser each: the visual projects' Chromium, and WebKit for
+ *  the rows macOS can't run it on. */
+const ENGINES = ['chromium', 'webkit'] as const;
+
+type Engine = (typeof ENGINES)[number];
+
+/** Each engine's browser on a port of its own, at a path named for it. */
+const PORTS: Readonly<Record<Engine, number>> = { chromium: 3000, webkit: 3001 };
+
+/** Where each engine's browser is served, by engine. */
+type Endpoints = Readonly<Record<Engine, string>>;
+
 export interface BrowserServer extends Service {
-  readonly wsEndpoint: string;
+  readonly endpoints: Endpoints;
 }
 
-const PORT = 3000;
+/** The endpoint of the image's browser for `engine` in `given`, the stack host's variable; none
+ *  where the variable is unset or names no such engine. */
+export const endpointOf = (engine: string, given?: string): string | undefined => {
+  const parsed: unknown = given === undefined ? undefined : JSON.parse(given);
+  const endpoint = isObject(parsed) ? parsed[engine] : undefined;
+  return typeof endpoint === 'string' ? endpoint : undefined;
+};
 
-/** Browser contexts the image serves at once, a seat each: 4 in one browser peak at 2,272 MiB, 4
- *  browsers of one each at 2,947, and neither stalls nor is killed. */
+/** Browser contexts the image serves at once, a seat each, taken by a test for its own time: each
+ *  engine's one browser is shared by every worker, so a connection costs nothing. */
 export const BROWSER_SEATS = 4;
 
 /** A test's devices against the `declared` contexts it may have open, its own included: each is
@@ -40,8 +59,8 @@ export const deviceBudget = (declared: number): (() => () => void) => {
   };
 };
 
-/** A browser costs about 225 MiB and a context about 460; the worst full pool, 4 browsers, peaked
- *  at 2,947 MiB of these 3,328 with no stall: a VM keeps what it touched. */
+/** A browser costs about 225 MiB and a context about 460, so the two browsers and a full pool's
+ *  contexts come to about 2,290 MiB of these 3,328: a VM keeps what it touched. */
 const MACHINE = { memory: 3328, cpus: availableParallelism() } as const;
 
 /** The playwright-core the tests run, mounted into the image of the same release. */
@@ -63,11 +82,60 @@ const LOG_FILE = '/tmp/browsers.log';
  *  server's Node makes a context's own requests, as `route.fetch` and `context.request` do. */
 const CA_FILE = '/usr/local/share/ca-certificates/tonalisk-kit-edge.crt';
 
-/** Takes the local CA's certificate into the image's store, then serves. */
+/** The pipe the host writes the proxy's address to, once it listens, and the IPv6 address the
+ *  container's network gives it: only then may the browsers launch, as each takes its proxy at its
+ *  launch. */
+const PROXY_PIPE = '/tmp/proxy';
+
+/** Launches each engine's browser behind the host's proxy, serving it to every worker. A browser
+ *  that crashes fails the tests it was serving, as one per worker did, and is launched again at
+ *  the same address for the workers that follow. On the container's stop, Playwright closes each
+ *  browser and keeps the process; it exits once both are closed, launching none again. */
+const SERVE = `
+const playwright = require('/playwright-core');
+const engines = ${JSON.stringify(Object.entries(PORTS))};
+const proxy = { server: process.env.PROXY };
+const state = { stopping: false, open: engines.length };
+// A browser launched before its network has given the container its IPv6 address hears the
+// address arrive as a network change, and Chromium fails the navigations it has open
+// (ERR_NETWORK_CHANGED): the browsers launch once it's there, or after 30 s without it.
+const settled = async (left = 300) => {
+  const wanted = process.env.SETTLED;
+  const addresses = Object.values(require('node:os').networkInterfaces()).flat();
+  if (!wanted || left === 0 || addresses.some((one) => one.address === wanted)) return;
+  await new Promise((resolve) => setTimeout(resolve, 100));
+  await settled(left - 1);
+};
+process.once('SIGTERM', () => { state.stopping = true; });
+const failed = (error) => { console.error(error); process.exit(1); };
+const serve = async ([engine, port]) => {
+  const server = await playwright[engine].launchServer({ port, host: '0.0.0.0', wsPath: '/' + engine, proxy });
+  server.once('close', () => {
+    if (state.stopping) {
+      state.open -= 1;
+      if (state.open === 0) process.exit(0);
+      return;
+    }
+    console.log(engine + ' closed, launched again');
+    serve([engine, port]).catch(failed);
+  });
+  console.log(engine + ' serves ' + server.wsEndpoint());
+};
+(async () => {
+  await settled();
+  for (const entry of engines) await serve(entry);
+})().catch(failed);
+`;
+
+/** Takes the local CA's certificate into the image's store, waits for the proxy's address, then
+ *  serves. A pipe the host wrote first is a plain file, which reads as well. */
 const TRUSTED_SERVER = [
   String.raw`printf "%s\n" "$EDGE_CA" > ${CA_FILE}`,
   'update-ca-certificates > /dev/null',
-  `exec node /playwright-core/cli.js run-server --port ${String(PORT)} --host 0.0.0.0 > ${LOG_FILE} 2>&1`,
+  `{ mkfifo ${PROXY_PIPE} 2> /dev/null || true; }`,
+  `{ read -r PROXY && read -r SETTLED; } < ${PROXY_PIPE}`,
+  'export PROXY SETTLED',
+  `exec node -e "$1" > ${LOG_FILE} 2>&1`,
 ].join(' && ');
 
 /** The image serving `core`'s browsers, trusting the CA in `ca`: a certificate passed in its
@@ -77,17 +145,40 @@ export const browserServerSpec = (
   ca: string,
 ): ContainerSpec => ({
   image: `mcr.microsoft.com/playwright:v${core.version}-noble`,
-  ports: [PORT],
-  ready: ['Listening on'],
+  ports: Object.values(PORTS),
+  ready: ENGINES.map((engine) => `${engine} serves`),
   logFile: LOG_FILE,
   env: { EDGE_CA: ca, NODE_EXTRA_CA_CERTS: CA_FILE },
   init: true,
   sharesMemory: true,
   machine: MACHINE,
   files: [core.root, '/playwright-core'],
-  command: ['sh', '-c', TRUSTED_SERVER],
+  command: ['sh', '-c', TRUSTED_SERVER, 'sh', SERVE],
 });
 
+/** Starts the host's proxy for the container `id`, on the host's address on its network and for
+ *  its address alone, and gives the browsers its address. */
+const proxyFor = async (id: string, signal: AbortSignal): Promise<Socks> => {
+  const { own, host, settled = '' } = await networkOf(id);
+  const socks = await startSocks({ host, peer: own });
+  try {
+    const address = `socks5://${host}:${String(socks.port)}`;
+    const told = [
+      'sh',
+      '-c',
+      `printf "%s\\n%s\\n" "$1" "$2" > ${PROXY_PIPE}`,
+      'sh',
+      address,
+      settled,
+    ];
+    await exec(id, told, signal);
+    return socks;
+  } catch (error) {
+    await socks.stop();
+    throw error;
+  }
+};
+
 /** Playwright's browsers in its official Linux image, served to the host's tests so screenshots
  *  render the same on every machine, trusting the local CA in `caDir` that signs every edge. */
 export const startBrowserServer = async (
@@ -95,7 +186,30 @@ export const startBrowserServer = async (
   caDir = LOCAL_CA_DIR,
 ): Promise<BrowserServer> => {
   const ca = await localCaCertificate(caDir, signal);
-  const server = await container(browserServerSpec(playwrightCore(), ca), signal);
-  const { host, port } = server.address(PORT);
-  return { wsEndpoint: `ws://${host}:${String(port)}/`, stop: server.stop };
+  const proxied: { socks?: Socks } = {};
+  const started = async (id: string): Promise<void> => {
+    proxied.socks = await proxyFor(id, signal);
+  };
+  const stopProxy = async (): Promise<void> => {
+    if (proxied.socks !== undefined) {
+      await proxied.socks.stop();
+    }
+  };
+  const server = await container(browserServerSpec(playwrightCore(), ca), signal, started).catch(
+    async (error: unknown) => {
+      await stopProxy();
+      throw error;
+    },
+  );
+  const endpointFor = (engine: Engine): string => {
+    const { host, port } = server.address(PORTS[engine]);
+    return `ws://${host}:${String(port)}/${engine}`;
+  };
+  return {
+    endpoints: { chromium: endpointFor('chromium'), webkit: endpointFor('webkit') },
+    stop: async () => {
+      await server.stop();
+      await stopProxy();
+    },
+  };
 };

package/testing/src/stack/containers.ts

@@ -3,7 +3,7 @@ import type { Service } from './process.ts';
 import { isGone, RUN } from './run.ts';
 import { appleContainerOf } from './runtime-apple.ts';
 import { dockerOf } from './runtime-docker.ts';
-import type { Address, ContainerSpec, Runtime } from './runtime.ts';
+import type { Address, ContainerSpec, Network, Runtime } from './runtime.ts';
 
 export interface Container extends Service {
   readonly id: string;
@@ -83,7 +83,13 @@ const addressIn =
     return found;
   };
 
-export const container = async (spec: ContainerSpec, signal: AbortSignal): Promise<Container> => {
+/** A container started from `spec` once it says it's ready; `started` runs as it starts, before
+ *  that, for a service that waits on something the host gives it. */
+export const container = async (
+  spec: ContainerSpec,
+  signal: AbortSignal,
+  started?: (id: string) => Promise<void>,
+): Promise<Container> => {
   signal.throwIfAborted();
   await sweptOnce();
   const { id, forget } = await created(spec);
@@ -92,6 +98,9 @@ export const container = async (spec: ContainerSpec, signal: AbortSignal): Promi
     forget();
   };
   try {
+    if (started !== undefined) {
+      await started(id);
+    }
     const texts = spec.readyInAnyOrder === true ? spec.ready.map((text) => [text]) : [spec.ready];
     await Promise.all(
       texts.map(async (text) => RUNTIME.ready(id, { texts: text, logFile: spec.logFile }, signal)),
@@ -104,6 +113,9 @@ export const container = async (spec: ContainerSpec, signal: AbortSignal): Promi
   }
 };
 
+/** A running container's address on its network, and the host's there. */
+export const networkOf = async (id: string): Promise<Network> => RUNTIME.network(id);
+
 export const exec = async (
   id: string,
   command: readonly string[],

package/testing/src/stack/control.ts

@@ -2,8 +2,9 @@ import { request } from 'node:http';
 
 import { test } from '@playwright/test';
 
+import { holdCpuTurn } from './gate-member.ts';
 import { WHOLE } from './gate-server.ts';
-import { type HoldKind, openHold } from './hold.ts';
+import { openHold } from './hold.ts';
 
 /** The variable naming the stack host, set by the global setup for every worker. */
 export const HOST_ENV = 'KIT_STACK_HOST';
@@ -21,7 +22,8 @@ export const STACK_ENV = 'KIT_STACK';
  *  a keychain prompt, and the visual projects' Chromium, so screenshots match on every machine. */
 export const LINUX_BROWSERS = 'linux-browsers';
 
-/** The variable WebKit rows and the visual projects read that server's endpoint from. */
+/** The variable WebKit rows and the visual projects read that server's endpoints from, by engine,
+ *  in JSON. */
 export const LINUX_ENV = 'KIT_LINUX_BROWSERS';
 
 export type Action = 'start' | 'stop' | 'restart';
@@ -65,6 +67,59 @@ export const control = async (action: Action, key: string): Promise<Record<strin
   return promise;
 };
 
+/** The running test's clock, as Playwright keeps its timeout: when it began, and how long it has
+ *  waited for holds in fixtures of their own timeout, which aren't charged to it. */
+export interface TestClock {
+  start: number;
+  waited: number;
+}
+
+/** This worker's running test's clock: a worker runs one test at a time. */
+const running: TestClock = { start: Date.now(), waited: 0 };
+
+/** Starts the running test's clock (fixtures.ts), at `now`. */
+export const testBegins = (now: number = Date.now()): void => {
+  running.start = now;
+  running.waited = 0;
+};
+
+/** When a test must end, by Playwright's count: its start, plus the hold waits it isn't charged,
+ *  plus its timeout; none for a test with no timeout. A hold's server lets a stillness pass while
+ *  its holders are inside their deadlines; the test moves them as its timeout moves. */
+export const deadlineFrom = (timeout: number, clock: TestClock = running): number | undefined =>
+  timeout > 0 ? clock.start + clock.waited + timeout : undefined;
+
+/** `wait` for a hold, its time added to the running test's waits, which its deadline is past. */
+export const waitedFor = async <Value>(wait: () => Promise<Value>): Promise<Value> => {
+  const asked = Date.now();
+  try {
+    return await wait();
+  } finally {
+    running.waited += Date.now() - asked;
+  }
+};
+
+const holderDeadline = (): number | undefined => {
+  try {
+    return deadlineFrom(test.info().timeout);
+  } catch {
+    return undefined;
+  }
+};
+
+/** Who asks for a hold, for its server's holder record: the running test, if any, its worker and
+ *  its process, so a wait past its bound names who held what. */
+const holderName = (): string => {
+  const worker = process.env['TEST_WORKER_INDEX'];
+  const where = `${worker === undefined ? '' : `worker ${worker}, `}pid ${String(process.pid)}`;
+  try {
+    const info = test.info();
+    return `${info.project.name} › ${info.title} (${where})`;
+  } catch {
+    return where;
+  }
+};
+
 /**
  * Holds a target's stack, shared or owned alone, until the returned release is called: the hold
  * is an open request, so a worker that dies releases it by closing its connection.
@@ -79,10 +134,11 @@ export const holdStack = async (hold: 'share' | 'own', key: string): Promise<()
       held: `a ${hold} hold on ${key}`,
       refused: () => `the stack host refused a ${hold} hold on ${key}`,
     },
+    { who: holderName(), deadline: holderDeadline() },
   );
 
-/** A seat among the Linux image's browsers for each context the worker's tests have open at once,
- *  all taken together and held as a stack hold is, for the worker's life. */
+/** A seat among the Linux image's browsers for each context the test may have open at once, all
+ *  taken together and held as a stack hold is, for the test's life. */
 export const holdBrowserSeat = async (contexts: number): Promise<() => void> =>
   openHold(
     new URL(
@@ -96,19 +152,14 @@ export const holdBrowserSeat = async (contexts: number): Promise<() => void> =>
           ? `the Linux browsers seat fewer than the ${String(contexts)} browserContexts declared`
           : 'the stack host refused seats among the Linux browsers',
     },
+    { who: holderName(), deadline: holderDeadline() },
   );
 
 const runTurns = (): string => required(CPU_GATE_ENV, 'the global setup joins the CPU gate');
 
-const TURN: HoldKind = {
-  held: 'a CPU turn',
-  refused: (status) =>
-    `the CPU gate refused a turn (${String(status)}): the run has left it, or lost it`,
-};
-
 /** A CPU turn, until the returned release; an abort gives up a turn still waiting. */
 export const holdTurn = async (turns: string, signal?: AbortSignal): Promise<() => void> =>
-  openHold(new URL(turns), TURN, signal);
+  holdCpuTurn(turns, { signal, who: holderName(), deadline: holderDeadline() });
 
 /** A test's CPU turn: its release, its widening once claimed, and the test's end, which gives up
  *  a widening still waiting, so no every-slot turn is granted after its test. */

package/testing/src/stack/gate-member.ts

@@ -11,10 +11,15 @@ import {
 } from 'node:http';
 
 import {
+  answerDeadline,
+  answerHolders,
   GATE_MARK,
   GATE_PORT,
   GATE_SCRIPT,
   GATE_VERSION,
+  deadlineOf,
+  holderOf,
+  holdIdOf,
   holdOpen,
   LAUNCH,
   SERVES,
@@ -183,7 +188,13 @@ const notYet = (error: unknown): boolean => isAbsent(error) || statusOf(error) =
  *  can't give it. One the gate ends cuts the holder's off, or ends it whole once the run let go. */
 const turnAtGate = async (
   held: Held,
-  asked: { readonly whole: boolean; readonly holder: ServerResponse },
+  asked: {
+    readonly whole: boolean;
+    readonly holder: ServerResponse;
+    readonly who: string;
+    readonly deadline: number | undefined;
+    readonly id: string | undefined;
+  },
   gone: AbortSignal,
 ): Promise<() => void> => {
   const { promise: changed } = held.changed;
@@ -201,7 +212,8 @@ const turnAtGate = async (
     },
   };
   try {
-    return await openHold(url, kind, gone);
+    const { who, deadline, id } = asked;
+    return await openHold(url, kind, { signal: gone, who, deadline, id });
   } catch (error) {
     if (held.left || !notYet(error)) {
       throw held.left ? new Error('the run has left the CPU gate', { cause: held.reason }) : error;
@@ -211,22 +223,54 @@ const turnAtGate = async (
   }
 };
 
+/** A turn's moved deadline, passed on to the gate the turn is held at, which a gate too old to
+ *  know the route refuses: the turn keeps the deadline it was asked with. */
+const passDeadline = (port: number, path: string): void => {
+  const call = request(`http://127.0.0.1:${String(port)}${path}`, { method: 'POST' }, (answer) => {
+    answer.resume();
+  });
+  call.once('error', () => {
+    // A gate gone: there is no turn there to move.
+  });
+  call.end();
+};
+
 /** Serves the run's turns to its holders, in this process or its workers, at a local URL; the
  *  server lets the process end, and stops taking turns as the run leaves. */
 const serveTurns = async (held: Held): Promise<Server> => {
   const server = createServer({ requestTimeout: 0 }, (incoming, holder) => {
     incoming.resume();
-    const asked = { whole: incoming.url === `/cpu/${WHOLE}`, holder };
-    const turn = { release: (): void => undefined };
-    holdOpen(
+    const path = incoming.url ?? '';
+    if (path === '/holders') {
+      answerHolders(holder);
+      return;
+    }
+    if (path.startsWith('/deadline/')) {
+      const parts = path.split('/');
+      answerDeadline(holder, parts[2] ?? '', parts[3] ?? '');
+      passDeadline(held.port, path);
+      return;
+    }
+    const asked = {
+      whole: path === `/cpu/${WHOLE}`,
       holder,
-      async (gone) => {
+      who: holderOf(incoming),
+      deadline: deadlineOf(incoming),
+      id: holdIdOf(incoming),
+    };
+    const turn = { release: (): void => undefined };
+    holdOpen(holder, {
+      take: async (gone) => {
         turn.release = await turnAtGate(held, asked, gone);
       },
-      () => {
+      give: () => {
         turn.release();
       },
-    );
+      what: asked.whole ? 'every CPU slot' : 'a CPU turn',
+      who: asked.who,
+      deadline: asked.deadline,
+      id: asked.id,
+    });
   });
   server.listen(0, '127.0.0.1');
   await once(server, 'listening');
@@ -268,3 +312,41 @@ export const joinGate = async (
     },
   };
 };
+
+/** What a CPU turn is called, and what a refusal of one says. */
+const TURN: HoldKind = {
+  held: 'a CPU turn',
+  refused: (status) =>
+    `the CPU gate refused a turn (${String(status)}): the run has left it, or lost it`,
+};
+
+/** A CPU turn at `turns` (a run's, or with `/whole`, every slot), until the returned release; an
+ *  abort gives up a turn still waiting. */
+export const holdCpuTurn = async (
+  turns: string,
+  asking: {
+    readonly signal?: AbortSignal | undefined;
+    readonly who: string;
+    readonly deadline?: number | undefined;
+  },
+): Promise<() => void> => openHold(new URL(turns), TURN, asking);
+
+/** Runs `work` holding every slot of the CPU gate on `port`, joined for it alone, once the gate's
+ *  queue reaches it: no other run's tests share the cores while it runs. */
+export const inWholeTurn = async <Value>(
+  work: () => Promise<Value>,
+  who: string,
+  port = GATE_PORT,
+): Promise<Value> => {
+  const gate = await joinGate(port);
+  try {
+    const release = await holdCpuTurn(`${gate.turns}/${WHOLE}`, { who });
+    try {
+      return await work();
+    } finally {
+      release();
+    }
+  } finally {
+    gate.leave();
+  }
+};