@tonalisk-labs/kit 1.0.1 → 1.0.6

package/testing/src/stack/gate-server.ts

@@ -5,6 +5,9 @@ import { once } from 'node:events';
 import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http';
 import { availableParallelism } from 'node:os';
 
+import { DEADLINE_HEADER, described, HOLD_ID_HEADER, HOLDER_HEADER } from './hold.ts';
+import { type Held, STILLNESS, type Stillness, type Watch, watchOf } from './stall.ts';
+
 /** The fixed local port the machine's one CPU gate listens on: every run finds it there. */
 export const GATE_PORT = 47_391;
 
@@ -26,18 +29,121 @@ export const SLOTS = availableParallelism();
 /** The last segment of a turn's path that asks for every slot: a measurement load would skew. */
 export const WHOLE = 'whole';
 
+/** A hold this process serves, held or waited for, by whom, since when and by when its test must
+ *  end (epoch ms). */
+interface Holding {
+  readonly what: string;
+  readonly who: string;
+  deadline: number | undefined;
+  since: number;
+  held: boolean;
+}
+
+/** Every hold this process serves, so a wait past its bound can name who holds and who waits. */
+const holdings = new Set<Holding>();
+
+/** Each hold's way to move its deadline, by the id its request gave. */
+const renewers = new Map<string, (deadline: number | undefined) => void>();
+
+/** A hold a server serves: how it is taken and given back, what it is, who asked for it and by
+ *  when their test must end, and the watch over its resource's waits, which counts its holders. */
+export interface Served {
+  readonly take: (gone: AbortSignal) => Promise<void>;
+  readonly give: () => void;
+  readonly what: string;
+  readonly who: string;
+  readonly deadline?: number | undefined;
+  readonly watch?: Watch;
+  readonly id?: string | undefined;
+}
+
+/** A deadline as given: epoch milliseconds, or none. */
+const deadlineIn = (given: unknown): number | undefined => {
+  const value = Number(given);
+  return typeof given === 'string' && Number.isFinite(value) && value > 0 ? value : undefined;
+};
+
+/** The deadline a request gives its holder's test, or none. */
+export const deadlineOf = (incoming: IncomingMessage): number | undefined =>
+  deadlineIn(incoming.headers[DEADLINE_HEADER]);
+
+/** The id a hold's request gives itself, or none. */
+export const holdIdOf = (incoming: IncomingMessage): string | undefined => {
+  const id = incoming.headers[HOLD_ID_HEADER];
+  return typeof id === 'string' && id !== '' ? id : undefined;
+};
+
+/** A hold's id as openHold makes it, a random UUID: nothing else names a hold. */
+const HOLD_ID = /^[\da-f]{8}-[\da-f]{4}-4[\da-f]{3}-[89ab][\da-f]{3}-[\da-f]{12}$/u;
+
+/** Moves the deadline of the hold `id` names to `given`, epoch milliseconds or none at all, as
+ *  its test's timeout moved: 204, or 404 for a hold this process doesn't serve. A request names
+ *  only an entry of the table: an id of another shape is looked up nowhere. */
+export const answerDeadline = (outgoing: ServerResponse, id: string, given: string): void => {
+  const renew = HOLD_ID.test(id) && renewers.has(id) ? renewers.get(id) : undefined;
+  if (typeof renew !== 'function') {
+    outgoing.writeHead(404).end('no such hold here');
+    return;
+  }
+  renew(deadlineIn(given));
+  outgoing.writeHead(204).end();
+};
+
+/** The holder a request names, decoded, or as sent where it isn't URI-encoded; or that it named
+ *  none. */
+export const holderOf = (incoming: IncomingMessage): string => {
+  const named = incoming.headers[HOLDER_HEADER];
+  if (typeof named !== 'string' || named === '') {
+    return 'an unnamed holder';
+  }
+  try {
+    return decodeURIComponent(named);
+  } catch {
+    return named;
+  }
+};
+
+/** This process's holdings, each named as GET /holders gives it: what, by whom, for how long. */
+export const holdingsNamed = (): string => {
+  const now = Date.now();
+  return holdings.size === 0
+    ? 'it records no holder or waiter'
+    : [...holdings].map((holding) => described(holding, now)).join('; ');
+};
+
+/** Answers a GET /holders with this process's holdings, as JSON. */
+export const answerHolders = (outgoing: ServerResponse): void => {
+  outgoing
+    .writeHead(200, { 'content-type': 'application/json' })
+    .end(JSON.stringify([...holdings]));
+};
+
 /** A hold (a stack lock, a CPU turn) lasts as long as its request stays open: granted with the
  *  response head, given back when the caller closes it, however its worker ends. */
-export const holdOpen = (
-  outgoing: ServerResponse,
-  take: (gone: AbortSignal) => Promise<void>,
-  give: () => void,
-): void => {
+export const holdOpen = (outgoing: ServerResponse, served: Served): void => {
+  const { take, give, what, who, deadline, watch, id } = served;
   const gone = new AbortController();
-  const granted = { now: false };
+  const granted: { now: boolean; held?: Held } = { now: false };
+  const holding: Holding = { what, who, deadline, since: Date.now(), held: false };
+  holdings.add(holding);
+  if (id !== undefined) {
+    renewers.set(id, (moved) => {
+      holding.deadline = moved;
+      if (granted.held !== undefined) {
+        granted.held.renew(moved);
+      }
+    });
+  }
   outgoing.once('close', () => {
+    holdings.delete(holding);
+    if (id !== undefined) {
+      renewers.delete(id);
+    }
     gone.abort();
     if (granted.now) {
+      if (granted.held !== undefined) {
+        granted.held.release();
+      }
       give();
     }
   });
@@ -49,10 +155,15 @@ export const holdOpen = (
         return;
       }
       granted.now = true;
+      if (watch !== undefined) {
+        granted.held = watch.holding(holding.deadline);
+      }
+      holding.held = true;
+      holding.since = Date.now();
       outgoing.writeHead(200).flushHeaders();
     })
     .catch((error: unknown) => {
-      outgoing.writeHead(503).end(String(error));
+      outgoing.writeHead(503).end(error instanceof Error ? error.message : String(error));
     });
 };
 
@@ -67,12 +178,15 @@ export interface Semaphore {
   readonly take: (run: string, slots: number, gone: AbortSignal) => Promise<void>;
   readonly give: (slots: number) => void;
   readonly refuse: (run: string) => void;
+  /** The watch over its waits, which a server counts each holder in. */
+  readonly watch: Watch;
 }
 
-/** The free slots and the turns waiting for them, in the order they asked. */
+/** The free slots and the turns waiting for them, in the order they asked, and its watch. */
 interface Queue {
   free: number;
   readonly waiting: Set<Waiting>;
+  readonly watch: Watch;
 }
 
 /** Grants the waiting turns in order while the first fits, so a turn asking more than are free
@@ -94,17 +208,28 @@ const enqueue = async (
   gone: AbortSignal,
 ): Promise<void> => {
   const { promise, resolve, reject } = Promise.withResolvers<boolean>();
+  const done = queue.watch.waiting((stalled) => {
+    queue.waiting.delete(waiting);
+    waiting.refuse(stalled);
+    admit(queue);
+  });
   const waiting: Waiting = {
     run,
     slots,
     grant: () => {
+      done();
+      queue.watch.moved();
       resolve(true);
     },
-    refuse: reject,
+    refuse: (reason) => {
+      done();
+      reject(reason);
+    },
   };
   gone.addEventListener(
     'abort',
     () => {
+      done();
       queue.waiting.delete(waiting);
       admit(queue);
     },
@@ -116,12 +241,19 @@ const enqueue = async (
 };
 
 /** The slots, first come, first served, so a whole-machine turn is never starved. A set keeps the
- *  order with O(1) removal. */
-export const semaphore = (slots: number): Semaphore => {
-  const queue: Queue = { free: slots, waiting: new Set() };
+ *  order with O(1) removal. A wait fails once no slot has changed hands for the stillness's stall,
+ *  saying `what` stood still. */
+export const semaphore = (
+  slots: number,
+  { stillness = STILLNESS, what = 'no slot' }: { stillness?: Stillness; what?: string } = {},
+): Semaphore => {
+  const watch = watchOf(stillness, { what, named: holdingsNamed });
+  const queue: Queue = { free: slots, waiting: new Set(), watch };
   return {
+    watch,
     take: async (run, asked, gone) => enqueue(queue, { run, slots: asked }, gone),
     give: (given) => {
+      watch.moved();
       queue.free += given;
       admit(queue);
     },
@@ -171,18 +303,26 @@ interface Runs {
  *  for a run never seen, whose gate was stopped and which is rejoining this one. */
 const turnFor = (
   runs: Runs,
-  { run, size }: { readonly run: string; readonly size: string },
+  {
+    run,
+    size,
+    incoming,
+  }: { readonly run: string; readonly size: string; readonly incoming: IncomingMessage },
   outgoing: ServerResponse,
 ): void => {
   if (runs.members.has(run)) {
     const asked = size === WHOLE ? runs.slots : 1;
-    holdOpen(
-      outgoing,
-      async (gone) => runs.turns.take(run, asked, gone),
-      () => {
+    holdOpen(outgoing, {
+      take: async (gone) => runs.turns.take(run, asked, gone),
+      give: () => {
         runs.turns.give(asked);
       },
-    );
+      what: `${String(asked)} of the CPU gate's slots for run ${run}`,
+      who: holderOf(incoming),
+      deadline: deadlineOf(incoming),
+      watch: runs.turns.watch,
+      id: holdIdOf(incoming),
+    });
   } else if (runs.left.has(run)) {
     outgoing.writeHead(410).end(`no cpu for ${run}, which has left`);
   } else {
@@ -196,7 +336,11 @@ const answer =
     const [, action = '', encoded = '', size = ''] = (incoming.url ?? '').split('/');
     const run = decodeURIComponent(encoded);
     incoming.resume();
-    if (run === '') {
+    if (action === 'holders') {
+      answerHolders(outgoing);
+    } else if (action === 'deadline') {
+      answerDeadline(outgoing, run, size);
+    } else if (run === '') {
       outgoing.writeHead(404).end(`no run named for ${action}`);
     } else if (action === 'join') {
       gate.hold(run);
@@ -205,7 +349,7 @@ const answer =
       });
       outgoing.writeHead(200, { [GATE_MARK]: GATE_VERSION }).flushHeaders();
     } else if (action === 'cpu') {
-      turnFor(runs, { run, size }, outgoing);
+      turnFor(runs, { run, size, incoming }, outgoing);
     } else {
       outgoing.writeHead(404).end(`no ${action} for ${run}, which has not joined`);
     }
@@ -214,7 +358,8 @@ const answer =
 /** The gate on a listening server: a run joins, holds turns while joined, and leaves. Its last
  *  member's leave closes the port before anything else, so a join it then refuses finds it free. */
 const gateOn = (server: Server, slots: number): Gate => {
-  const runs: Runs = { turns: semaphore(slots), slots, members: new Set(), left: new Set() };
+  const turns = semaphore(slots, { what: 'no CPU turn' });
+  const runs: Runs = { turns, slots, members: new Set(), left: new Set() };
   const gate: Gate = {
     hold: (run) => {
       runs.left.delete(run);

package/testing/src/stack/hold.ts

@@ -1,6 +1,19 @@
 // A hold (a stack lock, a CPU turn) is an open request: granted with the response head, and held
 // until the holder closes it, so a holder that dies gives it back.
-import { type ClientRequest, type IncomingMessage, request } from 'node:http';
+import { randomUUID } from 'node:crypto';
+import { type ClientRequest, get, type IncomingMessage, request } from 'node:http';
+
+import { isObject } from '../unknown.ts';
+
+/** The header a hold's request names its holder in (its test, worker and process), URI-encoded
+ *  for the characters a test's title holds and a header can't. */
+export const HOLDER_HEADER = 'x-kit-holder';
+
+/** The header a hold's request gives its test's deadline in, as epoch milliseconds. */
+export const DEADLINE_HEADER = 'x-kit-deadline';
+
+/** The header a hold's request gives its id in, by which its test moves its deadline later. */
+export const HOLD_ID_HEADER = 'x-kit-hold';
 
 /** A kind of hold: what it is called, what a refusal of it says, and who hears should the server
  *  end it first; with no one named, the holder's release fails instead. */
@@ -10,6 +23,87 @@ export interface HoldKind {
   readonly ended?: () => void;
 }
 
+/** One holding a server lists: what, by whom, since when, held or waited for. */
+export const described = (entry: unknown, now: number): string => {
+  if (!isObject(entry)) {
+    return String(entry);
+  }
+  const { what, who, since, held, deadline } = entry;
+  const seconds = typeof since === 'number' ? Math.round((now - since) / 1000) : '?';
+  const state = held === true ? 'held' : 'waited for';
+  const left = typeof deadline === 'number' ? Math.round((deadline - now) / 1000) : undefined;
+  const due =
+    left === undefined
+      ? ''
+      : `, ${left >= 0 ? `due in ${String(left)} s` : `${String(-left)} s past its deadline`}`;
+  return `${String(what)} ${state} by ${String(who)} for ${String(seconds)} s${due}`;
+};
+
+/** The holdings a GET /holders listed, each named. */
+const namedIn = (listed: unknown): string => {
+  if (!Array.isArray(listed) || listed.length === 0) {
+    return 'it records no holder or waiter';
+  }
+  const now = Date.now();
+  return listed.map((entry) => described(entry, now)).join('; ');
+};
+
+/** Who holds and waits at `origin`, as its GET /holders lists them, or why it couldn't say. Read
+ *  through node:http, which tells a reset as its request's error: Node 24.21's fetch (undici
+ *  7.29.1) throws it uncaught from `setTypeOfService` on macOS, ending the process. */
+const holdersAt = async (origin: string): Promise<string> => {
+  const { promise, resolve } = Promise.withResolvers<string>();
+  const failed = (error: unknown): void => {
+    resolve(`its holders could not be read: ${String(error)}`);
+  };
+  const call = get(new URL('/holders', origin), { signal: AbortSignal.timeout(5000) }, (answer) => {
+    const chunks: Buffer[] = [];
+    answer
+      .on('data', (chunk: Buffer) => {
+        chunks.push(chunk);
+      })
+      .once('end', () => {
+        try {
+          const listed: unknown = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+          resolve(namedIn(listed));
+        } catch (error) {
+          failed(error);
+        }
+      })
+      .once('error', failed);
+  });
+  call.once('error', failed);
+  return promise;
+};
+
+/** The holds this process has open, granted or waiting, by id: where each is served. */
+const live = new Map<string, string>();
+
+/** Moves the deadline of every hold this process has open to `deadline` (epoch milliseconds, or
+ *  none), as its test's timeout moved; a server that can't be told keeps the one it had. */
+export const announceDeadline = async (deadline?: number): Promise<void> => {
+  const given = deadline === undefined ? 'none' : String(Math.ceil(deadline));
+  await Promise.all(
+    [...live].map(async ([id, origin]) => {
+      const { promise, resolve } = Promise.withResolvers<boolean>();
+      const call = request(
+        new URL(`/deadline/${id}/${given}`, origin),
+        { method: 'POST' },
+        (answer) => {
+          answer.resume().once('end', () => {
+            resolve(true);
+          });
+        },
+      );
+      call.once('error', () => {
+        resolve(false);
+      });
+      call.end();
+      await promise;
+    }),
+  );
+};
+
 /** A hold refused, with the status the server answered. */
 const refusal = (message: string, status: number | undefined): Error =>
   Object.assign(new Error(message), { status });
@@ -42,26 +136,84 @@ const releaseOf = (
   };
 };
 
-/** Asks `url` for a hold, answering its release once granted; an abort gives up a hold not yet
- *  granted, and ends one that is. */
+/** How a hold is asked for: the signal that gives it up, who asks, by when their test must end,
+ *  and the id its test moves that deadline by (one of its own, unless a relay passes it on). */
+interface Asking {
+  readonly signal?: AbortSignal | undefined;
+  readonly who?: string;
+  readonly deadline?: number | undefined;
+  readonly id?: string | undefined;
+}
+
+/** A hold request's headers: its id, its holder's name, URI-encoded, and its test's deadline. */
+const headersOf = ({ who, deadline }: Asking, id: string): Record<string, string> => {
+  const headers: Record<string, string> = { [HOLD_ID_HEADER]: id };
+  if (who !== undefined) {
+    headers[HOLDER_HEADER] = encodeURIComponent(who);
+  }
+  if (deadline !== undefined) {
+    headers[DEADLINE_HEADER] = String(Math.ceil(deadline));
+  }
+  return headers;
+};
+
+/** Counts a hold among this process's open ones, waiting or granted, until its request closes: its
+ *  test may move its deadline meanwhile. */
+const counted = (call: ClientRequest, id: string, origin: string): void => {
+  live.set(id, origin);
+  call.once('close', () => {
+    live.delete(id);
+  });
+};
+
+/** Asks `url` for a hold, answering its release once granted; an abort ends one that is, and gives
+ *  up one not yet granted with an AbortError naming who held what it waited for. */
 export const openHold = async (
   url: URL,
   kind: HoldKind,
-  signal?: AbortSignal,
+  asking: Asking = {},
 ): Promise<() => void> => {
+  const { signal, id = randomUUID() } = asking;
   const { promise, resolve, reject } = Promise.withResolvers<() => void>();
-  const call = request(url, { method: 'POST', signal }, (response) => {
+  const headers = headersOf(asking, id);
+  const waiting: { granted: boolean; failing?: Promise<void> } = { granted: false };
+  const call = request(url, { method: 'POST', signal, headers }, (response) => {
     const { statusCode: status } = response;
-    // Read: a refusal to its end, so its socket is freed, as one left unread holds the process open;
-    // a grant, so that its server ending it whole is heard.
-    response.resume();
+    // Read: a grant, so that its server ending it whole is heard; a refusal to its end, so its
+    // socket is freed, as one left unread holds the process open, and what it says is told.
     if (status === 200) {
+      waiting.granted = true;
+      response.resume();
       resolve(releaseOf(call, response, kind));
-    } else {
-      reject(refusal(kind.refused(status), status));
+      return;
+    }
+    const said: Buffer[] = [];
+    response
+      .on('data', (chunk: Buffer) => {
+        said.push(chunk);
+      })
+      .once('close', () => {
+        const reason = Buffer.concat(said).toString('utf8').trim();
+        const message = kind.refused(status);
+        reject(refusal(reason === '' ? message : `${message}: ${reason}`, status));
+      });
+  });
+  call.on('error', (error) => {
+    // A granted hold's end, by its release or its server, is the release's to hear.
+    if (waiting.granted) {
+      return;
+    }
+    if (signal === undefined || !signal.aborted) {
+      reject(error);
+      return;
     }
+    waiting.failing = holdersAt(url.origin).then((holders) => {
+      const given = `the wait for ${kind.held} was given up before a grant`;
+      const named = new Error(`${given}: at its server, ${holders}`, { cause: error });
+      reject(Object.assign(named, { name: 'AbortError' }));
+    });
   });
-  call.on('error', reject);
+  counted(call, id, url.origin);
   call.end();
   return promise;
 };

package/testing/src/stack/host.ts

@@ -12,7 +12,14 @@ import {
   STACK_ENV,
 } from './control.ts';
 import { joinGate, type Membership } from './gate-member.ts';
-import { holdOpen } from './gate-server.ts';
+import {
+  answerDeadline,
+  answerHolders,
+  deadlineOf,
+  holderOf,
+  holdIdOf,
+  holdOpen,
+} from './gate-server.ts';
 import { seatPool, stackLocks } from './lock.ts';
 import type { Service } from './process.ts';
 import { startTarget } from './stack.ts';
@@ -32,7 +39,7 @@ interface Entry {
 const startKey = async (key: string, turns: string, signal: AbortSignal): Promise<Started> => {
   if (key === LINUX_BROWSERS) {
     const server = await startBrowserServer(signal);
-    return { env: { [LINUX_ENV]: server.wsEndpoint }, stop: server.stop };
+    return { env: { [LINUX_ENV]: JSON.stringify(server.endpoints) }, stop: server.stop };
   }
   const target = discover().find(({ name }) => name === key);
   if (target === undefined) {
@@ -105,20 +112,36 @@ interface Hosted {
 const handle =
   (hosted: Hosted) =>
   (incoming: IncomingMessage, outgoing: ServerResponse): void => {
-    const [, action = '', key = ''] = (incoming.url ?? '').split('/');
+    const [, action = '', key = '', given = ''] = (incoming.url ?? '').split('/');
+    if (action === 'holders') {
+      answerHolders(outgoing);
+      return;
+    }
+    if (action === 'deadline') {
+      answerDeadline(outgoing, key, given);
+      return;
+    }
     if (action === 'browser') {
-      browsers(outgoing, key);
+      browsers(outgoing, key, {
+        who: holderOf(incoming),
+        deadline: deadlineOf(incoming),
+        id: holdIdOf(incoming),
+      });
       return;
     }
     if (action === 'share' || action === 'own') {
       const stack = decodeURIComponent(key);
-      holdOpen(
-        outgoing,
-        async (gone) => locks.take(stack, action, gone),
-        () => {
+      holdOpen(outgoing, {
+        take: async (gone) => locks.take(stack, action, gone),
+        give: () => {
           locks.give(stack, action);
         },
-      );
+        what: `a ${action} hold on ${stack}`,
+        who: holderOf(incoming),
+        deadline: deadlineOf(incoming),
+        watch: locks.watch(stack),
+        id: holdIdOf(incoming),
+      });
       return;
     }
     const called = ACTIONS.find((known) => known === action);