@shotkit/shotium 0.0.1 → 0.1.0

package/src/index.ts

@@ -0,0 +1,161 @@
+import {EventEmitter} from 'node:events';
+
+import * as client from './lib/client.js';
+import type {DaemonClient} from './lib/client.js';
+import {resolveStartOptions} from './lib/config.js';
+import {Pool} from './lib/pool.js';
+import {SUPERVISOR_MARGIN_MS, timeoutFor, toRequest} from './lib/request.js';
+import type {
+  DaemonOptions,
+  DaemonStatus,
+  ScreenshotOptions,
+  StartOptions,
+  WorkerEvent,
+} from './types.js';
+
+export type {
+  Clip,
+  DaemonOptions,
+  DaemonStatus,
+  PageGotoParams,
+  PurgeOptions,
+  ScreenshotOptions,
+  StartOptions,
+  Viewport,
+  WorkerEvent,
+} from './types.js';
+export type {DaemonClient} from './lib/client.js';
+
+/** The five things a caller does with the resident pool. */
+export interface Daemon {
+  /** Connects, starting a daemon if none is listening. */
+  connect(options?: DaemonOptions): Promise<DaemonClient>;
+  /** One screenshot through the daemon, connection and all. */
+  screenshot(options: ScreenshotOptions&{daemon?: DaemonOptions}):
+      Promise<Buffer|null>;
+  /** Starts one if it is not up, and reports what is there either way. */
+  start(options?: DaemonOptions): Promise<DaemonStatus&{spawned: boolean}>;
+  status(options?: DaemonOptions):
+      Promise<Partial<DaemonStatus>&{running: boolean, endpoint: string}>;
+  stop(options?: DaemonOptions): Promise<{stopped: boolean, endpoint: string}>;
+}
+
+// The events the pool forwards, and the only ones. Declared as an interface
+// merged into the class below rather than as a catch-all `on(string, ...)`,
+// so that a listener for an event this runtime never emits is a compile error
+// rather than a callback nobody ever calls.
+export interface Runtime {
+  on(event: 'ready', listener: (info: {workers: number}) => void): this;
+  on(event: 'exit', listener: (event: WorkerEvent) => void): this;
+  on(event: 'crash', listener: (event: WorkerEvent) => void): this;
+  on(event: 'timeout',
+     listener: (event: {worker: number, timeout: number}) => void): this;
+  on(event: 'worker-restart',
+     listener: (event: {worker: number, reason: string, delay: number}) => void):
+      this;
+  /** A worker could not be started at all -- a missing or unusable binary. */
+  on(event: 'worker-error',
+     listener: (event: {worker: number, error: Error}) => void): this;
+  on(event: 'stderr',
+     listener: (event: {worker: number, line: string}) => void): this;
+}
+
+/**
+ * The library's one runtime: a pool of worker processes plus its lifecycle.
+ *
+ * `runtime` below is the singleton, because the expensive part is the
+ * processes and a second runtime would double them for no gain. Anyone who
+ * genuinely wants two constructs a Runtime directly.
+ *
+ * Its pool lives and dies with this process. `daemon` is the same pool behind
+ * a socket, for callers whose process does not live long enough to be worth
+ * starting one.
+ */
+export class Runtime extends EventEmitter {
+  private pool: Pool|null = null;
+
+  get running(): boolean {
+    return this.pool !== null;
+  }
+
+  /**
+   * Starts the pool. Safe to call twice; the second call is a no-op, so that
+   * library code can call it defensively.
+   *
+   * Every option has a default: the binary is `$SHOTIUM_BINARY`, then the
+   * platform package, then `./bin/shotium.exe`; the worker count is half the
+   * cores, at least one and at most four; the cache root is a directory under
+   * the system temp, and `null` disables caching.
+   */
+  start(options: StartOptions = {}): this {
+    if (this.pool) {
+      return this;
+    }
+    const pool = new Pool(resolveStartOptions(options));
+    this.pool = pool;
+    for (const event
+             of ['ready', 'exit', 'crash', 'timeout', 'worker-restart',
+                 'worker-error', 'stderr']) {
+      pool.on(event, (payload) => this.emit(event, payload));
+    }
+    pool.start();
+    return this;
+  }
+
+  /** Stops every worker. The pool can be started again afterwards. */
+  async stop(): Promise<void> {
+    if (!this.pool) {
+      return;
+    }
+    const pool = this.pool;
+    this.pool = null;
+    await pool.stop();
+  }
+
+  /**
+   * Renders one screenshot. Resolves to the encoded image, or to `null` when
+   * `path` was given and the worker wrote the file itself.
+   */
+  async screenshot(options: ScreenshotOptions): Promise<Buffer|null> {
+    // Validate before starting anything. A malformed request should not cost a
+    // pool of worker processes to discover, and toRequest() is the only check
+    // that can be made without one.
+    const request = toRequest(options);
+    if (!this.pool) {
+      this.start();
+    }
+    const retry = typeof options.retry === 'number' ? options.retry : 0;
+    const result = await this.pool!.submit(request, {
+      timeout: timeoutFor(options) + SUPERVISOR_MARGIN_MS,
+      retry,
+    });
+    return result.image;
+  }
+}
+
+/** The shared pool: one per process, started on first use. */
+const runtime = new Runtime();
+
+/** One screenshot through the shared pool, starting it if it is not up. */
+const screenshot = (options: ScreenshotOptions): Promise<Buffer|null> =>
+    runtime.screenshot(options);
+
+/**
+ * The resident pool: workers that outlive the process that started them,
+ * reachable over a named pipe on Windows and a unix socket elsewhere. For
+ * callers that are short-lived themselves. See lib/daemon.ts.
+ */
+const daemon: Daemon = {
+  connect: client.connect,
+  screenshot: client.screenshot,
+  start: client.start,
+  status: client.status,
+  stop: client.stop,
+};
+
+export {runtime, screenshot, daemon};
+
+// A default as well as the names, because `import shotium from` is what a
+// caller coming from `require` writes first, and the two have to be the same
+// object rather than two views that drift.
+export default {Runtime, runtime, screenshot, daemon};

package/src/lib/client.ts

@@ -0,0 +1,377 @@
+import {spawn} from 'node:child_process';
+import {EventEmitter} from 'node:events';
+import fs from 'node:fs';
+import net from 'node:net';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+import type {
+  DaemonOptions,
+  DaemonStatus,
+  ScreenshotOptions,
+} from '../types.js';
+
+import {resolveStartOptions} from './config.js';
+import {endpointFor} from './endpoint.js';
+import {FrameReader, encodeFrame} from './protocol.js';
+import {timeoutFor, toRequest} from './request.js';
+
+// ESM has no __dirname. This is the same thing, from the module's own URL.
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+
+// The detached daemon's entry point, which is a build output beside this one.
+// It is spawned as `node <path>`, so it has to be a file on disk with a name
+// that does not move -- see tsdown.config.ts, where it is an entry of its own
+// for exactly that reason.
+const DAEMON_MAIN = path.join(HERE, 'daemon_main.js');
+// How long to wait for a daemon this process just started to bind its
+// endpoint. Binding happens after the workers are spawned but before they are
+// warm, so this covers process startup and nothing else.
+const START_TIMEOUT_MS = 20000;
+const CONNECT_RETRY_MS = 20;
+
+interface ClientReply {
+  id: number;
+  ok?: boolean;
+  error?: string;
+  path?: string;
+}
+
+interface ClientResult {
+  header: ClientReply;
+  image: Buffer|null;
+}
+
+interface Pending {
+  resolve: (result: ClientResult) => void;
+  reject: (error: Error) => void;
+}
+
+interface ResolvedDaemonOptions {
+  binary: string;
+  workers: number;
+  cacheDir: string|null;
+  args: string[];
+  name: string|undefined;
+  endpoint: string;
+  idleTimeoutMs: number|undefined;
+  prewarm: boolean|undefined;
+  logFile: string|null;
+}
+
+// The client half of the resident daemon.
+//
+// One connection can carry several requests at once, which is the difference
+// between this and the worker protocol underneath: every message carries an
+// `id` and the answers are matched back by it, so a caller can fire ten
+// screenshots down one socket and let the pool on the other side spread them
+// across workers.
+class DaemonClient extends EventEmitter {
+  private readonly socket: net.Socket;
+  private readonly endpointPath: string;
+  private readonly pending = new Map<number, Pending>();
+  private nextId = 1;
+  private header: ClientReply|null = null;
+  private reader = new FrameReader();
+
+  constructor(socket: net.Socket, endpoint: string) {
+    super();
+    this.socket = socket;
+    this.endpointPath = endpoint;
+
+    socket.on('data', (chunk: Buffer) => this.onData(chunk));
+    socket.on('error', (error: Error) => this.failAll(error));
+    socket.on('close', () => {
+      this.failAll(new Error('shotium: the daemon closed the connection'));
+      this.emit('close', {});
+    });
+  }
+
+  get endpoint(): string {
+    return this.endpointPath;
+  }
+
+  get closed(): boolean {
+    return this.socket.destroyed;
+  }
+
+  private onData(chunk: Buffer): void {
+    this.reader.push(chunk);
+    for (;;) {
+      const frame = this.reader.next();
+      if (frame === null) {
+        return;
+      }
+      if (this.header === null) {
+        try {
+          this.header = JSON.parse(frame.toString('utf8')) as ClientReply;
+        } catch {
+          this.failAll(
+              new Error('shotium: the daemon sent a header that is not JSON'));
+          return;
+        }
+        continue;
+      }
+      const header = this.header;
+      this.header = null;
+      this.settle(header, frame);
+    }
+  }
+
+  private settle(header: ClientReply, payload: Buffer): void {
+    const pending = this.pending.get(header.id);
+    if (!pending) {
+      return;
+    }
+    this.pending.delete(header.id);
+    if (header.ok) {
+      pending.resolve({header, image: header.path ? null : payload});
+    } else {
+      pending.reject(new Error(header.error || 'shotium: request failed'));
+    }
+  }
+
+  private failAll(error: Error): void {
+    for (const [, pending] of this.pending) {
+      pending.reject(error);
+    }
+    this.pending.clear();
+  }
+
+  // Sends one message and resolves with {header, image}.
+  send(message: Record<string, unknown>): Promise<ClientResult> {
+    return new Promise<ClientResult>((resolve, reject) => {
+      if (this.socket.destroyed) {
+        reject(new Error('shotium: not connected to a daemon'));
+        return;
+      }
+      const id = this.nextId++;
+      this.pending.set(id, {resolve, reject});
+      this.socket.write(
+          encodeFrame(Buffer.from(JSON.stringify({...message, id}), 'utf8')));
+    });
+  }
+
+  /** Resolves to the image, or to null when `path` was given. */
+  async screenshot(options: ScreenshotOptions): Promise<Buffer|null> {
+    const request = toRequest(options);
+    const retry = typeof options.retry === 'number' ? options.retry : 0;
+    const result = await this.send({
+      op: 'screenshot',
+      request,
+      timeout: timeoutFor(options),
+      retry,
+    });
+    return result.image;
+  }
+
+  async status(): Promise<DaemonStatus> {
+    const {header} = await this.send({op: 'status'});
+    return header as unknown as DaemonStatus;
+  }
+
+  async shutdown(): Promise<{ok: boolean}> {
+    const {header} = await this.send({op: 'shutdown'});
+    return {ok: header.ok === true};
+  }
+
+  close(): void {
+    this.socket.end();
+    this.socket.destroy();
+  }
+}
+
+// Opens a connection to a daemon that is already listening, and fails if there
+// is not one. Nothing is spawned here: a caller that wants a daemon started
+// says so, because starting one is a side effect on the machine and not the
+// sort of thing a status query should do.
+function connectOnly(endpoint: string): Promise<DaemonClient> {
+  return new Promise<DaemonClient>((resolve, reject) => {
+    const socket = net.connect(endpoint);
+    const onError = (error: Error) => {
+      socket.destroy();
+      reject(error);
+    };
+    socket.once('error', onError);
+    socket.once('connect', () => {
+      socket.removeListener('error', onError);
+      resolve(new DaemonClient(socket, endpoint));
+    });
+  });
+}
+
+function resolveDaemonOptions(options: DaemonOptions = {}):
+    ResolvedDaemonOptions {
+  const resolved = resolveStartOptions(options);
+  return {
+    ...resolved,
+    name: options.name,
+    endpoint: endpointFor({
+      ...resolved,
+      name: options.name,
+      endpoint: options.endpoint,
+    }),
+    idleTimeoutMs: options.idleTimeoutMs,
+    prewarm: options.prewarm,
+    logFile: options.logFile || process.env.SHOTIUM_DAEMON_LOG || null,
+  };
+}
+
+function spawnDaemon(options: ResolvedDaemonOptions): void {
+  const config = {
+    binary: options.binary,
+    workers: options.workers,
+    cacheDir: options.cacheDir,
+    args: options.args,
+    endpoint: options.endpoint,
+    idleTimeoutMs: options.idleTimeoutMs,
+    prewarm: options.prewarm,
+  };
+  const encoded =
+      Buffer.from(JSON.stringify(config), 'utf8').toString('base64');
+
+  // Detached, with the standard streams let go of: the daemon has to outlive
+  // the process that started it, and a child still holding this process's pipes
+  // would keep it from exiting -- the exact failure that makes a "background"
+  // daemon hang a shell.
+  let stdio: 'ignore'|['ignore', number, number] = 'ignore';
+  let logFd: number|null = null;
+  if (options.logFile) {
+    logFd = fs.openSync(options.logFile, 'a');
+    stdio = ['ignore', logFd, logFd];
+  }
+  const child = spawn(process.execPath, [DAEMON_MAIN, encoded], {
+    detached: true,
+    stdio,
+    windowsHide: true,
+  });
+  child.unref();
+  if (logFd !== null) {
+    fs.closeSync(logFd);
+  }
+}
+
+const sleep = (ms: number) =>
+    new Promise<void>((resolve) => setTimeout(resolve, ms));
+
+export interface EnsuredClient {
+  client: DaemonClient;
+  spawned: boolean;
+  endpoint: string;
+}
+
+// Connects, starting a daemon if none answers.
+//
+// The endpoint existing is the readiness signal, so this is a connect loop
+// rather than a handshake: a daemon that has bound can be talked to, and one
+// that has not is indistinguishable from one that was never started. Several
+// processes racing here is fine -- the losers' daemons exit on EADDRINUSE and
+// everyone ends up on the winner.
+async function ensureClient(options: DaemonOptions = {}):
+    Promise<EnsuredClient> {
+  const resolved = resolveDaemonOptions(options);
+  try {
+    const client = await connectOnly(resolved.endpoint);
+    return {client, spawned: false, endpoint: resolved.endpoint};
+  } catch {
+    if (options.spawn === false) {
+      throw new Error(`shotium: no daemon at ${resolved.endpoint}`);
+    }
+  }
+
+  spawnDaemon(resolved);
+  const deadline = Date.now() +
+      (options.startTimeoutMs === undefined ? START_TIMEOUT_MS :
+                                              options.startTimeoutMs);
+  for (;;) {
+    try {
+      const client = await connectOnly(resolved.endpoint);
+      return {client, spawned: true, endpoint: resolved.endpoint};
+    } catch {
+      if (Date.now() >= deadline) {
+        throw new Error(
+            `shotium: the daemon did not come up at ${resolved.endpoint}`);
+      }
+      await sleep(CONNECT_RETRY_MS);
+    }
+  }
+}
+
+// The five things a caller does with a daemon. Each opens a connection, does
+// one thing and closes it, which is the shape a short-lived process wants; a
+// service that will send more than one request calls connect() and keeps the
+// client.
+async function connect(options: DaemonOptions = {}): Promise<DaemonClient> {
+  const {client} = await ensureClient(options);
+  return client;
+}
+
+async function start(options: DaemonOptions = {}):
+    Promise<DaemonStatus&{spawned: boolean}> {
+  const {client, spawned, endpoint} = await ensureClient(options);
+  try {
+    const status = await client.status();
+    return {...status, endpoint, spawned};
+  } finally {
+    client.close();
+  }
+}
+
+async function status(options: DaemonOptions = {}):
+    Promise<Partial<DaemonStatus>&{running: boolean, endpoint: string}> {
+  const resolved = resolveDaemonOptions(options);
+  let client: DaemonClient;
+  try {
+    client = await connectOnly(resolved.endpoint);
+  } catch {
+    return {running: false, endpoint: resolved.endpoint};
+  }
+  try {
+    return {...(await client.status()), running: true};
+  } finally {
+    client.close();
+  }
+}
+
+async function stop(options: DaemonOptions = {}):
+    Promise<{stopped: boolean, endpoint: string}> {
+  const resolved = resolveDaemonOptions(options);
+  let client: DaemonClient;
+  try {
+    client = await connectOnly(resolved.endpoint);
+  } catch {
+    return {stopped: false, endpoint: resolved.endpoint};
+  }
+  try {
+    await client.shutdown();
+    return {stopped: true, endpoint: resolved.endpoint};
+  } finally {
+    client.close();
+  }
+}
+
+// One screenshot through the daemon, connection and all. `daemon` carries the
+// pool's configuration -- binary, workers, cache root -- and is stripped out
+// here rather than sent, because it says which daemon to talk to and not what
+// to photograph.
+async function screenshot(options: ScreenshotOptions&{daemon?: DaemonOptions}):
+    Promise<Buffer|null> {
+  const {daemon, ...rest} = options;
+  const client = await connect(daemon || {});
+  try {
+    return await client.screenshot(rest);
+  } finally {
+    client.close();
+  }
+}
+
+export {
+  DaemonClient,
+  connect,
+  ensureClient,
+  resolveDaemonOptions,
+  screenshot,
+  start,
+  status,
+  stop,
+};

package/src/lib/config.ts

@@ -0,0 +1,86 @@
+import os from 'node:os';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+import type {StartOptions} from '../types.js';
+
+import * as platform from './platform.js';
+
+// ESM has no __dirname. This is the same thing, from the module's own URL.
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+
+// StartOptions with every hole filled in. `cacheDir` is still nullable here
+// because null is an answer -- "no disk cache" -- and not an absent one.
+export interface ResolvedStartOptions {
+  binary: string;
+  workers: number;
+  cacheDir: string|null;
+  args: string[];
+}
+
+// The one place that decides what "no options" means.
+//
+// It is shared rather than duplicated because the daemon's address is a hash of
+// its configuration: if two callers filled in defaults even slightly
+// differently, one would compute an address no daemon is listening on and
+// start a second pool next to the first one that was already warm. See
+// endpoint.ts.
+//
+// Three places, in the order a caller means them: what they said, what npm
+// installed, and what they unpacked by hand. The middle one is the normal case
+// and the only one that needs no instructions.
+function defaultBinary(): string {
+  if (process.env.SHOTIUM_BINARY) {
+    return process.env.SHOTIUM_BINARY;
+  }
+  const dir = platform.packageDir();
+  if (dir) {
+    return path.join(dir, platform.binaryName());
+  }
+  // No platform package: an archive from the releases page, unpacked into
+  // bin/ beside this file. This is also the path a checkout takes, where
+  // nothing was installed from a registry at all.
+  return path.join(HERE, '..', 'bin', platform.binaryName());
+}
+
+// How many worker processes, when nobody said.
+//
+// Half the cores, capped. The cap is there because a worker is a process with
+// blink in it, not a thread: measured on this tree it settles around 14 MB of
+// private working set and holds a further ~30 MB of shotium.exe resident, so
+// "half the cores" on a 32-core machine is sixteen of them and most of a
+// gigabyte for a queue that is almost never sixteen deep. Four is past the
+// point where a screenshot workload gets much from another one -- the corpus
+// runs at 41 pages/s on four -- and anyone who has measured otherwise passes
+// `workers`.
+const MAXIMUM_DEFAULT_WORKERS = 4;
+
+function defaultWorkers(): number {
+  const half = Math.floor((os.cpus().length || 2) / 2);
+  return Math.max(1, Math.min(MAXIMUM_DEFAULT_WORKERS, half));
+}
+
+function defaultCacheDir(): string {
+  return path.join(os.tmpdir(), 'shotium-cache');
+}
+
+// binary / workers / cacheDir / args, filled in and normalised. `cacheDir:
+// null` survives as null -- it means "no disk cache", which is not the same
+// request as "use the default one".
+function resolveStartOptions(options: StartOptions = {}): ResolvedStartOptions {
+  return {
+    binary: options.binary || defaultBinary(),
+    workers: options.workers || defaultWorkers(),
+    cacheDir: options.cacheDir === null ?
+        null :
+        (options.cacheDir || defaultCacheDir()),
+    args: options.args || [],
+  };
+}
+
+export {
+  defaultBinary,
+  defaultCacheDir,
+  defaultWorkers,
+  resolveStartOptions,
+};