@shotkit/shotium 0.1.0 → 0.3.0

package/src/lib/cache.ts

@@ -0,0 +1,323 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import type {
+  CacheClearOptions,
+  CacheClearResult,
+  CacheEntry,
+  CacheTarget,
+} from '../types.js';
+
+import * as binding from './binding.js';
+import type {Engine as Handle} from './binding.js';
+import {cacheRoot, defaultCacheDir, normalizePath} from './config.js';
+
+/**
+ * Turns one glob into a regular expression over a URL.
+ *
+ * The dialect is the small one everybody already knows -- `*`, `**`, `?`,
+ * `{a,b}` -- and it is implemented here rather than depended on because this
+ * package has no runtime dependencies and a matcher is thirty lines. `*` stops
+ * at `/` and `**` does not, which is the distinction that makes
+ * `https://example.com/*` mean one level and `https://example.com/**` mean the
+ * site.
+ *
+ * Everything else is escaped, which matters more than usual here: the subjects
+ * are URLs, and a URL is mostly characters that mean something to a regular
+ * expression.
+ */
+function globToRegExp(pattern: string): RegExp {
+  let out = '';
+  for (let i = 0; i < pattern.length; i++) {
+    const c = pattern[i];
+    if (c === '*') {
+      if (pattern[i + 1] === '*') {
+        out += '.*';
+        i++;
+        // `/**/` should also match the zero-segment case, so that
+        // `https://x/**/y` matches `https://x/y`.
+        if (pattern[i + 1] === '/') {
+          out += '/?';
+          i++;
+        }
+      } else {
+        out += '[^/]*';
+      }
+    } else if (c === '?') {
+      out += '[^/]';
+    } else if (c === '{') {
+      const end = pattern.indexOf('}', i);
+      if (end === -1) {
+        out += '\\{';
+      } else {
+        const alternatives =
+            pattern.slice(i + 1, end).split(',').map(escapeLiteral);
+        out += `(?:${alternatives.join('|')})`;
+        i = end;
+      }
+    } else {
+      out += escapeLiteral(c);
+    }
+  }
+  return new RegExp(`^${out}$`);
+}
+
+function escapeLiteral(text: string): string {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/** Whether `url` matches any of `patterns`. No patterns matches nothing. */
+function matchesAny(url: string, patterns: RegExp[]): boolean {
+  return patterns.some((pattern) => pattern.test(url));
+}
+
+/**
+ * What a cache directory occupies, for the one path that reports a size
+ * without a backend to ask.
+ *
+ * The sum of the files rather than the sum of the entries, so it will differ
+ * from what `clear()` reports through the backend by the index and by whatever
+ * rounding the filesystem does. It is the honest number for "what is about to
+ * be deleted", which is what it is used for.
+ */
+function directorySize(dir: string): number {
+  let total = 0;
+  let names: fs.Dirent[] = [];
+  try {
+    names = fs.readdirSync(dir, {withFileTypes: true});
+  } catch {
+    return 0;
+  }
+  for (const entry of names) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      total += directorySize(full);
+      continue;
+    }
+    try {
+      total += fs.statSync(full).size;
+    } catch {
+      // Raced with something else clearing the same directory. Not an error:
+      // a file that is already gone contributes nothing to what is left.
+    }
+  }
+  return total;
+}
+
+/**
+ * Which directories an operation covers.
+ *
+ * `current` is this project's, `all` is every directory under the shared root,
+ * and anything else is taken as a project hash. `all` reads the root rather
+ * than remembering what it created: another process's directory is as much
+ * shotium's as this one's, and a caller asking to clear them all means the
+ * ones on disk.
+ */
+function resolveTargets(target: CacheTarget['target']): string[] {
+  if (target === 'all') {
+    const root = cacheRoot();
+    let names: string[] = [];
+    try {
+      names = fs.readdirSync(root);
+    } catch {
+      // No root means nothing has been cached yet, which is an empty list and
+      // not an error: a caller clearing an empty cache asked for a state that
+      // already holds.
+      return [];
+    }
+    return names.map((name) => normalizePath(path.join(root, name)))
+        .filter((dir) => {
+          try {
+            return fs.statSync(dir).isDirectory();
+          } catch {
+            return false;
+          }
+        });
+  }
+  if (target === undefined || target === 'current') {
+    return [defaultCacheDir()];
+  }
+  // A directory, if it looks like one. `start({cacheDir})` takes any path, so
+  // a caller who chose their own has to be able to name it here -- otherwise
+  // the cache they configured is the one cache these methods cannot see.
+  if (path.isAbsolute(target)) {
+    return [normalizePath(target)];
+  }
+  // Otherwise a project hash. Resolved against the root rather than used as a
+  // path, so that a relative string cannot reach outside it by accident.
+  return [normalizePath(path.join(cacheRoot(), target))];
+}
+
+/**
+ * The cache, from the outside.
+ *
+ * Every method takes the engine handle if there is one, and that is not an
+ * optimisation. Within one process a cache directory has one backend: asking
+ * for a second one on the directory the engine holds waits for the engine's to
+ * go away, which it will not do while the engine is up. Borrowing is the only
+ * thing that returns.
+ *
+ * "If there is one" means the process, not the lifecycle. `stop()` stands the
+ * engine down without tearing it down, so an engine that has been stopped
+ * still holds its directory and still has to be borrowed from -- which is also
+ * what makes the cache survive a stop, and outlive one, and be worth having.
+ *
+ * Across processes there is no such constraint -- several of them may share a
+ * directory and all of them cache.
+ *
+ * The engine is fetched through a callback rather than held, because this
+ * object is built once at import time and the engine comes and goes.
+ */
+export class Cache {
+  constructor(private readonly engineHandle: () => Handle | null) {}
+
+  /**
+   * This project's cache directory, absolute and with forward slashes.
+   *
+   * It exists whether or not anything has been written to it -- the answer is
+   * "where the cache goes", not "where a cache is".
+   */
+  getDir(options: CacheTarget = {}): string {
+    const targets = resolveTargets(options.target);
+    return targets.length > 0 ? targets[0] : defaultCacheDir();
+  }
+
+  /** Every directory the target names. `all` can be several; the rest, one. */
+  getDirs(options: CacheTarget = {}): string[] {
+    return resolveTargets(options.target);
+  }
+
+  /**
+   * What the cache is holding, by URL.
+   *
+   * Named `getFiles` for the operation callers reach for, and deliberately not
+   * returning filenames: the files in a cache directory are called things like
+   * `5349fbae98c6d9a1_0`, because the name is a hash of the entry key. A list
+   * of those answers no question anybody has. The URLs are what the entries
+   * are, and they are what `clear({glob})` matches against.
+   *
+   * This opens every entry to read its key and size, so it is a diagnostic
+   * rather than something to put on a request path.
+   */
+  async getFiles(options: CacheTarget = {}): Promise<CacheEntry[]> {
+    const native = binding.load();
+    const entries: CacheEntry[] = [];
+    for (const dir of resolveTargets(options.target)) {
+      if (!fs.existsSync(dir)) {
+        continue;
+      }
+      const json = await native.cache(
+          this.handleFor(), /*clearing=*/ false, JSON.stringify({
+            cacheDir: dir,
+          }));
+      const listed = JSON.parse(json) as Array<Omit<CacheEntry, 'dir'>>;
+      for (const entry of listed) {
+        entries.push({...entry, dir});
+      }
+    }
+    return entries;
+  }
+
+  /**
+   * Removes what the options select. With no options, everything.
+   *
+   * The three filters compose, and `glob` is applied here rather than in the
+   * engine: the entries are listed, their URLs are matched, and the ones that
+   * matched are what the engine is asked to remove. That keeps the pattern
+   * dialect in the layer whose users have opinions about pattern dialects, and
+   * keeps the engine's interface to exact URLs.
+   *
+   * Removal goes through the cache backend, never through the filesystem.
+   * Deleting the files directly would leave the backend's index naming entries
+   * that are no longer there, and the next process to open the directory
+   * either rebuilds the index from disk or, having found it inconsistent,
+   * discards it. That is the difference between clearing a cache and
+   * corrupting one.
+   */
+  async clear(options: CacheClearOptions = {}): Promise<CacheClearResult[]> {
+    const native = binding.load();
+    const patterns = (options.glob ?? []).map(globToRegExp);
+    const results: CacheClearResult[] = [];
+
+    // Clearing everything, in a process that has no engine at all: remove the
+    // directory.
+    //
+    // This is the one case where touching the filesystem is correct rather
+    // than reckless. The danger in deleting cache files by hand is a partial
+    // delete -- an index left naming entries that are gone -- and there is no
+    // such thing when the index goes with them. What is left is a directory
+    // that does not exist, which is exactly what an empty cache looks like
+    // before anything has written to it.
+    //
+    // It is also the fast path a short script gets: emptying a cache without
+    // starting Blink to do it costs a few milliseconds instead of the tens
+    // that building an engine does.
+    const unfiltered = patterns.length === 0 && !options.maxAge &&
+        !options.maxSize;
+    if (unfiltered && !this.handleFor()) {
+      for (const dir of resolveTargets(options.target)) {
+        const before = directorySize(dir);
+        fs.rmSync(dir, {recursive: true, force: true});
+        results.push(
+            {removed: -1, bytesBefore: before, bytesAfter: 0, dir});
+      }
+      return results;
+    }
+
+    for (const dir of resolveTargets(options.target)) {
+      if (!fs.existsSync(dir)) {
+        continue;
+      }
+      const request: Record<string, unknown> = {cacheDir: dir};
+
+      if (patterns.length > 0) {
+        const json = await native.cache(
+            this.handleFor(), /*clearing=*/ false,
+            JSON.stringify({cacheDir: dir}));
+        const listed = JSON.parse(json) as Array<Omit<CacheEntry, 'dir'>>;
+        const urls =
+            listed.filter((entry) => matchesAny(entry.url, patterns))
+                .map((entry) => entry.url);
+        // Nothing matched, so nothing is asked for. Falling through with an
+        // empty `urls` would be read by the engine as "no URL filter", which
+        // combined with no other filter empties the directory -- the opposite
+        // of what a pattern that matched nothing means.
+        if (urls.length === 0 && options.maxAge === undefined &&
+            options.maxSize === undefined) {
+          results.push({removed: 0, bytesBefore: 0, bytesAfter: 0, dir});
+          continue;
+        }
+        request.urls = urls;
+      }
+
+      if (options.maxAge) {
+        request.unusedSinceMs = Date.now() - options.maxAge * 1000;
+      }
+      if (options.maxSize) {
+        request.maxBytes = options.maxSize;
+      }
+
+      const json = await native.cache(
+          this.handleFor(), /*clearing=*/ true, JSON.stringify(request));
+      results.push({
+        ...(JSON.parse(json) as Omit<CacheClearResult, 'dir'>),
+        dir,
+      });
+    }
+    return results;
+  }
+
+  /**
+   * The engine handle, when there is an engine.
+   *
+   * Passed for every directory and not only the engine's own. It is never
+   * wrong to pass it -- the engine's thread can open any directory, and for
+   * the one it already has open, borrowing its backend is the only thing that
+   * returns. It is passing `null` while an engine is up that hangs, which is
+   * why this is conditional on neither the directory asked for nor on whether
+   * the engine is currently accepting captures.
+   */
+  private handleFor(): Handle|null {
+    return this.engineHandle();
+  }
+}

package/src/lib/client.ts

@@ -6,12 +6,15 @@ import path from 'node:path';
 import {fileURLToPath} from 'node:url';
 
 import type {
+  CaptureStats,
   DaemonOptions,
   DaemonStatus,
   ScreenshotOptions,
+  ScreenshotResult,
 } from '../types.js';
 
 import {resolveStartOptions} from './config.js';
+import {emptyStats} from './engine.js';
 import {endpointFor} from './endpoint.js';
 import {FrameReader, encodeFrame} from './protocol.js';
 import {timeoutFor, toRequest} from './request.js';
@@ -35,6 +38,10 @@ interface ClientReply {
   ok?: boolean;
   error?: string;
   path?: string;
+  // The daemon reports the same CaptureStats the in-process engine does, in
+  // its response header. It rides on the failure header too, which is why the
+  // rejection below carries it.
+  stats?: CaptureStats;
 }
 
 interface ClientResult {
@@ -48,10 +55,9 @@ interface Pending {
 }
 
 interface ResolvedDaemonOptions {
-  binary: string;
-  workers: number;
   cacheDir: string|null;
-  args: string[];
+  userAgent?: string;
+  resourceDir?: string;
   name: string|undefined;
   endpoint: string;
   idleTimeoutMs: number|undefined;
@@ -61,11 +67,11 @@ interface ResolvedDaemonOptions {
 
 // The client half of the resident daemon.
 //
-// One connection can carry several requests at once, which is the difference
-// between this and the worker protocol underneath: every message carries an
+// One connection can carry several requests at once: every message carries an
 // `id` and the answers are matched back by it, so a caller can fire ten
-// screenshots down one socket and let the pool on the other side spread them
-// across workers.
+// screenshots down one socket without waiting between them. They still come
+// back one at a time -- there is one renderer on the other side -- so this
+// saves the round trips, not the renders.
 class DaemonClient extends EventEmitter {
   private readonly socket: net.Socket;
   private readonly endpointPath: string;
@@ -127,7 +133,14 @@ class DaemonClient extends EventEmitter {
     if (header.ok) {
       pending.resolve({header, image: header.path ? null : payload});
     } else {
-      pending.reject(new Error(header.error || 'shotium: request failed'));
+      const error = new Error(header.error || 'shotium: request failed');
+      // Attached rather than dropped: a capture that failed part of the way
+      // through has already measured what it did, and that is usually the
+      // explanation. The in-process engine does the same.
+      if (header.stats) {
+        (error as Error & {stats?: CaptureStats}).stats = header.stats;
+      }
+      pending.reject(error);
     }
   }
 
@@ -152,17 +165,20 @@ class DaemonClient extends EventEmitter {
     });
   }
 
-  /** Resolves to the image, or to null when `path` was given. */
-  async screenshot(options: ScreenshotOptions): Promise<Buffer|null> {
+  /**
+   * One screenshot, and what taking it cost.
+   *
+   * The same shape the in-process engine returns, so that moving a program
+   * between the two is an import change and nothing else.
+   */
+  async screenshot(options: ScreenshotOptions): Promise<ScreenshotResult> {
     const request = toRequest(options);
-    const retry = typeof options.retry === 'number' ? options.retry : 0;
     const result = await this.send({
       op: 'screenshot',
       request,
       timeout: timeoutFor(options),
-      retry,
     });
-    return result.image;
+    return {image: result.image, stats: result.header.stats ?? emptyStats()};
   }
 
   async status(): Promise<DaemonStatus> {
@@ -219,10 +235,9 @@ function resolveDaemonOptions(options: DaemonOptions = {}):
 
 function spawnDaemon(options: ResolvedDaemonOptions): void {
   const config = {
-    binary: options.binary,
-    workers: options.workers,
     cacheDir: options.cacheDir,
-    args: options.args,
+    userAgent: options.userAgent,
+    resourceDir: options.resourceDir,
     endpoint: options.endpoint,
     idleTimeoutMs: options.idleTimeoutMs,
     prewarm: options.prewarm,
@@ -355,7 +370,7 @@ async function stop(options: DaemonOptions = {}):
 // here rather than sent, because it says which daemon to talk to and not what
 // to photograph.
 async function screenshot(options: ScreenshotOptions&{daemon?: DaemonOptions}):
-    Promise<Buffer|null> {
+    Promise<ScreenshotResult> {
   const {daemon, ...rest} = options;
   const client = await connect(daemon || {});
   try {

package/src/lib/config.ts

@@ -1,86 +1,138 @@
+import crypto from 'node:crypto';
+import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
-import {fileURLToPath} from 'node:url';
 
 import type {StartOptions} from '../types.js';
 
-import * as platform from './platform.js';
-
-// ESM has no __dirname. This is the same thing, from the module's own URL.
-const HERE = path.dirname(fileURLToPath(import.meta.url));
-
 // StartOptions with every hole filled in. `cacheDir` is still nullable here
 // because null is an answer -- "no disk cache" -- and not an absent one.
 export interface ResolvedStartOptions {
-  binary: string;
-  workers: number;
   cacheDir: string|null;
-  args: string[];
+  cacheMaxBytes: number;
+  userAgent?: string;
+  resourceDir?: string;
 }
 
-// The one place that decides what "no options" means.
-//
-// It is shared rather than duplicated because the daemon's address is a hash of
-// its configuration: if two callers filled in defaults even slightly
-// differently, one would compute an address no daemon is listening on and
-// start a second pool next to the first one that was already warm. See
-// endpoint.ts.
+// One number, chosen rather than delegated.
 //
-// Three places, in the order a caller means them: what they said, what npm
-// installed, and what they unpacked by hand. The middle one is the normal case
-// and the only one that needs no instructions.
-function defaultBinary(): string {
-  if (process.env.SHOTIUM_BINARY) {
-    return process.env.SHOTIUM_BINARY;
-  }
-  const dir = platform.packageDir();
-  if (dir) {
-    return path.join(dir, platform.binaryName());
+// Passing 0 hands the decision to the disk cache backend, which sizes itself
+// against the volume's free space -- a defensible default for a browser
+// profile the user knows about, and a poor one for a directory that appears
+// under ~/.shotium because somebody imported a library. 256 MB holds a large
+// corpus of pages and their fonts, and is small enough that nobody has to
+// think about it.
+const DEFAULT_CACHE_MAX_BYTES = 256 * 1024 * 1024;
+
+/**
+ * One spelling of a path: absolute, with forward slashes.
+ *
+ * Every path this module hands back goes through here. On Windows the two
+ * separators are interchangeable to the filesystem and not to a caller
+ * comparing strings or writing a glob, and a library that returns whichever
+ * one `path.join` happened to produce makes that the caller's problem.
+ */
+export function normalizePath(target: string): string {
+  return path.resolve(target).replace(/\\/g, '/');
+}
+
+/**
+ * The project the current process belongs to: the nearest directory at or
+ * above the working directory that has a package.json.
+ *
+ * The working directory itself would be the obvious key and is the wrong one.
+ * It moves -- `process.chdir`, or a script run from a subdirectory -- and each
+ * value it takes would get a cache of its own, so a project would slowly
+ * accumulate directories that each know a third of its pages. The package root
+ * is the thing that stays put.
+ *
+ * Falls back to the working directory when there is no package.json above it,
+ * which is what a bare script has and is still better than nothing: it is at
+ * least stable for as long as the script runs from one place.
+ */
+function projectRoot(): string {
+  let dir = process.cwd();
+  for (;;) {
+    if (fs.existsSync(path.join(dir, 'package.json'))) {
+      return dir;
+    }
+    const parent = path.dirname(dir);
+    if (parent === dir) {
+      return process.cwd();
+    }
+    dir = parent;
   }
-  // No platform package: an archive from the releases page, unpacked into
-  // bin/ beside this file. This is also the path a checkout takes, where
-  // nothing was installed from a registry at all.
-  return path.join(HERE, '..', 'bin', platform.binaryName());
 }
 
-// How many worker processes, when nobody said.
-//
-// Half the cores, capped. The cap is there because a worker is a process with
-// blink in it, not a thread: measured on this tree it settles around 14 MB of
-// private working set and holds a further ~30 MB of shotium.exe resident, so
-// "half the cores" on a 32-core machine is sixteen of them and most of a
-// gigabyte for a queue that is almost never sixteen deep. Four is past the
-// point where a screenshot workload gets much from another one -- the corpus
-// runs at 41 pages/s on four -- and anyone who has measured otherwise passes
-// `workers`.
-const MAXIMUM_DEFAULT_WORKERS = 4;
+/**
+ * Where every shotium cache directory lives. One level up from any single
+ * project's, which is what makes `target: 'all'` answerable.
+ *
+ * Under the home directory and not the temporary one, which is where this was
+ * until 0.3 was cut. $TMPDIR is defined by not surviving: /tmp is emptied on
+ * reboot, systemd-tmpfiles removes anything untouched for ten days, and macOS
+ * sweeps it on a schedule of its own. The entire value of an HTTP cache is the
+ * *next* run, so a default that lives somewhere designed to be cleared is a
+ * cache that stops working at exactly the moment it would have started paying
+ * for itself.
+ *
+ * `~/.shotium`, spelled the same on every platform. One place a user can look
+ * for it, one path to say in a bug report, and one directory to delete.
+ *
+ * $TMPDIR remains only as a fallback for a process with no home to speak of --
+ * some containers, some service accounts. That is a degradation and not a
+ * second location: there is no home directory holding a cache that would
+ * otherwise have been found.
+ */
+function shotiumHome(): string {
+  return path.join(os.homedir() || os.tmpdir(), '.shotium');
+}
+
+export function cacheRoot(): string {
+  return normalizePath(path.join(shotiumHome(), 'cache'));
+}
 
-function defaultWorkers(): number {
-  const half = Math.floor((os.cpus().length || 2) / 2);
-  return Math.max(1, Math.min(MAXIMUM_DEFAULT_WORKERS, half));
+/**
+ * The identifier for a project's cache directory: a hash of its root path.
+ *
+ * A hash rather than the path itself because the path contains separators,
+ * drive letters and whatever the user called their directory, none of which
+ * survive being a directory name. It is not a security measure and does not
+ * need to be one -- it is a fixed-length name for a variable-length string.
+ */
+export function projectKey(root: string = projectRoot()): string {
+  return crypto.createHash('sha1').update(normalizePath(root)).digest('hex');
 }
 
-function defaultCacheDir(): string {
-  return path.join(os.tmpdir(), 'shotium-cache');
+/** This project's cache directory. */
+export function defaultCacheDir(): string {
+  return normalizePath(path.join(cacheRoot(), projectKey()));
 }
 
-// binary / workers / cacheDir / args, filled in and normalised. `cacheDir:
-// null` survives as null -- it means "no disk cache", which is not the same
-// request as "use the default one".
+// The one place that decides what "no options" means.
+//
+// It is shared rather than duplicated because the daemon's address is a hash of
+// its configuration: if two callers filled in defaults even slightly
+// differently, one would compute an address no daemon is listening on and
+// start a second engine next to the first one that was already warm. See
+// endpoint.ts.
+//
+// `cacheDir` defaults to this project's directory rather than to null, which
+// is the reverse of 0.2. The reason is measured: without a cache every capture
+// of an `https:` URL pays DNS, TLS and a round trip, which for a small page is
+// most of the wall clock and all of the surprise. The objection to a default
+// -- that a short-lived program leaves a directory behind -- is answered by
+// the directory being per-project, size-capped, and somewhere the platform's
+// own tooling knows how to clear, rather than by there being no cache.
+// `cacheDir: null` still turns it off.
 function resolveStartOptions(options: StartOptions = {}): ResolvedStartOptions {
   return {
-    binary: options.binary || defaultBinary(),
-    workers: options.workers || defaultWorkers(),
-    cacheDir: options.cacheDir === null ?
-        null :
-        (options.cacheDir || defaultCacheDir()),
-    args: options.args || [],
+    cacheDir: options.cacheDir === null ? null :
+                                          (options.cacheDir ?? defaultCacheDir()),
+    cacheMaxBytes: options.cacheMaxBytes ?? DEFAULT_CACHE_MAX_BYTES,
+    userAgent: options.userAgent,
+    resourceDir: options.resourceDir,
   };
 }
 
-export {
-  defaultBinary,
-  defaultCacheDir,
-  defaultWorkers,
-  resolveStartOptions,
-};
+export {DEFAULT_CACHE_MAX_BYTES, resolveStartOptions};