@shotkit/shotium 0.2.0 → 0.3.0

package/src/lib/binding.ts

@@ -17,12 +17,33 @@ const HERE = path.dirname(fileURLToPath(import.meta.url));
  */
 export type Engine = unknown;
 
+/** One capture's answer, as the addon hands it over. */
+export interface NativeCapture {
+  image: Buffer;
+  /**
+   * CaptureStats as JSON, unparsed. The addon carries JSON between the engine
+   * and this layer without reading it -- anything it understood would be a
+   * third opinion about the shape, and the third opinion is the one that
+   * drifts. Undefined when the engine reported none.
+   */
+  stats?: string;
+}
+
 /** What native/binding.cc exports. See shot/shot_api.h for the C ABI. */
 export interface NativeBinding {
   create(optionsJson: string): Engine;
   destroy(engine: Engine): void;
   purge(engine: Engine, releaseWorkingSet: boolean): void;
-  capture(engine: Engine, requestJson: string): Promise<Buffer>;
+  status(engine: Engine): string;
+  capture(engine: Engine, requestJson: string): Promise<NativeCapture>;
+  /**
+   * List or clear a cache directory. `engine` is nullable and that is the
+   * interface: with one, the operation runs on the engine's thread and borrows
+   * the backend it already holds; without one, the library opens the directory
+   * itself. Resolves to JSON.
+   */
+  cache(engine: Engine|null, clearing: boolean, optionsJson: string):
+      Promise<string>;
 }
 
 // Where the addon and the library beside it live.

package/src/lib/cache.ts

@@ -0,0 +1,323 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import type {
+  CacheClearOptions,
+  CacheClearResult,
+  CacheEntry,
+  CacheTarget,
+} from '../types.js';
+
+import * as binding from './binding.js';
+import type {Engine as Handle} from './binding.js';
+import {cacheRoot, defaultCacheDir, normalizePath} from './config.js';
+
+/**
+ * Turns one glob into a regular expression over a URL.
+ *
+ * The dialect is the small one everybody already knows -- `*`, `**`, `?`,
+ * `{a,b}` -- and it is implemented here rather than depended on because this
+ * package has no runtime dependencies and a matcher is thirty lines. `*` stops
+ * at `/` and `**` does not, which is the distinction that makes
+ * `https://example.com/*` mean one level and `https://example.com/**` mean the
+ * site.
+ *
+ * Everything else is escaped, which matters more than usual here: the subjects
+ * are URLs, and a URL is mostly characters that mean something to a regular
+ * expression.
+ */
+function globToRegExp(pattern: string): RegExp {
+  let out = '';
+  for (let i = 0; i < pattern.length; i++) {
+    const c = pattern[i];
+    if (c === '*') {
+      if (pattern[i + 1] === '*') {
+        out += '.*';
+        i++;
+        // `/**/` should also match the zero-segment case, so that
+        // `https://x/**/y` matches `https://x/y`.
+        if (pattern[i + 1] === '/') {
+          out += '/?';
+          i++;
+        }
+      } else {
+        out += '[^/]*';
+      }
+    } else if (c === '?') {
+      out += '[^/]';
+    } else if (c === '{') {
+      const end = pattern.indexOf('}', i);
+      if (end === -1) {
+        out += '\\{';
+      } else {
+        const alternatives =
+            pattern.slice(i + 1, end).split(',').map(escapeLiteral);
+        out += `(?:${alternatives.join('|')})`;
+        i = end;
+      }
+    } else {
+      out += escapeLiteral(c);
+    }
+  }
+  return new RegExp(`^${out}$`);
+}
+
+function escapeLiteral(text: string): string {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/** Whether `url` matches any of `patterns`. No patterns matches nothing. */
+function matchesAny(url: string, patterns: RegExp[]): boolean {
+  return patterns.some((pattern) => pattern.test(url));
+}
+
+/**
+ * What a cache directory occupies, for the one path that reports a size
+ * without a backend to ask.
+ *
+ * The sum of the files rather than the sum of the entries, so it will differ
+ * from what `clear()` reports through the backend by the index and by whatever
+ * rounding the filesystem does. It is the honest number for "what is about to
+ * be deleted", which is what it is used for.
+ */
+function directorySize(dir: string): number {
+  let total = 0;
+  let names: fs.Dirent[] = [];
+  try {
+    names = fs.readdirSync(dir, {withFileTypes: true});
+  } catch {
+    return 0;
+  }
+  for (const entry of names) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      total += directorySize(full);
+      continue;
+    }
+    try {
+      total += fs.statSync(full).size;
+    } catch {
+      // Raced with something else clearing the same directory. Not an error:
+      // a file that is already gone contributes nothing to what is left.
+    }
+  }
+  return total;
+}
+
+/**
+ * Which directories an operation covers.
+ *
+ * `current` is this project's, `all` is every directory under the shared root,
+ * and anything else is taken as a project hash. `all` reads the root rather
+ * than remembering what it created: another process's directory is as much
+ * shotium's as this one's, and a caller asking to clear them all means the
+ * ones on disk.
+ */
+function resolveTargets(target: CacheTarget['target']): string[] {
+  if (target === 'all') {
+    const root = cacheRoot();
+    let names: string[] = [];
+    try {
+      names = fs.readdirSync(root);
+    } catch {
+      // No root means nothing has been cached yet, which is an empty list and
+      // not an error: a caller clearing an empty cache asked for a state that
+      // already holds.
+      return [];
+    }
+    return names.map((name) => normalizePath(path.join(root, name)))
+        .filter((dir) => {
+          try {
+            return fs.statSync(dir).isDirectory();
+          } catch {
+            return false;
+          }
+        });
+  }
+  if (target === undefined || target === 'current') {
+    return [defaultCacheDir()];
+  }
+  // A directory, if it looks like one. `start({cacheDir})` takes any path, so
+  // a caller who chose their own has to be able to name it here -- otherwise
+  // the cache they configured is the one cache these methods cannot see.
+  if (path.isAbsolute(target)) {
+    return [normalizePath(target)];
+  }
+  // Otherwise a project hash. Resolved against the root rather than used as a
+  // path, so that a relative string cannot reach outside it by accident.
+  return [normalizePath(path.join(cacheRoot(), target))];
+}
+
+/**
+ * The cache, from the outside.
+ *
+ * Every method takes the engine handle if there is one, and that is not an
+ * optimisation. Within one process a cache directory has one backend: asking
+ * for a second one on the directory the engine holds waits for the engine's to
+ * go away, which it will not do while the engine is up. Borrowing is the only
+ * thing that returns.
+ *
+ * "If there is one" means the process, not the lifecycle. `stop()` stands the
+ * engine down without tearing it down, so an engine that has been stopped
+ * still holds its directory and still has to be borrowed from -- which is also
+ * what makes the cache survive a stop, and outlive one, and be worth having.
+ *
+ * Across processes there is no such constraint -- several of them may share a
+ * directory and all of them cache.
+ *
+ * The engine is fetched through a callback rather than held, because this
+ * object is built once at import time and the engine comes and goes.
+ */
+export class Cache {
+  constructor(private readonly engineHandle: () => Handle | null) {}
+
+  /**
+   * This project's cache directory, absolute and with forward slashes.
+   *
+   * It exists whether or not anything has been written to it -- the answer is
+   * "where the cache goes", not "where a cache is".
+   */
+  getDir(options: CacheTarget = {}): string {
+    const targets = resolveTargets(options.target);
+    return targets.length > 0 ? targets[0] : defaultCacheDir();
+  }
+
+  /** Every directory the target names. `all` can be several; the rest, one. */
+  getDirs(options: CacheTarget = {}): string[] {
+    return resolveTargets(options.target);
+  }
+
+  /**
+   * What the cache is holding, by URL.
+   *
+   * Named `getFiles` for the operation callers reach for, and deliberately not
+   * returning filenames: the files in a cache directory are called things like
+   * `5349fbae98c6d9a1_0`, because the name is a hash of the entry key. A list
+   * of those answers no question anybody has. The URLs are what the entries
+   * are, and they are what `clear({glob})` matches against.
+   *
+   * This opens every entry to read its key and size, so it is a diagnostic
+   * rather than something to put on a request path.
+   */
+  async getFiles(options: CacheTarget = {}): Promise<CacheEntry[]> {
+    const native = binding.load();
+    const entries: CacheEntry[] = [];
+    for (const dir of resolveTargets(options.target)) {
+      if (!fs.existsSync(dir)) {
+        continue;
+      }
+      const json = await native.cache(
+          this.handleFor(), /*clearing=*/ false, JSON.stringify({
+            cacheDir: dir,
+          }));
+      const listed = JSON.parse(json) as Array<Omit<CacheEntry, 'dir'>>;
+      for (const entry of listed) {
+        entries.push({...entry, dir});
+      }
+    }
+    return entries;
+  }
+
+  /**
+   * Removes what the options select. With no options, everything.
+   *
+   * The three filters compose, and `glob` is applied here rather than in the
+   * engine: the entries are listed, their URLs are matched, and the ones that
+   * matched are what the engine is asked to remove. That keeps the pattern
+   * dialect in the layer whose users have opinions about pattern dialects, and
+   * keeps the engine's interface to exact URLs.
+   *
+   * Removal goes through the cache backend, never through the filesystem.
+   * Deleting the files directly would leave the backend's index naming entries
+   * that are no longer there, and the next process to open the directory
+   * either rebuilds the index from disk or, having found it inconsistent,
+   * discards it. That is the difference between clearing a cache and
+   * corrupting one.
+   */
+  async clear(options: CacheClearOptions = {}): Promise<CacheClearResult[]> {
+    const native = binding.load();
+    const patterns = (options.glob ?? []).map(globToRegExp);
+    const results: CacheClearResult[] = [];
+
+    // Clearing everything, in a process that has no engine at all: remove the
+    // directory.
+    //
+    // This is the one case where touching the filesystem is correct rather
+    // than reckless. The danger in deleting cache files by hand is a partial
+    // delete -- an index left naming entries that are gone -- and there is no
+    // such thing when the index goes with them. What is left is a directory
+    // that does not exist, which is exactly what an empty cache looks like
+    // before anything has written to it.
+    //
+    // It is also the fast path a short script gets: emptying a cache without
+    // starting Blink to do it costs a few milliseconds instead of the tens
+    // that building an engine does.
+    const unfiltered = patterns.length === 0 && !options.maxAge &&
+        !options.maxSize;
+    if (unfiltered && !this.handleFor()) {
+      for (const dir of resolveTargets(options.target)) {
+        const before = directorySize(dir);
+        fs.rmSync(dir, {recursive: true, force: true});
+        results.push(
+            {removed: -1, bytesBefore: before, bytesAfter: 0, dir});
+      }
+      return results;
+    }
+
+    for (const dir of resolveTargets(options.target)) {
+      if (!fs.existsSync(dir)) {
+        continue;
+      }
+      const request: Record<string, unknown> = {cacheDir: dir};
+
+      if (patterns.length > 0) {
+        const json = await native.cache(
+            this.handleFor(), /*clearing=*/ false,
+            JSON.stringify({cacheDir: dir}));
+        const listed = JSON.parse(json) as Array<Omit<CacheEntry, 'dir'>>;
+        const urls =
+            listed.filter((entry) => matchesAny(entry.url, patterns))
+                .map((entry) => entry.url);
+        // Nothing matched, so nothing is asked for. Falling through with an
+        // empty `urls` would be read by the engine as "no URL filter", which
+        // combined with no other filter empties the directory -- the opposite
+        // of what a pattern that matched nothing means.
+        if (urls.length === 0 && options.maxAge === undefined &&
+            options.maxSize === undefined) {
+          results.push({removed: 0, bytesBefore: 0, bytesAfter: 0, dir});
+          continue;
+        }
+        request.urls = urls;
+      }
+
+      if (options.maxAge) {
+        request.unusedSinceMs = Date.now() - options.maxAge * 1000;
+      }
+      if (options.maxSize) {
+        request.maxBytes = options.maxSize;
+      }
+
+      const json = await native.cache(
+          this.handleFor(), /*clearing=*/ true, JSON.stringify(request));
+      results.push({
+        ...(JSON.parse(json) as Omit<CacheClearResult, 'dir'>),
+        dir,
+      });
+    }
+    return results;
+  }
+
+  /**
+   * The engine handle, when there is an engine.
+   *
+   * Passed for every directory and not only the engine's own. It is never
+   * wrong to pass it -- the engine's thread can open any directory, and for
+   * the one it already has open, borrowing its backend is the only thing that
+   * returns. It is passing `null` while an engine is up that hangs, which is
+   * why this is conditional on neither the directory asked for nor on whether
+   * the engine is currently accepting captures.
+   */
+  private handleFor(): Handle|null {
+    return this.engineHandle();
+  }
+}

package/src/lib/client.ts

@@ -6,12 +6,15 @@ import path from 'node:path';
 import {fileURLToPath} from 'node:url';
 
 import type {
+  CaptureStats,
   DaemonOptions,
   DaemonStatus,
   ScreenshotOptions,
+  ScreenshotResult,
 } from '../types.js';
 
 import {resolveStartOptions} from './config.js';
+import {emptyStats} from './engine.js';
 import {endpointFor} from './endpoint.js';
 import {FrameReader, encodeFrame} from './protocol.js';
 import {timeoutFor, toRequest} from './request.js';
@@ -35,6 +38,10 @@ interface ClientReply {
   ok?: boolean;
   error?: string;
   path?: string;
+  // The daemon reports the same CaptureStats the in-process engine does, in
+  // its response header. It rides on the failure header too, which is why the
+  // rejection below carries it.
+  stats?: CaptureStats;
 }
 
 interface ClientResult {
@@ -126,7 +133,14 @@ class DaemonClient extends EventEmitter {
     if (header.ok) {
       pending.resolve({header, image: header.path ? null : payload});
     } else {
-      pending.reject(new Error(header.error || 'shotium: request failed'));
+      const error = new Error(header.error || 'shotium: request failed');
+      // Attached rather than dropped: a capture that failed part of the way
+      // through has already measured what it did, and that is usually the
+      // explanation. The in-process engine does the same.
+      if (header.stats) {
+        (error as Error & {stats?: CaptureStats}).stats = header.stats;
+      }
+      pending.reject(error);
     }
   }
 
@@ -151,15 +165,20 @@ class DaemonClient extends EventEmitter {
     });
   }
 
-  /** Resolves to the image, or to null when `path` was given. */
-  async screenshot(options: ScreenshotOptions): Promise<Buffer|null> {
+  /**
+   * One screenshot, and what taking it cost.
+   *
+   * The same shape the in-process engine returns, so that moving a program
+   * between the two is an import change and nothing else.
+   */
+  async screenshot(options: ScreenshotOptions): Promise<ScreenshotResult> {
     const request = toRequest(options);
     const result = await this.send({
       op: 'screenshot',
       request,
       timeout: timeoutFor(options),
     });
-    return result.image;
+    return {image: result.image, stats: result.header.stats ?? emptyStats()};
   }
 
   async status(): Promise<DaemonStatus> {
@@ -351,7 +370,7 @@ async function stop(options: DaemonOptions = {}):
 // here rather than sent, because it says which daemon to talk to and not what
 // to photograph.
 async function screenshot(options: ScreenshotOptions&{daemon?: DaemonOptions}):
-    Promise<Buffer|null> {
+    Promise<ScreenshotResult> {
   const {daemon, ...rest} = options;
   const client = await connect(daemon || {});
   try {

package/src/lib/config.ts

@@ -1,13 +1,114 @@
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
 import type {StartOptions} from '../types.js';
 
 // StartOptions with every hole filled in. `cacheDir` is still nullable here
 // because null is an answer -- "no disk cache" -- and not an absent one.
 export interface ResolvedStartOptions {
   cacheDir: string|null;
+  cacheMaxBytes: number;
   userAgent?: string;
   resourceDir?: string;
 }
 
+// One number, chosen rather than delegated.
+//
+// Passing 0 hands the decision to the disk cache backend, which sizes itself
+// against the volume's free space -- a defensible default for a browser
+// profile the user knows about, and a poor one for a directory that appears
+// under ~/.shotium because somebody imported a library. 256 MB holds a large
+// corpus of pages and their fonts, and is small enough that nobody has to
+// think about it.
+const DEFAULT_CACHE_MAX_BYTES = 256 * 1024 * 1024;
+
+/**
+ * One spelling of a path: absolute, with forward slashes.
+ *
+ * Every path this module hands back goes through here. On Windows the two
+ * separators are interchangeable to the filesystem and not to a caller
+ * comparing strings or writing a glob, and a library that returns whichever
+ * one `path.join` happened to produce makes that the caller's problem.
+ */
+export function normalizePath(target: string): string {
+  return path.resolve(target).replace(/\\/g, '/');
+}
+
+/**
+ * The project the current process belongs to: the nearest directory at or
+ * above the working directory that has a package.json.
+ *
+ * The working directory itself would be the obvious key and is the wrong one.
+ * It moves -- `process.chdir`, or a script run from a subdirectory -- and each
+ * value it takes would get a cache of its own, so a project would slowly
+ * accumulate directories that each know a third of its pages. The package root
+ * is the thing that stays put.
+ *
+ * Falls back to the working directory when there is no package.json above it,
+ * which is what a bare script has and is still better than nothing: it is at
+ * least stable for as long as the script runs from one place.
+ */
+function projectRoot(): string {
+  let dir = process.cwd();
+  for (;;) {
+    if (fs.existsSync(path.join(dir, 'package.json'))) {
+      return dir;
+    }
+    const parent = path.dirname(dir);
+    if (parent === dir) {
+      return process.cwd();
+    }
+    dir = parent;
+  }
+}
+
+/**
+ * Where every shotium cache directory lives. One level up from any single
+ * project's, which is what makes `target: 'all'` answerable.
+ *
+ * Under the home directory and not the temporary one, which is where this was
+ * until 0.3 was cut. $TMPDIR is defined by not surviving: /tmp is emptied on
+ * reboot, systemd-tmpfiles removes anything untouched for ten days, and macOS
+ * sweeps it on a schedule of its own. The entire value of an HTTP cache is the
+ * *next* run, so a default that lives somewhere designed to be cleared is a
+ * cache that stops working at exactly the moment it would have started paying
+ * for itself.
+ *
+ * `~/.shotium`, spelled the same on every platform. One place a user can look
+ * for it, one path to say in a bug report, and one directory to delete.
+ *
+ * $TMPDIR remains only as a fallback for a process with no home to speak of --
+ * some containers, some service accounts. That is a degradation and not a
+ * second location: there is no home directory holding a cache that would
+ * otherwise have been found.
+ */
+function shotiumHome(): string {
+  return path.join(os.homedir() || os.tmpdir(), '.shotium');
+}
+
+export function cacheRoot(): string {
+  return normalizePath(path.join(shotiumHome(), 'cache'));
+}
+
+/**
+ * The identifier for a project's cache directory: a hash of its root path.
+ *
+ * A hash rather than the path itself because the path contains separators,
+ * drive letters and whatever the user called their directory, none of which
+ * survive being a directory name. It is not a security measure and does not
+ * need to be one -- it is a fixed-length name for a variable-length string.
+ */
+export function projectKey(root: string = projectRoot()): string {
+  return crypto.createHash('sha1').update(normalizePath(root)).digest('hex');
+}
+
+/** This project's cache directory. */
+export function defaultCacheDir(): string {
+  return normalizePath(path.join(cacheRoot(), projectKey()));
+}
+
 // The one place that decides what "no options" means.
 //
 // It is shared rather than duplicated because the daemon's address is a hash of
@@ -16,16 +117,22 @@ export interface ResolvedStartOptions {
 // start a second engine next to the first one that was already warm. See
 // endpoint.ts.
 //
-// The default for `cacheDir` is null -- no disk cache. A program holding the
-// engine is often short-lived, and a cache it never reads twice is a directory
-// it leaves behind. The daemon, which is the case where a cache does pay for
-// itself, is also the case where the caller is already passing options.
+// `cacheDir` defaults to this project's directory rather than to null, which
+// is the reverse of 0.2. The reason is measured: without a cache every capture
+// of an `https:` URL pays DNS, TLS and a round trip, which for a small page is
+// most of the wall clock and all of the surprise. The objection to a default
+// -- that a short-lived program leaves a directory behind -- is answered by
+// the directory being per-project, size-capped, and somewhere the platform's
+// own tooling knows how to clear, rather than by there being no cache.
+// `cacheDir: null` still turns it off.
 function resolveStartOptions(options: StartOptions = {}): ResolvedStartOptions {
   return {
-    cacheDir: options.cacheDir ?? null,
+    cacheDir: options.cacheDir === null ? null :
+                                          (options.cacheDir ?? defaultCacheDir()),
+    cacheMaxBytes: options.cacheMaxBytes ?? DEFAULT_CACHE_MAX_BYTES,
     userAgent: options.userAgent,
     resourceDir: options.resourceDir,
   };
 }
 
-export {resolveStartOptions};
+export {DEFAULT_CACHE_MAX_BYTES, resolveStartOptions};

package/src/lib/daemon.ts

@@ -4,7 +4,11 @@ import net from 'node:net';
 import os from 'node:os';
 import path from 'node:path';
 
-import type {DaemonOptions, DaemonStatus} from '../types.js';
+import type {
+  CaptureStats,
+  DaemonOptions,
+  DaemonStatus,
+} from '../types.js';
 
 import {resolveStartOptions} from './config.js';
 import type {ResolvedStartOptions} from './config.js';
@@ -46,6 +50,11 @@ interface DaemonReply {
   bytes?: number;
   path?: string;
   stopping?: boolean;
+  // What the capture cost, on the success header and on the failure one. The
+  // client turns it back into the same CaptureStats the in-process engine
+  // returns, so a program moving between the two changes an import and
+  // nothing else.
+  stats?: CaptureStats;
 }
 
 // An engine that outlives the process that asked for it.
@@ -306,7 +315,7 @@ class Daemon extends EventEmitter {
     this.armIdleTimer();
     this.emit('request', {id, file: request.file});
     this.engine.capture(request)
-        .then((image) => {
+        .then(({image, stats}) => {
           this.served += 1;
           this.reply(
               socket,
@@ -315,12 +324,20 @@ class Daemon extends EventEmitter {
                 ok: true,
                 bytes: image ? image.length : 0,
                 path: request.path,
+                stats,
               },
               image);
         })
-        .catch((error: Error) => {
-          this.reply(
-              socket, {id, ok: false, error: String(error.message || error)});
+        .catch((error: Error&{stats?: CaptureStats}) => {
+          // The counters go back with the failure, matching the in-process
+          // engine: a capture that timed out after fetching forty subresources
+          // has already said why, and the message alone has not.
+          this.reply(socket, {
+            id,
+            ok: false,
+            error: String(error.message || error),
+            stats: error.stats,
+          });
         })
         .finally(() => {
           this.inFlight -= 1;
@@ -374,7 +391,13 @@ class Daemon extends EventEmitter {
     }
     this.sockets.clear();
     await new Promise<void>((resolve) => this.server!.close(() => resolve()));
-    await this.engine.stop();
+    // dispose() rather than stop(), and this is the one caller that should.
+    // The daemon owns its process and is leaving it, so the real teardown is
+    // available and worth taking: joining the engine thread unwinds the
+    // network stack, which is what lets the disk cache write its index. A
+    // daemon that merely stood the engine down would leave the index dirty and
+    // make the next daemon rebuild it by scanning the directory.
+    await this.engine.dispose();
     this.emit('close', {});
   }
 }