@shotkit/shotium 0.2.0 → 0.3.1

package/src/index.ts

@@ -1,25 +1,38 @@
+import {Cache} from './lib/cache.js';
 import * as client from './lib/client.js';
 import type {DaemonClient} from './lib/client.js';
 import {Engine} from './lib/engine.js';
 import type {
   DaemonOptions,
   DaemonStatus,
-  PurgeOptions,
+  ReleaseMemoryOptions,
   ScreenshotOptions,
+  ScreenshotResult,
   StartOptions,
+  StartResult,
 } from './types.js';
 
 export type {
+  CacheClearOptions,
+  CacheClearResult,
+  CacheEntry,
+  CacheMode,
+  CacheTarget,
+  CaptureStats,
+  CaptureTiming,
   Clip,
   DaemonOptions,
   DaemonStatus,
   PageGotoParams,
-  PurgeOptions,
+  ReleaseMemoryOptions,
   ScreenshotOptions,
+  ScreenshotResult,
   StartOptions,
+  StartResult,
   Viewport,
 } from './types.js';
 export type {DaemonClient} from './lib/client.js';
+export {Cache} from './lib/cache.js';
 
 /** The five things a caller does with the resident engine. */
 export interface Daemon {
@@ -27,7 +40,7 @@ export interface Daemon {
   connect(options?: DaemonOptions): Promise<DaemonClient>;
   /** One screenshot through the daemon, connection and all. */
   screenshot(options: ScreenshotOptions&{daemon?: DaemonOptions}):
-      Promise<Buffer|null>;
+      Promise<ScreenshotResult>;
   /** Starts one if it is not up, and reports what is there either way. */
   start(options?: DaemonOptions): Promise<DaemonStatus&{spawned: boolean}>;
   status(options?: DaemonOptions):
@@ -40,9 +53,11 @@ export interface Daemon {
  *
  *     import shotium from '@shotkit/shotium';
  *
- *     shotium.runtime.start();
- *     const png = await shotium.screenshot({file: 'https://example.com'});
- *     await shotium.runtime.stop();
+ *     shotium.start();
+ *     const {image, stats} = await shotium.screenshot({
+ *       file: 'https://example.com',
+ *     });
+ *     await shotium.stop();
  *
  * `start` and `stop` are explicit because starting Blink is the expensive part
  * -- tens of milliseconds and a working set that stays resident -- and only
@@ -51,11 +66,24 @@ export interface Daemon {
  * What they buy is control over when that cost is paid, and the certainty that
  * it has been given back.
  *
- * `runtime` below is the singleton because there is nothing else it could be:
- * Blink starts once per process and cannot be restarted, so a second Runtime
- * in the same process has no engine to have. Construct one directly only to
- * own the lifecycle yourself instead of using `runtime`. Parallelism is more
- * processes, not more Runtimes.
+ * Neither is rationed, either. They may be called in any order and as often as
+ * a program likes: `stop()` stands the engine down and `start()` picks the
+ * same one back up, warm cache and all. What cannot happen is a *second*
+ * engine -- Blink is initialised once per process and there is no undo -- but
+ * that is a fact about how many there are, not about how many times the one
+ * may be asked for.
+ *
+ * The methods are on the module rather than under a `runtime` namespace, which
+ * they were until 0.3. There was never anything else to start, so the word
+ * carried nothing; and `runtime.cache` would have been the wrong place for the
+ * cache besides, since a cache directory outlives every engine that writes to
+ * it and can be read when no engine is running at all.
+ *
+ * `Runtime` is still exported for a caller who wants to own a lifecycle rather
+ * than share the module's. It is a lifecycle and not an engine: there is one
+ * engine per process, and a second Runtime that starts adopts the same one
+ * rather than building another. Parallelism is more processes, not more
+ * Runtimes.
  *
  * `daemon` is the same engine in a process of its own, behind a socket, for
  * callers whose own process does not live long enough to be worth starting
@@ -64,49 +92,91 @@ export interface Daemon {
 export class Runtime {
   private engine = new Engine();
 
+  /**
+   * The HTTP cache: where it is, what is in it, and how to empty it.
+   *
+   * On the Runtime as well as on the module because a caller holding their own
+   * Runtime needs the engine handle to reach a directory that engine has open:
+   * within one process a directory has one backend, so borrowing is the only
+   * way in.
+   */
+  readonly cache = new Cache(() => this.engine.nativeHandle);
+
   get running(): boolean {
     return this.engine.running;
   }
 
   /**
-   * Starts the engine. Safe to call twice; the second call is a no-op, so that
-   * library code can call it defensively. Not safe after `stop()` -- see there.
+   * Starts the engine, or picks the running one back up.
    *
-   * Every option has a default. `cacheDir` is the HTTP disk cache and `null`
-   * disables it; `resourceDir` is where `shotium_data.pak` and
+   * Callable as often as you like, in any order with `stop()`; library code
+   * can call it defensively. The first call in a process builds the engine and
+   * every later one adopts it -- the same engine, the same warm cache. The one
+   * thing it will refuse is a *different* configuration: the options below are
+   * fixed when the engine is built, and there is no second build, so naming
+   * one that disagrees with what is running throws rather than rendering with
+   * a value you did not ask for.
+   *
+   * Every option has a default. `cacheDir` is the HTTP disk cache and defaults
+   * to a per-project directory under `~/.shotium/cache`, and not under the
+   * temporary directory, which is defined by not surviving. `null` turns it
+   * off. `resourceDir` is where `shotium_data.pak` and
    * `shotium_strings.pak` are, and defaults to the directory the engine was
    * loaded from, which is where they ship.
+   *
+   * The return value is worth reading once. `cacheActive: false` with a
+   * `cacheDir` set means the directory could not be opened and this engine is
+   * running without a cache -- correctly, silently, and a round trip slower on
+   * everything.
    */
-  start(options: StartOptions = {}): this {
-    this.engine.start(options);
-    return this;
+  start(options: StartOptions = {}): StartResult {
+    return this.engine.start(options);
+  }
+
+  /** What `start()` returned, asked again. */
+  status(): StartResult {
+    return this.engine.status();
   }
 
   /**
-   * Stops the engine, after whatever is queued.
+   * Stands the engine down, after whatever is queued.
+   *
+   * The queue drains, the memory the engine can rebuild goes back to the OS,
+   * and `running` becomes false. Blink itself stays initialised, because there
+   * is no way to un-initialise it -- so the disk cache stays where it is, and
+   * `start()` or the next `screenshot()` picks the same engine back up.
    *
-   * Final for this process. Blink writes process-wide state that it has no
-   * path to undo, so starting again -- here or on another Runtime -- throws
-   * rather than quietly handing back something that cannot render. A program
-   * that wants another screenshot later should stay started and `purge()`.
+   * Which makes this a caller saying they are done for now rather than a
+   * destructor. It does the same work as `releaseMemory({releaseWorkingSet:
+   * true})` and additionally stops accepting captures.
    */
   stop(): Promise<void> {
     return this.engine.stop();
   }
 
   /**
-   * Hands back what the engine is holding but can rebuild. Worth calling when
-   * a batch has ended and the next one may be a while away.
+   * Hands back what the engine is holding but can rebuild: Blink's heap,
+   * skia's caches, PartitionAlloc's free lists. Worth calling when a batch has
+   * ended and the next one may be a while away.
+   *
+   * This is memory and nothing else. It was called `purge()` until 0.3, which
+   * next to `cache.clear()` read as though it emptied the HTTP cache; it does
+   * not touch the disk at all.
    */
-  purge(options: PurgeOptions = {}): void {
-    this.engine.purge(options);
+  releaseMemory(options: ReleaseMemoryOptions = {}): void {
+    this.engine.releaseMemory(options);
   }
 
   /**
-   * Renders one screenshot. Resolves to the encoded image, or to `null` when
-   * `path` was given and the engine wrote the file itself.
+   * Renders one screenshot, and reports what it cost.
+   *
+   * `image` is the encoded bytes, or `null` when `path` was given and the
+   * engine wrote the file itself. `stats` says how many resources were
+   * fetched, how many came from the cache, and where the milliseconds went --
+   * which for an `https:` URL is usually the answer to "why did this take so
+   * long", because a cold connection costs more than the render does.
    */
-  screenshot(options: ScreenshotOptions): Promise<Buffer|null> {
+  screenshot(options: ScreenshotOptions): Promise<ScreenshotResult> {
     return this.engine.screenshot(options);
   }
 }
@@ -115,13 +185,37 @@ export class Runtime {
 const runtime = new Runtime();
 
 /** One screenshot through the shared engine, starting it if it is not up. */
-const screenshot = (options: ScreenshotOptions): Promise<Buffer|null> =>
+const screenshot = (options: ScreenshotOptions): Promise<ScreenshotResult> =>
     runtime.screenshot(options);
 
+const start = (options?: StartOptions): StartResult => runtime.start(options);
+const status = (): StartResult => runtime.status();
+const stop = (): Promise<void> => runtime.stop();
+const releaseMemory = (options?: ReleaseMemoryOptions): void =>
+    runtime.releaseMemory(options);
+
+/**
+ * The HTTP cache.
+ *
+ * At the top level rather than under the engine because it outlives one: the
+ * directory is on disk whether or not anything is running, `getDir()` answers
+ * before the first `start()`, and clearing it is something a program may want
+ * to do without bringing Blink up at all. When an engine *is* up, these
+ * borrow its cache backend, because within one process a directory has one
+ * backend and that is the only way in.
+ */
+const cache = runtime.cache;
+
 /**
  * The resident engine: a process that outlives the one that started it,
  * reachable over a named pipe on Windows and a unix socket elsewhere. For
  * callers that are short-lived themselves. See lib/daemon.ts.
+ *
+ * It has no `cache` of its own. A daemon's cache directory is reported by
+ * `daemon.status()`, and clearing it is done by pointing `cache.clear()` at
+ * that directory or by stopping the daemon -- a cross-process cache protocol
+ * would be a second implementation of this module for something nobody does on
+ * a request path.
  */
 const daemon: Daemon = {
   connect: client.connect,
@@ -131,9 +225,27 @@ const daemon: Daemon = {
   stop: client.stop,
 };
 
-export {runtime, screenshot, daemon};
+export {cache, daemon, releaseMemory, runtime, screenshot, start, status, stop};
 
 // A default as well as the names, because `import shotium from` is what a
 // caller coming from `require` writes first, and the two have to be the same
 // object rather than two views that drift.
-export default {Runtime, runtime, screenshot, daemon};
+export default {
+  Runtime,
+  cache,
+  daemon,
+  releaseMemory,
+  runtime,
+  screenshot,
+  start,
+  status,
+  stop,
+  // A getter and not a value, because it changes. It is only on the default
+  // export: a named `running` would have to be a live binding that something
+  // remembered to update, and the two would disagree the first time anybody
+  // forgot. Callers who import by name have `status().running`, which is the
+  // same answer with the cache directory attached.
+  get running(): boolean {
+    return runtime.running;
+  },
+};

package/src/lib/binding.ts

@@ -17,12 +17,33 @@ const HERE = path.dirname(fileURLToPath(import.meta.url));
  */
 export type Engine = unknown;
 
+/** One capture's answer, as the addon hands it over. */
+export interface NativeCapture {
+  image: Buffer;
+  /**
+   * CaptureStats as JSON, unparsed. The addon carries JSON between the engine
+   * and this layer without reading it -- anything it understood would be a
+   * third opinion about the shape, and the third opinion is the one that
+   * drifts. Undefined when the engine reported none.
+   */
+  stats?: string;
+}
+
 /** What native/binding.cc exports. See shot/shot_api.h for the C ABI. */
 export interface NativeBinding {
   create(optionsJson: string): Engine;
   destroy(engine: Engine): void;
   purge(engine: Engine, releaseWorkingSet: boolean): void;
-  capture(engine: Engine, requestJson: string): Promise<Buffer>;
+  status(engine: Engine): string;
+  capture(engine: Engine, requestJson: string): Promise<NativeCapture>;
+  /**
+   * List or clear a cache directory. `engine` is nullable and that is the
+   * interface: with one, the operation runs on the engine's thread and borrows
+   * the backend it already holds; without one, the library opens the directory
+   * itself. Resolves to JSON.
+   */
+  cache(engine: Engine|null, clearing: boolean, optionsJson: string):
+      Promise<string>;
 }
 
 // Where the addon and the library beside it live.

package/src/lib/cache.ts

@@ -0,0 +1,323 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import type {
+  CacheClearOptions,
+  CacheClearResult,
+  CacheEntry,
+  CacheTarget,
+} from '../types.js';
+
+import * as binding from './binding.js';
+import type {Engine as Handle} from './binding.js';
+import {cacheRoot, defaultCacheDir, normalizePath} from './config.js';
+
+/**
+ * Turns one glob into a regular expression over a URL.
+ *
+ * The dialect is the small one everybody already knows -- `*`, `**`, `?`,
+ * `{a,b}` -- and it is implemented here rather than depended on because this
+ * package has no runtime dependencies and a matcher is thirty lines. `*` stops
+ * at `/` and `**` does not, which is the distinction that makes
+ * `https://example.com/*` mean one level and `https://example.com/**` mean the
+ * site.
+ *
+ * Everything else is escaped, which matters more than usual here: the subjects
+ * are URLs, and a URL is mostly characters that mean something to a regular
+ * expression.
+ */
+function globToRegExp(pattern: string): RegExp {
+  let out = '';
+  for (let i = 0; i < pattern.length; i++) {
+    const c = pattern[i];
+    if (c === '*') {
+      if (pattern[i + 1] === '*') {
+        out += '.*';
+        i++;
+        // `/**/` should also match the zero-segment case, so that
+        // `https://x/**/y` matches `https://x/y`.
+        if (pattern[i + 1] === '/') {
+          out += '/?';
+          i++;
+        }
+      } else {
+        out += '[^/]*';
+      }
+    } else if (c === '?') {
+      out += '[^/]';
+    } else if (c === '{') {
+      const end = pattern.indexOf('}', i);
+      if (end === -1) {
+        out += '\\{';
+      } else {
+        const alternatives =
+            pattern.slice(i + 1, end).split(',').map(escapeLiteral);
+        out += `(?:${alternatives.join('|')})`;
+        i = end;
+      }
+    } else {
+      out += escapeLiteral(c);
+    }
+  }
+  return new RegExp(`^${out}$`);
+}
+
+function escapeLiteral(text: string): string {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/** Whether `url` matches any of `patterns`. No patterns matches nothing. */
+function matchesAny(url: string, patterns: RegExp[]): boolean {
+  return patterns.some((pattern) => pattern.test(url));
+}
+
+/**
+ * What a cache directory occupies, for the one path that reports a size
+ * without a backend to ask.
+ *
+ * The sum of the files rather than the sum of the entries, so it will differ
+ * from what `clear()` reports through the backend by the index and by whatever
+ * rounding the filesystem does. It is the honest number for "what is about to
+ * be deleted", which is what it is used for.
+ */
+function directorySize(dir: string): number {
+  let total = 0;
+  let names: fs.Dirent[] = [];
+  try {
+    names = fs.readdirSync(dir, {withFileTypes: true});
+  } catch {
+    return 0;
+  }
+  for (const entry of names) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      total += directorySize(full);
+      continue;
+    }
+    try {
+      total += fs.statSync(full).size;
+    } catch {
+      // Raced with something else clearing the same directory. Not an error:
+      // a file that is already gone contributes nothing to what is left.
+    }
+  }
+  return total;
+}
+
+/**
+ * Which directories an operation covers.
+ *
+ * `current` is this project's, `all` is every directory under the shared root,
+ * and anything else is taken as a project hash. `all` reads the root rather
+ * than remembering what it created: another process's directory is as much
+ * shotium's as this one's, and a caller asking to clear them all means the
+ * ones on disk.
+ */
+function resolveTargets(target: CacheTarget['target']): string[] {
+  if (target === 'all') {
+    const root = cacheRoot();
+    let names: string[] = [];
+    try {
+      names = fs.readdirSync(root);
+    } catch {
+      // No root means nothing has been cached yet, which is an empty list and
+      // not an error: a caller clearing an empty cache asked for a state that
+      // already holds.
+      return [];
+    }
+    return names.map((name) => normalizePath(path.join(root, name)))
+        .filter((dir) => {
+          try {
+            return fs.statSync(dir).isDirectory();
+          } catch {
+            return false;
+          }
+        });
+  }
+  if (target === undefined || target === 'current') {
+    return [defaultCacheDir()];
+  }
+  // A directory, if it looks like one. `start({cacheDir})` takes any path, so
+  // a caller who chose their own has to be able to name it here -- otherwise
+  // the cache they configured is the one cache these methods cannot see.
+  if (path.isAbsolute(target)) {
+    return [normalizePath(target)];
+  }
+  // Otherwise a project hash. Resolved against the root rather than used as a
+  // path, so that a relative string cannot reach outside it by accident.
+  return [normalizePath(path.join(cacheRoot(), target))];
+}
+
+/**
+ * The cache, from the outside.
+ *
+ * Every method takes the engine handle if there is one, and that is not an
+ * optimisation. Within one process a cache directory has one backend: asking
+ * for a second one on the directory the engine holds waits for the engine's to
+ * go away, which it will not do while the engine is up. Borrowing is the only
+ * thing that returns.
+ *
+ * "If there is one" means the process, not the lifecycle. `stop()` stands the
+ * engine down without tearing it down, so an engine that has been stopped
+ * still holds its directory and still has to be borrowed from -- which is also
+ * what makes the cache survive a stop, and outlive one, and be worth having.
+ *
+ * Across processes there is no such constraint -- several of them may share a
+ * directory and all of them cache.
+ *
+ * The engine is fetched through a callback rather than held, because this
+ * object is built once at import time and the engine comes and goes.
+ */
+export class Cache {
+  constructor(private readonly engineHandle: () => Handle | null) {}
+
+  /**
+   * This project's cache directory, absolute and with forward slashes.
+   *
+   * It exists whether or not anything has been written to it -- the answer is
+   * "where the cache goes", not "where a cache is".
+   */
+  getDir(options: CacheTarget = {}): string {
+    const targets = resolveTargets(options.target);
+    return targets.length > 0 ? targets[0] : defaultCacheDir();
+  }
+
+  /** Every directory the target names. `all` can be several; the rest, one. */
+  getDirs(options: CacheTarget = {}): string[] {
+    return resolveTargets(options.target);
+  }
+
+  /**
+   * What the cache is holding, by URL.
+   *
+   * Named `getFiles` for the operation callers reach for, and deliberately not
+   * returning filenames: the files in a cache directory are called things like
+   * `5349fbae98c6d9a1_0`, because the name is a hash of the entry key. A list
+   * of those answers no question anybody has. The URLs are what the entries
+   * are, and they are what `clear({glob})` matches against.
+   *
+   * This opens every entry to read its key and size, so it is a diagnostic
+   * rather than something to put on a request path.
+   */
+  async getFiles(options: CacheTarget = {}): Promise<CacheEntry[]> {
+    const native = binding.load();
+    const entries: CacheEntry[] = [];
+    for (const dir of resolveTargets(options.target)) {
+      if (!fs.existsSync(dir)) {
+        continue;
+      }
+      const json = await native.cache(
+          this.handleFor(), /*clearing=*/ false, JSON.stringify({
+            cacheDir: dir,
+          }));
+      const listed = JSON.parse(json) as Array<Omit<CacheEntry, 'dir'>>;
+      for (const entry of listed) {
+        entries.push({...entry, dir});
+      }
+    }
+    return entries;
+  }
+
+  /**
+   * Removes what the options select. With no options, everything.
+   *
+   * The three filters compose, and `glob` is applied here rather than in the
+   * engine: the entries are listed, their URLs are matched, and the ones that
+   * matched are what the engine is asked to remove. That keeps the pattern
+   * dialect in the layer whose users have opinions about pattern dialects, and
+   * keeps the engine's interface to exact URLs.
+   *
+   * Removal goes through the cache backend, never through the filesystem.
+   * Deleting the files directly would leave the backend's index naming entries
+   * that are no longer there, and the next process to open the directory
+   * either rebuilds the index from disk or, having found it inconsistent,
+   * discards it. That is the difference between clearing a cache and
+   * corrupting one.
+   */
+  async clear(options: CacheClearOptions = {}): Promise<CacheClearResult[]> {
+    const native = binding.load();
+    const patterns = (options.glob ?? []).map(globToRegExp);
+    const results: CacheClearResult[] = [];
+
+    // Clearing everything, in a process that has no engine at all: remove the
+    // directory.
+    //
+    // This is the one case where touching the filesystem is correct rather
+    // than reckless. The danger in deleting cache files by hand is a partial
+    // delete -- an index left naming entries that are gone -- and there is no
+    // such thing when the index goes with them. What is left is a directory
+    // that does not exist, which is exactly what an empty cache looks like
+    // before anything has written to it.
+    //
+    // It is also the fast path a short script gets: emptying a cache without
+    // starting Blink to do it costs a few milliseconds instead of the tens
+    // that building an engine does.
+    const unfiltered = patterns.length === 0 && !options.maxAge &&
+        !options.maxSize;
+    if (unfiltered && !this.handleFor()) {
+      for (const dir of resolveTargets(options.target)) {
+        const before = directorySize(dir);
+        fs.rmSync(dir, {recursive: true, force: true});
+        results.push(
+            {removed: -1, bytesBefore: before, bytesAfter: 0, dir});
+      }
+      return results;
+    }
+
+    for (const dir of resolveTargets(options.target)) {
+      if (!fs.existsSync(dir)) {
+        continue;
+      }
+      const request: Record<string, unknown> = {cacheDir: dir};
+
+      if (patterns.length > 0) {
+        const json = await native.cache(
+            this.handleFor(), /*clearing=*/ false,
+            JSON.stringify({cacheDir: dir}));
+        const listed = JSON.parse(json) as Array<Omit<CacheEntry, 'dir'>>;
+        const urls =
+            listed.filter((entry) => matchesAny(entry.url, patterns))
+                .map((entry) => entry.url);
+        // Nothing matched, so nothing is asked for. Falling through with an
+        // empty `urls` would be read by the engine as "no URL filter", which
+        // combined with no other filter empties the directory -- the opposite
+        // of what a pattern that matched nothing means.
+        if (urls.length === 0 && options.maxAge === undefined &&
+            options.maxSize === undefined) {
+          results.push({removed: 0, bytesBefore: 0, bytesAfter: 0, dir});
+          continue;
+        }
+        request.urls = urls;
+      }
+
+      if (options.maxAge) {
+        request.unusedSinceMs = Date.now() - options.maxAge * 1000;
+      }
+      if (options.maxSize) {
+        request.maxBytes = options.maxSize;
+      }
+
+      const json = await native.cache(
+          this.handleFor(), /*clearing=*/ true, JSON.stringify(request));
+      results.push({
+        ...(JSON.parse(json) as Omit<CacheClearResult, 'dir'>),
+        dir,
+      });
+    }
+    return results;
+  }
+
+  /**
+   * The engine handle, when there is an engine.
+   *
+   * Passed for every directory and not only the engine's own. It is never
+   * wrong to pass it -- the engine's thread can open any directory, and for
+   * the one it already has open, borrowing its backend is the only thing that
+   * returns. It is passing `null` while an engine is up that hangs, which is
+   * why this is conditional on neither the directory asked for nor on whether
+   * the engine is currently accepting captures.
+   */
+  private handleFor(): Handle|null {
+    return this.engineHandle();
+  }
+}