@shotkit/shotium 0.2.0 → 0.3.0

package/src/lib/engine.ts

@@ -2,23 +2,80 @@ import * as binding from './binding.js';
 import type {Engine as Handle} from './binding.js';
 import {toRequest} from './request.js';
 import type {WireRequest} from './request.js';
-import type {PurgeOptions, ScreenshotOptions, StartOptions} from '../types.js';
+import type {
+  CaptureStats,
+  ReleaseMemoryOptions,
+  ScreenshotOptions,
+  ScreenshotResult,
+  StartOptions,
+  StartResult,
+} from '../types.js';
 
+import type {ResolvedStartOptions} from './config.js';
 import {resolveStartOptions} from './config.js';
 
-// One per process, ever. Not one at a time -- one.
+// The engine this process has, held above every Engine object that uses it.
 //
-// This is not a rule of this file, it is what Blink is: initialising it writes
-// process-wide statics it has no path to undo, so shot_engine_destroy() gives
-// back what it can and the process still cannot make another. The C API
-// returns SHOT_ERR_STATE for a second create whether or not the first is
-// still alive. See shot/shot_api.h.
+// Blink is initialised once and has no undo: it writes process-wide statics
+// that shot_engine_destroy() cannot take back, and the C API refuses a second
+// create for the lifetime of the process whether or not the first is still
+// alive.
 //
-// So `stop()` is final for the process, and this flag exists to say that in
-// words at the call site. Without it a caller who stops and starts again gets
-// SHOT_ERR_STATE out of the addon -- a true error, arriving one layer too deep
-// to explain that the answer is a second process rather than a retry.
-let startedInThisProcess = false;
+// That fact used to be exposed directly -- `stop()` destroyed the engine and
+// every later `start()` threw. It was the wrong shape. `stop()` and `start()`
+// are a caller saying "I am done for now" and "I want it again", and a library
+// whose engine can be asked for exactly once turns an ordinary pair of calls
+// into a thing that has to be rationed. It also made the disk cache
+// nonsensical: the whole point of a cache is the *next* run, and the next run
+// could not have the engine that reads it.
+//
+// So the handle lives here rather than on the instance. `stop()` stands the
+// engine down -- the queue drains, the memory goes back, nothing more is
+// accepted -- and `start()` picks the same one up again, as many times as a
+// caller likes. The process is the engine's lifetime, which is what it always
+// was; the difference is that the API no longer pretends to offer a shorter
+// one.
+let shared: Handle|null = null;
+
+// What `shared` was created with. Kept because those options are fixed for the
+// life of the process -- a later `start()` asking for a different cache
+// directory cannot be given one, and is told so rather than handed an engine
+// that quietly uses the first caller's.
+let sharedOptions: ResolvedStartOptions|null = null;
+
+// Whether the one create this process gets has been spent.
+//
+// Separate from `shared` being non-null because `dispose()` clears the handle
+// and does not give the ability back: after a real teardown there is no engine
+// and there cannot be another. Nothing in the public surface calls dispose()
+// -- the daemon does, on its way out of a process it owns.
+let spent = false;
+
+/** The engine handle this process has, or null if it has none. */
+function sharedHandle(): Handle|null {
+  return shared;
+}
+
+// The options that are fixed at create time, and the report a mismatch gets.
+//
+// Only the ones the caller actually named are checked: `start()` with no
+// arguments is a caller saying "whatever is there", which is exactly what
+// adopting a running engine gives them. Naming an option that disagrees is
+// different -- it is a request that cannot be honoured, and silently rendering
+// with the other value is the failure this exists to prevent.
+function conflictingOption(
+    options: StartOptions, current: ResolvedStartOptions): string|null {
+  const wanted = resolveStartOptions(options);
+  for (const key of ['cacheDir', 'cacheMaxBytes', 'userAgent', 'resourceDir'] as
+       const) {
+    if (options[key] === undefined || wanted[key] === current[key]) {
+      continue;
+    }
+    return `${key} is ${JSON.stringify(current[key])}, and this start() ` +
+        `asked for ${JSON.stringify(wanted[key])}`;
+  }
+  return null;
+}
 
 /**
  * Blink, in this process, and the queue in front of it.
@@ -36,45 +93,73 @@ let startedInThisProcess = false;
  * gaining nothing, since the engine serialises them anyway.
  */
 export class Engine {
-  private handle: Handle|null = null;
-  private stopped = false;
+  // Whether *this* object considers itself started. The engine behind it may
+  // well be up for somebody else; `running` is about this lifecycle, not about
+  // whether the process has an engine.
+  private active = false;
   private tail: Promise<unknown> = Promise.resolve();
 
   get running(): boolean {
-    return this.handle !== null;
+    return this.active && shared !== null;
   }
 
   /**
-   * Starts the engine. Safe to call twice; the second call is a no-op, so that
-   * library code can call it defensively.
+   * The addon's engine handle, or null when this process has never had one.
    *
-   * Not safe to call after `stop()`, and not because of anything here: Blink
-   * starts once per process and cannot be restarted. Another engine means
-   * another process.
+   * Deliberately not conditional on `running`. It is the cache that asks, and
+   * what the cache needs to know is whether a backend exists in this process
+   * -- because within one process a cache directory has one backend, so
+   * reading or clearing the directory the engine holds means borrowing it
+   * rather than opening a second one. A stood-down engine still holds its
+   * directory, so a caller who calls `stop()` and then `cache.getFiles()` is
+   * asking about a live backend and has to be routed to it. Nothing else
+   * should reach for this.
    */
-  start(options: StartOptions = {}): this {
-    if (this.handle) {
-      return this;
-    }
-    if (this.stopped) {
-      throw new Error(
-          'shotium: this engine was stopped, and Blink cannot be started ' +
-          'again in a process that has already run it. Start another ' +
-          'process, or keep the engine up between screenshots.');
+  get nativeHandle(): Handle|null {
+    return sharedHandle();
+  }
+
+  /**
+   * Starts the engine, or picks the running one back up.
+   *
+   * Callable as often as a caller likes, in any order with `stop()`. The first
+   * call in a process builds the engine; every later one adopts it, which is
+   * the same engine and the same warm cache. Library code can call it
+   * defensively.
+   *
+   * The one thing that cannot be adopted is a different configuration. The
+   * options below are fixed when the engine is built and there is no second
+   * build, so naming one that disagrees with what is running throws rather
+   * than rendering with a value the caller did not ask for.
+   */
+  start(options: StartOptions = {}): StartResult {
+    if (shared) {
+      const conflict = conflictingOption(options, sharedOptions!);
+      if (conflict) {
+        throw new Error(
+            'shotium: this process already has an engine, and its ' +
+            conflict + '. Blink is initialised once per process and cannot ' +
+            'be built again, so an engine\'s options are fixed for as long ' +
+            'as the process lives -- stop() does not undo them. Use the ' +
+            'engine that is up, or run another process.');
+      }
+      this.active = true;
+      return this.status();
     }
-    if (startedInThisProcess) {
+    if (spent) {
       throw new Error(
-          'shotium: an engine has already run in this process. Blink is a ' +
-          'process-wide singleton -- there is one per process, ever -- so a ' +
-          'second Runtime cannot have one. Use the shared `runtime`, or run ' +
-          'another process.');
+          'shotium: this process had an engine and it was disposed of. ' +
+          'Blink is initialised once per process and cannot be built again. ' +
+          'Run another process.');
     }
+
     const native = binding.load();
     const resolved = resolveStartOptions(options);
 
     const engineOptions: Record<string, unknown> = {};
     if (resolved.cacheDir !== null) {
       engineOptions.cacheDir = resolved.cacheDir;
+      engineOptions.cacheMaxBytes = resolved.cacheMaxBytes;
     }
     if (resolved.userAgent !== undefined) {
       engineOptions.userAgent = resolved.userAgent;
@@ -85,30 +170,84 @@ export class Engine {
     // teaching the engine a second way to look. See shot_api.h.
     engineOptions.resourceDir = resolved.resourceDir ?? binding.directory();
 
-    this.handle = native.create(JSON.stringify(engineOptions));
-    startedInThisProcess = true;
-    return this;
+    shared = native.create(JSON.stringify(engineOptions));
+    sharedOptions = resolved;
+    this.active = true;
+    return this.status();
   }
 
   /**
-   * Stops the engine, after whatever is queued.
+   * What the engine came up as: whether this lifecycle is started, which cache
+   * directory the engine has, and whether it actually got it.
+   *
+   * The last of those is the one worth reading. A directory that cannot be
+   * created or written to -- no permission, no space, a path that is a file --
+   * fails invisibly: the engine renders exactly as well without a cache, only
+   * slower, and every capture pays the network again for a reason nothing
+   * reports. The engine opens the cache during `start()` so that this is
+   * answerable before the first screenshot rather than after it.
    *
-   * Final for this process: see the note above. A program that will want
-   * another screenshot later should leave the engine up and call `purge()`
-   * instead, which hands back the memory without giving up the engine.
+   * The cache half is answered from the engine whenever this process has one,
+   * including after `stop()`. A stood-down engine still holds its directory,
+   * and reporting `null` for it would say the cache had gone away when what
+   * went away was the willingness to render.
+   */
+  status(): StartResult {
+    if (!shared) {
+      return {running: false, cacheDir: null, cacheActive: false};
+    }
+    const reported =
+        JSON.parse(binding.load().status(shared)) as Omit<StartResult, 'running'>;
+    return {running: this.running, ...reported};
+  }
+
+  /**
+   * Stands the engine down, after whatever is queued.
+   *
+   * The queue drains, the memory the engine can rebuild goes back to the OS,
+   * and `running` becomes false. What does not happen is a teardown of Blink,
+   * because there is no such thing -- see the note at the top of this file --
+   * so the disk cache stays where it is and `start()` picks the same engine up
+   * again whenever the caller wants it.
+   *
+   * Which makes this exactly what it says: not a destructor, a caller saying
+   * they are done for now. A program that will want another screenshot in a
+   * moment can equally well stay started and call `releaseMemory()`; the two
+   * do the same work, and this one also stops accepting captures.
    */
   async stop(): Promise<void> {
-    if (!this.handle) {
+    if (!this.active) {
       return;
     }
-    this.stopped = true;
-    // After the queue, not before: destroy() waits for a capture in flight
-    // anyway, and doing it in order means a caller's last screenshot resolves
-    // rather than racing the shutdown.
-    const handle = this.handle;
-    this.handle = null;
+    this.active = false;
+    // After the queue, not before: a caller's last screenshot should resolve
+    // rather than race the stand-down, and the memory is not worth handing
+    // back until the thing still using it has finished.
+    await this.tail.catch(() => {});
+    if (shared) {
+      binding.load().purge(shared, /*releaseWorkingSet=*/ true);
+    }
+  }
+
+  /**
+   * The real teardown: joins the engine thread, unwinds the network stack, and
+   * lets the disk cache write its index.
+   *
+   * Final, and final for the process rather than for this object -- which is
+   * why it is not on the public surface. The daemon calls it as it exits a
+   * process it owns, where the index flush is worth having and nothing is
+   * going to ask for another screenshot. Everything else wants `stop()`.
+   */
+  async dispose(): Promise<void> {
+    this.active = false;
     await this.tail.catch(() => {});
-    binding.load().destroy(handle);
+    const handle = shared;
+    shared = null;
+    sharedOptions = null;
+    if (handle) {
+      spent = true;
+      binding.load().destroy(handle);
+    }
   }
 
   /**
@@ -121,11 +260,11 @@ export class Engine {
    * request stream go quiet. Here the queue belongs to the caller, so the
    * caller is the one who knows a batch has ended.
    */
-  purge({releaseWorkingSet = false}: PurgeOptions = {}): void {
-    if (!this.handle) {
+  releaseMemory({releaseWorkingSet = false}: ReleaseMemoryOptions = {}): void {
+    if (!shared) {
       return;
     }
-    binding.load().purge(this.handle, releaseWorkingSet);
+    binding.load().purge(shared, releaseWorkingSet);
   }
 
   /**
@@ -136,7 +275,7 @@ export class Engine {
   // throws, and a caller who wrote `screenshot(bad).catch(...)` would get the
   // throw past the catch and into the surrounding frame. The whole surface is
   // promise-shaped, so a bad request is a rejection like everything else.
-  async screenshot(options: ScreenshotOptions): Promise<Buffer|null> {
+  async screenshot(options: ScreenshotOptions): Promise<ScreenshotResult> {
     // Before anything else, and before the queue: a malformed request should
     // be a rejection now rather than one that waits its turn.
     return this.capture(toRequest(options));
@@ -150,11 +289,13 @@ export class Engine {
    * ScreenshotOptions would mean the daemon validating a request it cannot see
    * the original of, and rejecting fields a newer client legitimately sent.
    */
-  async capture(request: WireRequest): Promise<Buffer|null> {
-    if (!this.handle) {
+  async capture(request: WireRequest): Promise<ScreenshotResult> {
+    // Starts, or restarts, or adopts -- a screenshot after `stop()` is an
+    // ordinary thing to ask for and gets the engine back.
+    if (!this.running) {
       this.start();
     }
-    const handle = this.handle;
+    const handle = shared!;
     const native = binding.load();
 
     // Chain onto the tail so that captures run one at a time. The catch keeps
@@ -162,7 +303,54 @@ export class Engine {
     const result = this.tail.catch(() => {}).then(
         () => native.capture(handle, JSON.stringify(request)));
     this.tail = result.catch(() => {});
-    const image = await result;
-    return request.path ? null : image;
+
+    let captured;
+    try {
+      captured = await result;
+    } catch (error) {
+      // The addon attaches the capture's statistics to the rejection as
+      // unparsed JSON, the same way it hands them back on success -- see
+      // NativeCapture. Parsing them here rather than leaving a string on the
+      // error is what makes `error.stats` the same CaptureStats a successful
+      // call returns, which is the whole point of attaching it: the failure is
+      // the case where the counters explain the most.
+      const withStats = error as Error&{stats?: string | CaptureStats};
+      if (typeof withStats.stats === 'string') {
+        withStats.stats = JSON.parse(withStats.stats) as CaptureStats;
+      }
+      throw error;
+    }
+
+    return {
+      image: request.path ? null : captured.image,
+      stats: parseStats(captured.stats),
+    };
   }
 }
+
+// A zeroed set of counters.
+//
+// Zeroes rather than undefined because the alternative is every caller writing
+// `stats?.timing?.total ?? 0` around a field that is present for every capture
+// that actually ran. The only case that produces none is a request rejected
+// before it started, and that path throws rather than returning.
+function emptyStats(): CaptureStats {
+  return {
+    requests: 0,
+    fromCache: 0,
+    failed: 0,
+    bytes: 0,
+    httpStatus: 0,
+    finalUrl: '',
+    timing: {fetch: 0, render: 0, encode: 0, total: 0},
+  };
+}
+
+// The addon hands statistics over as unparsed JSON -- see NativeCapture -- so
+// this is where the string becomes an object. The daemon's client has them
+// parsed already, from its own response header, and uses emptyStats directly.
+function parseStats(json: string|undefined): CaptureStats {
+  return json ? JSON.parse(json) as CaptureStats : emptyStats();
+}
+
+export {emptyStats, parseStats, sharedHandle};

package/src/lib/request.ts

@@ -1,4 +1,9 @@
-import type {Clip, PageGotoParams, ScreenshotOptions} from '../types.js';
+import type {
+  CacheMode,
+  Clip,
+  PageGotoParams,
+  ScreenshotOptions,
+} from '../types.js';
 
 const DEFAULT_TIMEOUT_MS = 30000;
 
@@ -16,6 +21,8 @@ export interface WireRequest {
   pageGotoParams?: PageGotoParams;
   clip?: Clip;
   allowFileAccess?: boolean;
+  cache?: CacheMode;
+  headers?: Record<string, string>;
   width?: number;
   height?: number;
 }
@@ -40,6 +47,8 @@ const WIRE_FIELDS = new Set([
   'clip',
   'viewport',
   'allowFileAccess',
+  'cache',
+  'headers',
 ]);
 
 // One ScreenshotOptions, checked and flattened into what goes on the wire.

package/src/types.ts

@@ -33,6 +33,69 @@ export interface Viewport {
   height?: number;
 }
 
+/**
+ * What a capture may do with the HTTP cache, spelled the way `fetch` spells
+ * it.
+ *
+ * - `default`: ordinary HTTP semantics. A fresh entry is used without asking,
+ *   a stale one is revalidated, and the response updates the cache.
+ * - `reload`: read nothing, write everything -- the browser's reload button.
+ *   The next capture is fast again.
+ * - `no-store`: neither read nor write. For a page that should not be left on
+ *   this machine's disk, which an authenticated one usually should not.
+ * - `only-if-cached`: the network may not be touched and a miss is an error.
+ *   Useful for a deterministic re-render of something already fetched.
+ */
+export type CacheMode = 'default'|'reload'|'no-store'|'only-if-cached';
+
+/** Where the milliseconds went. */
+export interface CaptureTiming {
+  /**
+   * Fetching the top-level document. For a cold `https:` URL this is DNS, TCP,
+   * TLS and a round trip, and it is routinely larger than everything below --
+   * which is the single most useful thing this object says.
+   */
+  fetch: number;
+  /** Parse, subresources, style, layout, prepaint, paint. */
+  render: number;
+  encode: number;
+  /** Wall clock for the whole capture, so the three above can be checked. */
+  total: number;
+}
+
+/** What one capture cost, and where its bytes came from. */
+export interface CaptureStats {
+  /** Every resource the document asked for, itself included. */
+  requests: number;
+  /**
+   * Answered from the HTTP cache -- the body came from disk.
+   *
+   * Not the same as "no network was touched". A stale entry that can be
+   * revalidated costs a conditional request and a 304, and counts here too;
+   * what the cache saved is the download rather than the round trip. That is
+   * why `timing.fetch` can be tens of milliseconds with this set.
+   */
+  fromCache: number;
+  failed: number;
+  /** Decoded body bytes, summed -- not the transfer size. */
+  bytes: number;
+  /** The document's own status. 0 for a `file:` URL. */
+  httpStatus: number;
+  /** After redirects, which is what relative URLs resolved against. */
+  finalUrl: string;
+  timing: CaptureTiming;
+}
+
+/** One screenshot, and what taking it cost. */
+export interface ScreenshotResult {
+  /**
+   * The encoded image, or `null` when `path` was given: the engine wrote the
+   * file itself and there is nothing left to hand back.
+   */
+  image: Buffer|null;
+  stats: CaptureStats;
+}
+
 export interface ScreenshotOptions {
   /** An http/https/file URL, or a local path. */
   file: string;
@@ -68,15 +131,48 @@ export interface ScreenshotOptions {
    * is rendered on.
    */
   allowFileAccess?: boolean;
+  /**
+   * What this capture may do with the HTTP cache. Default `default`.
+   *
+   * It applies to the subresources as well as the document: a `reload` that
+   * refreshed the HTML and reused yesterday's stylesheet would be a confusing
+   * thing to have asked for.
+   */
+  cache?: CacheMode;
+  /**
+   * Extra request headers, sent with the document and with the subresources
+   * that are same-origin with it.
+   *
+   * Same-origin is the whole rule and it is not configurable. A caller passing
+   * `Authorization` or `Cookie` means it for the site being photographed; a
+   * page that pulls a script from a CDN must not have the credential
+   * forwarded there.
+   */
+  headers?: Record<string, string>;
 }
 
 export interface StartOptions {
   /**
-   * Root of the HTTP disk cache. `null` disables caching entirely, which is
-   * the default: a program holding the engine is often short-lived, and a
-   * cache it never reads twice is a directory it leaves behind.
+   * Root of the HTTP disk cache. `null` disables caching entirely.
+   *
+   * The default is a per-project directory under the system temporary
+   * directory -- see `cache.getDir()`. Caching is on by default because the
+   * alternative turned out to be worse: without it every capture of an
+   * `https:` URL pays for DNS, TLS and a round trip, which for a small page is
+   * most of the time the call takes and all of the time the caller did not
+   * expect to spend.
    */
   cacheDir?: string|null;
+  /**
+   * Ceiling on the cache directory, in bytes. Default 256 MB.
+   *
+   * Zero is not "unlimited" -- it hands the decision to the backend, which
+   * sizes itself against the volume's free space. That was a reasonable
+   * default when every user of the cache had named a directory on purpose; for
+   * one that appears by default under `~/.shotium` because somebody imported a
+   * library, a number somebody chose is better than a number nobody did.
+   */
+  cacheMaxBytes?: number;
   /** Overrides the built-in user agent string. */
   userAgent?: string;
   /**
@@ -86,6 +182,36 @@ export interface StartOptions {
   resourceDir?: string;
 }
 
+/** What `start()` reports about the engine it brought up. */
+export interface StartResult {
+  /**
+   * Whether this lifecycle is started.
+   *
+   * A process has at most one engine, so `false` here does not mean there is
+   * nothing running -- it means this `Runtime` is stood down. `cacheDir` below
+   * is still answered from the engine, because a stood-down engine keeps its
+   * cache directory and reporting `null` would say the cache had gone away
+   * when what went away was the willingness to render.
+   */
+  running: boolean;
+  /** The directory in use, or `null` when caching is off. */
+  cacheDir: string|null;
+  /**
+   * Whether that directory is actually being cached into.
+   *
+   * A directory that cannot be created or written to costs nothing visible:
+   * the engine renders exactly as well without a cache, only slower, and every
+   * capture pays for the network again for a reason nothing reports. `false`
+   * with a `cacheDir` set means the open failed; `false` with `cacheDir: null`
+   * means no cache was asked for.
+   *
+   * It is not about sharing. Several processes may use one directory and all
+   * of them cache -- the backend takes no cross-process lock -- so `true` in
+   * two processes at once is the ordinary answer.
+   */
+  cacheActive: boolean;
+}
+
 export interface DaemonOptions extends StartOptions {
   /**
    * Address the daemon by name instead of by configuration. Without it the
@@ -133,7 +259,16 @@ export interface DaemonStatus {
   version: string;
 }
 
-export interface PurgeOptions {
+/**
+ * Options for `releaseMemory()`.
+ *
+ * Named for what it does rather than for `purge`, which it was called until
+ * 0.3. With `cache.clear()` in the API the old name reads as though it clears
+ * the cache, and it does not: it hands back blink's heap, skia's caches and
+ * PartitionAlloc's free lists, all of which the engine rebuilds on demand.
+ * Nothing on disk is touched.
+ */
+export interface ReleaseMemoryOptions {
   /**
    * Also ask the OS to take the engine's pages back. The next screenshot pays
    * them back in soft page faults -- a few milliseconds -- so this is for when
@@ -141,3 +276,68 @@ export interface PurgeOptions {
    */
   releaseWorkingSet?: boolean;
 }
+
+/** Which cache directory an operation is about. */
+export interface CacheTarget {
+  /**
+   * `current` (the default) is this project's directory, `all` is every
+   * directory shotium has created under the shared root -- `~/.shotium/cache`
+   * -- and a string is either an absolute path or one project hash as
+   * `getDir()` reports it.
+   *
+   * The absolute path is there because `start({cacheDir})` accepts any
+   * directory: without it, a caller who chose their own cache would have the
+   * one cache these methods could not see.
+   *
+   * `all` exists because the directories are per-project by default, so
+   * "clear shotium's caches" is otherwise something a caller cannot express
+   * without already knowing where the other projects were.
+   */
+  target?: 'current'|'all'|(string&{});
+}
+
+/** One resource the cache is holding. */
+export interface CacheEntry {
+  /** The resource, not the backend's key -- see `cache.getFiles()`. */
+  url: string;
+  /** Milliseconds since the Unix epoch. */
+  lastUsedMs: number;
+  bytes: number;
+  /** Which cache directory it was found in. */
+  dir: string;
+}
+
+export interface CacheClearOptions extends CacheTarget {
+  /**
+   * Glob patterns matched against entry URLs -- not against filenames, which
+   * are hashes and would match nothing anybody would think to write.
+   *
+   * Supports `*` (within a path segment), `**` (across segments), `?` and
+   * `{a,b}`. Matching happens here rather than in the engine: the entries come
+   * back first, the patterns are applied to their URLs, and the ones that
+   * matched are what gets removed.
+   */
+  glob?: string[];
+  /**
+   * Remove entries not used for this many seconds. `0`, the default, means no
+   * age limit.
+   */
+  maxAge?: number;
+  /**
+   * Evict least-recently-used entries until the directory is at or below this
+   * many bytes. `0`, the default, means no size limit.
+   */
+  maxSize?: number;
+}
+
+export interface CacheClearResult {
+  /**
+   * How many entries went. `-1` when the whole directory was dropped in one
+   * operation, which the backend does without counting them.
+   */
+  removed: number;
+  bytesBefore: number;
+  bytesAfter: number;
+  /** Which directory this result is for. */
+  dir: string;
+}