browser-commander 0.18.0 → 0.19.0

package/.jscpd-baseline.json

@@ -46,6 +46,7 @@
     "266b9cb95f783cdf": 1,
     "26bb09996580437c": 1,
     "26ebb2e251c9a39e": 1,
+    "29edfb9e08094991": 1,
     "2a78ee4b05679c8d": 1,
     "2b1a759e2cfd21a2": 1,
     "2c0f8b5c6f28420b": 1,
@@ -55,6 +56,7 @@
     "2f9e15467cc2f12a": 1,
     "3101802d3ea65339": 1,
     "33357ce94e07cc4b": 1,
+    "35cf2769645f3234": 1,
     "37bd4114d0089f49": 1,
     "389df6fb4b8315a1": 1,
     "39eaada6d1879f4a": 1,
@@ -97,6 +99,7 @@
     "608e1dd27138ec6f": 1,
     "6098544c77ff17a2": 1,
     "60d0c58a01fc8746": 1,
+    "61ae5da6cc003836": 1,
     "61bcadcd8dc9f63e": 1,
     "63ff613b1780c67a": 1,
     "642cc89ed49f8a21": 1,
@@ -110,7 +113,9 @@
     "6af3c307939ef75e": 1,
     "6b35b48512756192": 1,
     "6cd00d6b6a039022": 1,
+    "6da3fe1d15b66ae0": 1,
     "6e2457cd3e1f69e7": 1,
+    "6f30e86d1407e88c": 1,
     "6fa62e2642cc73c3": 1,
     "7009c5e9bd61dbe8": 1,
     "709348ea44394b52": 1,
@@ -168,10 +173,12 @@
     "acc95d694eb2f581": 1,
     "ad4469f2e8131535": 1,
     "ad4cfa76daa1b6fc": 1,
+    "ad50b48938af8aef": 1,
     "adae1329361a81a1": 1,
     "ae477ba64bafba3f": 1,
     "aee1e5c65f783848": 1,
     "aee45e8b58c2d87f": 1,
+    "af702b20ebd45ef2": 1,
     "af9d6a2f4eba146e": 1,
     "b010686af23efa90": 1,
     "b2a0686231220c8a": 1,
@@ -182,6 +189,7 @@
     "b8c2d21f2a26f77d": 1,
     "bb269582b94ed42d": 1,
     "bb3e6d0f7e7edccd": 1,
+    "bc9115be3a462930": 1,
     "bccf0d5cc57c08df": 1,
     "bd0ae42c13d5b4f9": 1,
     "bd6c62a6e1c6a071": 1,

package/CHANGELOG.md

@@ -1,5 +1,39 @@
 # Changelog
 
+## 0.19.0
+
+### Minor Changes
+
+- 4401d63: Record traces continuously across navigation and live state, and export them as
+  Links Notation.
+
+  `startTrace({mode: 'continuous'})` now keeps recording when the page navigates,
+  changes route or creates a frame: the observers are reinstalled from an init
+  script that runs before the page's own code, so nothing that happens during a
+  load is lost. What a person changes but the DOM does not show - typing,
+  checking, selecting, focus and scroll - is recorded as semantic live-state
+  records and applied on replay, child-list records carry the position a node was
+  inserted at or removed from, and every record carries stable trace, browser
+  context, page, navigation, frame and action identities. `initialCheckpoint` (on
+  by default) captures the page before the first action so the first interval has
+  a base to replay onto, and the viewer says what it is: a diagnostic replay of
+  what was recorded, not a re-execution of the session.
+
+  `writeTraceLinks(trace, './run.lino', {include})` writes a portable Links
+  Notation view of a bundle, and `startTrace({links: {output}})` writes the same
+  view incrementally as the run happens. One link per line: one per ordered
+  timeline event with its sequence, time, kind, owner ids, actor, action, target
+  and outcome; one per checkpoint naming its HTML, state and screenshot members by
+  their path inside the bundle; one per control that changed between two
+  checkpoints. Dropped and partial records are explicit, binary content is never
+  duplicated, and redaction is whatever the bundle already decided. The JSON
+  bundle stays authoritative - this is an adapter, not a replacement.
+
+  A managed download whose CDP completion arrives before the file does is no
+  longer reported as a missing file: the staged bytes are waited for, and a
+  timeout or a genuinely missing file is still a failure rather than a path that
+  does not exist.
+
 ## 0.18.0
 
 ### Minor Changes

package/README.md

@@ -657,8 +657,56 @@ await writeTraceViewer(bundle, '/tmp/traces/checkout/viewer.html');
 ```
 
 Redaction runs before anything reaches disk, and a run cut short by a closed
-page or a size limit still leaves a readable partial trace. The viewer seeks,
-diffs and replays the captured DOM with scripts and network disabled.
+page or a size limit still leaves a readable partial trace. In `continuous`
+mode the observers are reinstalled from an init script before the page's own
+code runs, so a trace keeps recording across navigations, SPA route changes and
+same-document updates, and it records what a person cannot see in the DOM -
+typing, checking, selecting, focus and scroll - as semantic live-state records
+(issue #93). `initialCheckpoint` (on by default) captures the page as it was
+before the first action, so the first interval has a base to replay onto. The
+viewer seeks, diffs and replays the captured DOM with scripts and network
+disabled; it is a diagnostic replay of what was recorded, not a pixel-accurate
+re-execution of the session.
+
+#### Links Notation Export
+
+The bundle stays authoritative, and `writeTraceLinks()` writes a second,
+portable view of it for tools that keep semantic, actor-aware histories - audit
+graphs, agent memory, cross-run diffing (issue #94):
+
+```javascript
+import { readTrace, writeTraceLinks } from 'browser-commander';
+
+const trace = await readTrace('./run.bc-trace');
+await writeTraceLinks(trace, './run.lino', {
+  include: ['timeline', 'checkpoints', 'control-diffs'],
+});
+```
+
+A long-lived agent can have the same export written as the run happens, so a
+process that is killed still leaves everything it had recorded:
+
+```javascript
+const trace = await commander.startTrace({
+  output: './run.bc-trace',
+  links: { output: './run.lino' },
+});
+// trace.links === './run.lino'
+```
+
+One link per line: one `(timeline: ...)` link per ordered event carrying its
+sequence, time, kind, owner ids, actor, action, target and outcome; one
+`(checkpoint: ...)` link naming its HTML, state and screenshot members by their
+path inside the bundle; one `(control-diff: ...)` link per control that changed
+between two checkpoints, with `path`, `before`, `after` and who changed it.
+Dropped and partial records are written as such rather than left out, no binary
+content is ever copied into the export, and redaction is whatever the bundle
+already decided - the export only reads what was written there. The
+representation is pinned by a golden test against
+[`links-notation`](https://www.npmjs.com/package/links-notation) 0.20, whose
+Python and Rust implementations read the same file;
+`experiments/trace-links-export.mjs` records a real run and parses its export
+from Python to prove it.
 
 ### Truthful Click Results
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "browser-commander",
-  "version": "0.18.0",
+  "version": "0.19.0",
   "description": "Universal browser automation library that supports both Playwright and Puppeteer with a unified API",
   "main": "src/index.js",
   "type": "module",
@@ -68,6 +68,7 @@
   },
   "dependencies": {
     "better-sqlite3": "^12.11.1",
+    "links-notation": "^0.20.0",
     "log-lazy": "^1.0.4",
     "test-anywhere": "^0.9.1"
   },

package/src/downloads/manager.js

@@ -85,6 +85,9 @@ function playwrightDownloadContext({ browser, page }) {
  * @param {string} [options.conflict='rename'] - 'rename', 'overwrite' or 'error'
  * @param {Function} [options.filename] - Naming callback for every download
  * @param {Function} [options.validate] - Validation for every download
+ * @param {number} [options.stagingTimeout] - How long a completed download has
+ *   to become readable on disk before it is reported as failed (issue #92)
+ * @param {number} [options.stagingPollInterval] - Milliseconds between readings
  * @param {Object} [options.log] - Logger
  * @returns {Promise<Object>} Download manager
  */
@@ -99,6 +102,8 @@ export async function createDownloadManager(options = {}) {
     conflict = DOWNLOAD_CONFLICT.RENAME,
     filename,
     validate,
+    stagingTimeout,
+    stagingPollInterval,
     log,
   } = options;
 
@@ -194,20 +199,30 @@ export async function createDownloadManager(options = {}) {
     },
 
     /**
-     * Save a download whose bytes the engine has finished writing.
+     * Register work a download is not finished without.
+     *
+     * Engine events are fire-and-forget, so anything a source starts in one -
+     * saving the bytes, or waiting for them to land on disk at all - has to be
+     * registered here: otherwise `dispose()` could return while a download is
+     * still in flight and leave a `.partial` file as its only trace.
      *
-     * Engine events are fire-and-forget, so the save is registered as in-flight
-     * work: otherwise `dispose()` could return while bytes are still being
-     * written and leave a `.partial` file as the only trace of the download.
+     * @param {Promise} work - Work to wait for in `idle()` and `dispose()`
+     * @returns {Promise<void>} The same work, tracked
+     */
+    track(work) {
+      inFlight.add(work);
+      return work.finally(() => inFlight.delete(work));
+    },
+
+    /**
+     * Save a download whose bytes the engine has finished writing.
      *
      * @param {Object} artifact - Artifact record
      * @param {Object} source - `{path}` or `{stream}` to read the bytes from
      * @returns {Promise<void>} Resolves once the artifact has been published
      */
     finished(artifact, source) {
-      const work = save(artifact, source);
-      inFlight.add(work);
-      return work.finally(() => inFlight.delete(work));
+      return sink.track(save(artifact, source));
     },
 
     /**
@@ -293,7 +308,15 @@ export async function createDownloadManager(options = {}) {
   try {
     cdpSession = await openBrowserCdpSession({ engine, browser, page });
     if (cdpSession) {
-      attached.push(await attachCdpSource({ session: cdpSession, root, sink }));
+      attached.push(
+        await attachCdpSource({
+          session: cdpSession,
+          root,
+          sink,
+          stagingTimeout,
+          stagingPollInterval,
+        })
+      );
     }
   } catch (error) {
     cdpSession = undefined;

package/src/downloads/sources.js

@@ -11,6 +11,12 @@ import path from 'node:path';
 import fs from 'node:fs/promises';
 
 import { ARTIFACT_DIRECTORY_MODE } from './destination.js';
+import {
+  DEFAULT_STAGING_POLL_INTERVAL,
+  DEFAULT_STAGING_TIMEOUT,
+  describeStagingTimeout,
+  waitForStagedFile,
+} from './staging.js';
 
 /** Why a download ended without a file. */
 export const DOWNLOAD_FAILURE = Object.freeze({
@@ -85,10 +91,18 @@ export function attachPlaywrightSource({ context, sink }) {
  * @param {Object} options - Source options
  * @param {Object} options.session - CDP session with Browser domain access
  * @param {string} options.root - Managed download directory
- * @param {Object} options.sink - `{started, finished, failed}` callbacks
+ * @param {Object} options.sink - `{started, finished, failed, track}` callbacks
+ * @param {number} [options.stagingTimeout] - Budget for the bytes to land on disk
+ * @param {number} [options.stagingPollInterval] - Milliseconds between readings
  * @returns {Promise<Object>} Handle with `detach()` and `stagingDirectory`
  */
-export async function attachCdpSource({ session, root, sink }) {
+export async function attachCdpSource({
+  session,
+  root,
+  sink,
+  stagingTimeout,
+  stagingPollInterval,
+}) {
   const stagingDirectory = path.join(root, STAGING_DIRECTORY);
   await fs.mkdir(stagingDirectory, {
     recursive: true,
@@ -117,6 +131,43 @@ export async function attachCdpSource({ session, root, sink }) {
     );
   };
 
+  /**
+   * Publish a completed download once its staged bytes are readable.
+   *
+   * `state: "completed"` is the browser's word, not the filesystem's. Chromium
+   * writes `<guid>.crdownload` and renames it, so opening `<guid>` the instant
+   * the event arrives could - and did, in issue #92 - fail with `ENOENT` on a
+   * download that had in fact succeeded. Waiting for the file turns that race
+   * into either a saved artifact or a named failure, never a lost one.
+   *
+   * @param {Object} record - Artifact record
+   * @param {string} stagedPath - Path Chromium staged the bytes under
+   * @returns {Promise<void>}
+   */
+  const publishWhenStaged = async (record, stagedPath) => {
+    const budget = stagingTimeout ?? DEFAULT_STAGING_TIMEOUT;
+    const settled = await waitForStagedFile({
+      path: stagedPath,
+      timeout: budget,
+      interval: stagingPollInterval ?? DEFAULT_STAGING_POLL_INTERVAL,
+    });
+
+    if (!settled.ready) {
+      sink.failed(
+        record,
+        DOWNLOAD_FAILURE.FAILED,
+        describeStagingTimeout({
+          path: stagedPath,
+          timeout: budget,
+          reason: settled.reason,
+        })
+      );
+      return;
+    }
+
+    await sink.finished(record, { path: stagedPath, removeSource: true });
+  };
+
   const onProgress = ({ guid, state }) => {
     const record = byGuid.get(guid);
     if (!record || state === 'inProgress') {
@@ -125,10 +176,9 @@ export async function attachCdpSource({ session, root, sink }) {
 
     byGuid.delete(guid);
     if (state === 'completed') {
-      sink.finished(record, {
-        path: path.join(stagingDirectory, guid),
-        removeSource: true,
-      });
+      // Tracked before it is awaited: the wait is part of the download's
+      // lifecycle, so `idle()` and `dispose()` must not return while it runs.
+      sink.track(publishWhenStaged(record, path.join(stagingDirectory, guid)));
       return;
     }
     sink.failed(

package/src/downloads/staging.js

@@ -0,0 +1,146 @@
+/**
+ * Waiting for a staged download to actually be on disk (issue #92).
+ *
+ * `Browser.downloadProgress` with `state: "completed"` is the browser saying it
+ * has stopped downloading, not the filesystem saying the bytes are there under
+ * the name we are about to open. Chromium writes into `<guid>.crdownload` and
+ * renames it, and the event can reach us before that rename is visible - which
+ * is how a completed download became
+ * `ENOENT: no such file or directory, open '<root>/.browser-commander-staging/<guid>'`.
+ *
+ * This module turns "the browser said completed" into "the file is readable and
+ * has stopped growing", within a bounded budget. Running out of that budget is
+ * an explicit failure, never a silent success: issue #88's rule that "a
+ * download UI entry is not completion evidence" is exactly the rule being
+ * applied one level lower down.
+ */
+
+import fs from 'node:fs/promises';
+import { setTimeout as delay } from 'node:timers/promises';
+
+import { createDeadline } from '../core/readiness.js';
+
+/** How long to wait for a completed download's bytes to appear, by default. */
+export const DEFAULT_STAGING_TIMEOUT = 10000;
+
+/** How often the staged file is looked at while waiting. */
+export const DEFAULT_STAGING_POLL_INTERVAL = 25;
+
+/** Suffixes Chromium uses for a file it is still writing. */
+export const STAGING_IN_PROGRESS_SUFFIXES = Object.freeze([
+  '.crdownload',
+  '.partial',
+]);
+
+/**
+ * Look at a staged file once.
+ *
+ * @param {string} filePath - Path the engine staged the download under
+ * @param {string[]} suffixes - Suffixes that mark a file still being written
+ * @returns {Promise<{size: number|null, reason: string|null}>} What was seen
+ */
+async function observeStagedFile(filePath, suffixes) {
+  for (const suffix of suffixes) {
+    try {
+      await fs.stat(`${filePath}${suffix}`);
+      return { size: null, reason: `a ${suffix} file is still being written` };
+    } catch {
+      // Absent is what we want: the rename to the final staged name has
+      // already happened, or Chromium never used this suffix at all.
+    }
+  }
+
+  let handle;
+  try {
+    // Opening rather than stat'ing is deliberate: "readable" is the property
+    // `saveDownload()` needs, and a name that resolves is not the same thing
+    // as a file this process is allowed to read.
+    handle = await fs.open(filePath, 'r');
+  } catch (error) {
+    return {
+      size: null,
+      reason:
+        error.code === 'ENOENT'
+          ? 'the file has not appeared yet'
+          : `the file could not be opened: ${error.message}`,
+    };
+  }
+
+  try {
+    const stats = await handle.stat();
+    return { size: stats.size, reason: null };
+  } finally {
+    await handle.close();
+  }
+}
+
+/**
+ * Wait until a staged download is readable and has stopped growing.
+ *
+ * Stability is decided by two consecutive readings of the same size, the same
+ * evidence the Selenium directory watcher uses: a file that merely exists is
+ * not evidence that its last byte was written.
+ *
+ * @param {Object} options - Wait options
+ * @param {string} options.path - Path the engine staged the download under
+ * @param {number} [options.timeout] - Total budget in milliseconds
+ * @param {number} [options.interval] - Milliseconds between readings
+ * @param {string[]} [options.inProgressSuffixes] - Suffixes of a partial file
+ * @returns {Promise<{ready: boolean, bytes: number|null, reason: string|null, waitedMs: number}>} Outcome
+ */
+export async function waitForStagedFile(options = {}) {
+  const {
+    path: filePath,
+    timeout = DEFAULT_STAGING_TIMEOUT,
+    interval = DEFAULT_STAGING_POLL_INTERVAL,
+    inProgressSuffixes = STAGING_IN_PROGRESS_SUFFIXES,
+  } = options;
+
+  const deadline = createDeadline({ timeout });
+  let previousSize;
+  // Always set by the reading below before the deadline can be checked.
+  let reason;
+
+  for (;;) {
+    const seen = await observeStagedFile(filePath, inProgressSuffixes);
+    if (seen.reason !== null) {
+      reason = seen.reason;
+      // A file that vanished has to prove itself stable again from scratch.
+      previousSize = undefined;
+    } else if (seen.size === previousSize) {
+      return {
+        ready: true,
+        bytes: seen.size,
+        reason: null,
+        waitedMs: deadline.elapsedMs(),
+      };
+    } else {
+      previousSize = seen.size;
+      reason = `the file was still growing at ${seen.size} bytes`;
+    }
+
+    const remaining = deadline.remainingMs();
+    if (remaining === 0) {
+      return {
+        ready: false,
+        bytes: null,
+        reason,
+        waitedMs: deadline.elapsedMs(),
+      };
+    }
+    await delay(Math.min(interval, remaining));
+  }
+}
+
+/**
+ * The failure text a staged download that never materialized is reported under.
+ *
+ * @param {Object} options - Failure details
+ * @param {string} options.path - Path the engine staged the download under
+ * @param {number} options.timeout - Budget that was exhausted
+ * @param {string} options.reason - What the last reading saw
+ * @returns {string} Message kept verbatim on the artifact
+ */
+export function describeStagingTimeout({ path: filePath, timeout, reason }) {
+  return `the browser reported the download as completed, but ${filePath} was not readable within ${timeout}ms: ${reason}`;
+}

package/src/exports.js

@@ -237,6 +237,20 @@ export {
 } from './traces/schema.js';
 export { diffControlState, parseNdjson, readTrace } from './traces/reader.js';
 export { renderTraceViewer, writeTraceViewer } from './traces/viewer.js';
+// A Links Notation view of the same bundle, for consumers that keep semantic,
+// actor-aware histories of a run (issue #94).
+export {
+  decodeLinkText,
+  encodeLinkText,
+  formatTraceLinks,
+  traceLinks,
+  TRACE_LINK_IDS,
+  TRACE_LINK_OUTCOME,
+  TRACE_LINKS_FILE,
+  TRACE_LINKS_SECTIONS,
+  TRACE_LINKS_VERSION,
+  writeTraceLinks,
+} from './traces/links.js';
 export {
   DEFAULT_REDACT_ATTRIBUTES,
   DEFAULT_REDACT_SELECTORS,

package/src/traces/bundle.js

@@ -32,7 +32,7 @@ export const TRACE_DIRECTORY_MODE = 0o700;
 /**
  * Open a bundle for writing.
  *
- * @param {Object} options - `{output, limits, strict, now}`
+ * @param {Object} options - `{output, limits, strict, now, onEvent}`
  * @returns {Promise<Object>} The bundle writer
  */
 export async function openTraceBundle(options = {}) {
@@ -41,6 +41,7 @@ export async function openTraceBundle(options = {}) {
     strict = false,
     limits = {},
     now = () => Date.now(),
+    onEvent = null,
   } = options;
 
   if (typeof output !== 'string' || output === '') {
@@ -138,6 +139,20 @@ export async function openTraceBundle(options = {}) {
       await pending;
       written += bytes;
       counts.events += 1;
+      if (onEvent) {
+        try {
+          await onEvent(record);
+        } catch (error) {
+          // A side export that fails is a gap in that export, not in the
+          // bundle. Reporting it through `drop` would write an event and call
+          // this hook again, so it is noted where a caller can still see it.
+          problems.push({
+            reason: TRACE_DROP_REASON.WRITE_FAILED,
+            member: 'links',
+            detail: error.message,
+          });
+        }
+      }
       return record;
     } catch (error) {
       if (retry) {

package/src/traces/deadline.js

@@ -0,0 +1,33 @@
+/**
+ * A budget for one capture (issue #87).
+ *
+ * A page that stopped answering must cost the trace one record, not the run.
+ */
+
+/**
+ * Run a capture with its own deadline.
+ *
+ * @param {Promise} work - The capture
+ * @param {number} timeoutMs - Budget in milliseconds; falsy means no deadline
+ * @param {string} what - Name used in the timeout message
+ * @returns {Promise<*>} The capture's result
+ */
+export async function withDeadline(work, timeoutMs, what) {
+  if (!timeoutMs) {
+    return work;
+  }
+  let timer;
+  try {
+    return await Promise.race([
+      work,
+      new Promise((_resolve, reject) => {
+        timer = setTimeout(
+          () => reject(new Error(`${what} timed out after ${timeoutMs}ms`)),
+          timeoutMs
+        );
+      }),
+    ]);
+  } finally {
+    clearTimeout(timer);
+  }
+}