browser-commander 0.28.0 → 0.29.0

package/src/traces/rolling-bundle.js

@@ -0,0 +1,240 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { openTraceBundle } from './bundle.js';
+import { createManifest } from './schema.js';
+
+/** Rotate complete bundles at the producer's write boundary, including background events. */
+export async function openRollingBundle(options, config, traceOptions) {
+  const root = path.resolve(options.output);
+  const maxBytes =
+    config.maxBytes ?? options.limits.maxBundleBytes ?? 32 * 1024 * 1024;
+  const maxSegments = config.maxSegments ?? Infinity;
+  if (
+    !Number.isSafeInteger(maxBytes) ||
+    maxBytes < 1024 ||
+    (maxSegments !== Infinity &&
+      (!Number.isSafeInteger(maxSegments) ||
+        maxSegments < 1 ||
+        maxSegments > 100))
+  ) {
+    throw new RangeError('invalid trace rotation bounds');
+  }
+  let current,
+    sequence = 0,
+    pinned = false,
+    base = null,
+    baseEvent = null;
+  let eventSequence = 0;
+  const segments = [],
+    writers = [];
+  let queue = Promise.resolve();
+  const run = (fn) => {
+    const result = queue.then(fn);
+    queue = result.catch(() => {});
+    return result;
+  };
+  const template = () =>
+    createManifest({
+      mode: traceOptions.mode ?? 'checkpoints',
+      engine: traceOptions.commander?.engine,
+      startedAt: new Date().toISOString(),
+      stoppedAt: new Date().toISOString(),
+      dom: traceOptions.dom,
+      limits: traceOptions.limits,
+    });
+  async function open() {
+    const name = `segment-${String(++sequence).padStart(6, '0')}`;
+    current = await openTraceBundle({
+      ...options,
+      output: path.join(root, name),
+      nextSequence: () => ++eventSequence,
+      limits: { ...options.limits, maxBundleBytes: Infinity },
+      onEvent: (event) => options.onEvent?.(event, { root: current.root }),
+    });
+    writers.push(current);
+    segments.push(name);
+    if (segments.length > maxSegments) {
+      await fs.rm(path.join(root, segments.shift()), {
+        recursive: true,
+        force: true,
+      });
+    }
+    await fs.writeFile(
+      path.join(root, 'segments.json.tmp'),
+      JSON.stringify({ segments }),
+      { mode: 0o600 }
+    );
+    await fs.rename(
+      path.join(root, 'segments.json.tmp'),
+      path.join(root, 'segments.json')
+    );
+  }
+  async function prepare(bytes, carry = false) {
+    if (
+      !pinned &&
+      current.bytesWritten > 0 &&
+      current.bytesWritten + bytes > maxBytes
+    ) {
+      await current.close(template());
+      if (sequence === 0) {
+        const name = 'segment-000001';
+        const destination = path.join(root, name);
+        await fs.mkdir(destination, { mode: 0o700 });
+        for (const member of [
+          'events.ndjson',
+          'manifest.json',
+          'checkpoints',
+          'mutations',
+          'artifacts',
+          'viewer.html',
+        ]) {
+          await fs
+            .rename(path.join(root, member), path.join(destination, member))
+            .catch((error) => {
+              if (error.code !== 'ENOENT') {
+                throw error;
+              }
+            });
+        }
+        sequence = 1;
+        segments.push(name);
+      }
+      await open();
+      if (carry && base && baseEvent) {
+        const members = await current.writeCheckpoint(base);
+        await current.appendEvent({
+          ...baseEvent,
+          members,
+          reason: 'rotation-base',
+        });
+      }
+    }
+  }
+  await fs.mkdir(root, { recursive: true, mode: 0o700 });
+  if (config.lazy) {
+    current = await openTraceBundle({
+      ...options,
+      nextSequence: () => ++eventSequence,
+      limits: { ...options.limits, maxBundleBytes: Infinity },
+    });
+    writers.push(current);
+  } else {
+    await open();
+  }
+  const counts = () =>
+    writers.reduce(
+      (sum, b) => {
+        for (const k of Object.keys(sum)) {
+          sum[k] += b.counts[k];
+        }
+        return sum;
+      },
+      { checkpoints: 0, events: 0, mutationBatches: 0 }
+    );
+  return {
+    root,
+    get currentPath() {
+      return current.root;
+    },
+    get segments() {
+      return [...segments];
+    },
+    get counts() {
+      return counts();
+    },
+    get problems() {
+      return writers.flatMap((b) => b.problems);
+    },
+    get dropped() {
+      return writers.reduce((sum, b) => sum + b.dropped, 0);
+    },
+    get bytesWritten() {
+      return current.bytesWritten;
+    },
+    get mutationTruncated() {
+      return current.mutationTruncated;
+    },
+    appendEvent: (event) =>
+      run(async () => {
+        await prepare(Buffer.byteLength(JSON.stringify(event)) + 256, true);
+        const result = await current.appendEvent(event);
+        if (event.kind === 'checkpoint') {
+          baseEvent = result;
+        }
+        if (
+          event.kind === 'checkpoint' ||
+          (event.kind === 'mutations' && pinned === 'mutations')
+        ) {
+          pinned = false;
+        }
+        return result;
+      }),
+    writeCheckpoint: (checkpoint) =>
+      run(async () => {
+        const bytes =
+          Buffer.byteLength(checkpoint.html ?? '') +
+          Buffer.byteLength(JSON.stringify(checkpoint.state ?? {})) +
+          (checkpoint.screenshot?.length ?? 0) +
+          1024;
+        await prepare(bytes);
+        pinned = 'checkpoint';
+        base = checkpoint;
+        return current.writeCheckpoint(checkpoint);
+      }),
+    writeMutations: (index, batches) =>
+      run(async () => {
+        await prepare(Buffer.byteLength(JSON.stringify(batches)) + 256, true);
+        const result = await current.writeMutations(index, batches);
+        if (pinned !== 'checkpoint') {
+          pinned = result ? 'mutations' : false;
+        }
+        return result;
+      }),
+    writeMember: (member, contents, writeOptions) =>
+      run(async () => {
+        await prepare(Buffer.byteLength(contents));
+        return current.writeMember(member, contents, writeOptions);
+      }),
+    writeArtifact: (contents, extension) =>
+      run(async () => {
+        await prepare(Buffer.byteLength(contents));
+        const result = await current.writeArtifact(contents, extension);
+        return result
+          ? {
+              ...result,
+              member: segments.length
+                ? `${segments.at(-1)}/${result.member}`
+                : result.member,
+            }
+          : null;
+      }),
+    drop: (record) => run(() => current.drop(record)),
+    abort: () => run(() => current.abort()),
+    close: (manifest) =>
+      run(async () => {
+        manifest = await current.close(manifest);
+        if (writers.some((writer) => writer.dropped)) {
+          manifest.outcome = 'partial';
+        }
+        // Finished segments get the final capabilities and outcome as well.
+        for (const name of segments) {
+          const file = path.join(root, name, 'manifest.json');
+          const previous = JSON.parse(await fs.readFile(file, 'utf8'));
+          await fs.writeFile(
+            file,
+            JSON.stringify({
+              ...manifest,
+              counts: previous.counts,
+              dropped: previous.dropped,
+            }),
+            { mode: 0o600 }
+          );
+        }
+        return {
+          ...manifest,
+          counts: counts(),
+          dropped: writers.reduce((sum, b) => sum + b.dropped, 0),
+        };
+      }),
+  };
+}

package/src/traces/rolling.js

@@ -1,120 +1,15 @@
-import fs from 'node:fs/promises';
 import path from 'node:path';
 
-/** Bounded complete bundles, each independently readable after a crash. */
+/** Keep the ordinary trace handle while routing every write through rotation. */
 export async function startRollingTrace(options, start) {
+  const { openRollingBundle } = await import('./rolling-bundle.js');
   const config = options.limits.rotate === true ? {} : options.limits.rotate;
-  const maxBytes =
-    config.maxBytes ?? options.limits.maxBundleBytes ?? 32 * 1024 * 1024;
-  const maxSegments = config.maxSegments ?? 4;
-  if (
-    !Number.isSafeInteger(maxBytes) ||
-    maxBytes < 1024 ||
-    !Number.isSafeInteger(maxSegments) ||
-    maxSegments < 1 ||
-    maxSegments > 100
-  ) {
-    throw new RangeError('invalid trace rotation bounds');
-  }
-  const root = path.resolve(options.output);
-  await fs.mkdir(root, { recursive: true, mode: 0o700 });
-  let current,
-    sequence = 0,
-    stopping,
-    failure,
-    ticking = false;
-  const segments = [];
-  let queue = Promise.resolve();
-  const enqueue = (action) => {
-    const task = queue.then(action);
-    queue = task.catch((error) => {
-      failure ??= error;
-    });
-    return task;
-  };
-  async function open() {
-    const name = `segment-${String(++sequence).padStart(6, '0')}`;
-    const output = path.join(root, name);
-    current = await start({
-      ...options,
-      output,
-      limits: { ...options.limits, rotate: false, maxBundleBytes: maxBytes },
-      links: options.links
-        ? { ...options.links, output: path.join(output, 'trace.lino') }
-        : null,
-    });
-    segments.push(name);
-    while (segments.length > maxSegments) {
-      await fs.rm(path.join(root, segments.shift()), {
-        recursive: true,
-        force: true,
-      });
-    }
-    const temporary = path.join(root, 'segments.json.tmp');
-    await fs.writeFile(temporary, JSON.stringify({ segments }), {
-      mode: 0o600,
-    });
-    await fs.rename(temporary, path.join(root, 'segments.json'));
-  }
-  await open();
-  const timer = setInterval(() => {
-    if (!stopping && !ticking) {
-      ticking = true;
-      enqueue(async () => {
-        if (current.bytesWritten >= maxBytes * 0.8) {
-          await current.stop();
-          await open();
-        }
-      })
-        .catch(() => clearInterval(timer))
-        .finally(() => {
-          ticking = false;
-        });
-    }
-  }, 100);
-  timer.unref?.();
-  return {
-    path: root,
-    mode: options.mode,
-    checkpoint: (...args) =>
-      enqueue(() => {
-        if (stopping) {
-          throw new Error('trace has stopped');
-        }
-        return current.checkpoint(...args);
-      }),
-    event: (...args) =>
-      enqueue(async () => {
-        if (stopping) {
-          throw new Error('trace has stopped');
-        }
-        if (
-          current.bytesWritten +
-            Buffer.byteLength(JSON.stringify(args)) +
-            256 >=
-          maxBytes * 0.8
-        ) {
-          await current.stop();
-          await open();
-        }
-        return current.event(...args);
-      }),
-    stop: (...args) => {
-      stopping ??= (async () => {
-        clearInterval(timer);
-        await queue;
-        const result = await current.stop(...args);
-        if (args[0]?.discard) {
-          await fs.rm(root, { recursive: true, force: true });
-        }
-        if (failure) {
-          throw failure;
-        }
-        return { ...result, path: root, segments: [...segments] };
-      })();
-      return stopping;
-    },
-  };
+  return start({
+    ...options,
+    limits: { ...options.limits, rotate: false },
+    openBundle: (bundleOptions) =>
+      openRollingBundle(bundleOptions, config, options),
+  });
 }
 
 /** Combine segment timelines while keeping checkpoint member paths distinct. */
@@ -174,6 +69,8 @@ export async function readRollingTrace(root, index, read) {
       locations.get(index)?.opened.html(locations.get(index).index),
     state: (index) =>
       locations.get(index)?.opened.state(locations.get(index).index),
+    screenshot: (index) =>
+      locations.get(index)?.opened.screenshot(locations.get(index).index),
     mutations: (index) =>
       locations.get(index)?.opened.mutations(locations.get(index).index) ?? [],
   };

package/src/traces/stop.js

@@ -134,6 +134,7 @@ export function createTraceStop(context) {
       checkpoints: [...checkpoints],
       problems: [...bundle.problems, ...(linksSink?.problems ?? [])],
       links: linksSink?.path ?? null,
+      ...(bundle.segments?.length ? { segments: bundle.segments } : {}),
     };
     setStopped(result);
 

package/tests/e2e/issue-160.e2e.test.js

@@ -0,0 +1,291 @@
+import { it } from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import net from 'node:net';
+import {
+  startTrace,
+  readTrace,
+  connectOrLaunch,
+  connectBrowser,
+} from '../../src/index.js';
+import { isNavigationError } from '../../src/core/navigation-safety.js';
+import { startProcess } from '../../src/utilities/subprocess.js';
+import {
+  closeOwnedSession,
+  closeRemoteBrowser,
+  SESSION_METADATA,
+  probeSession,
+} from '../../src/browser/persistent-session.js';
+import {
+  launchE2EBrowser,
+  CHROME_LAUNCH_OPTIONS,
+  PARITY_CHROME,
+} from '../helpers/e2e-browser.js';
+import { sendHtml, startFixtureHost } from '../helpers/fixture-server.js';
+
+const options = { skip: !process.env.RUN_E2E, timeout: 60000 };
+async function removeProfile(userDataDir) {
+  await fs.rm(userDataDir, {
+    recursive: true,
+    force: true,
+    maxRetries: 10,
+    retryDelay: 100,
+  });
+}
+async function port() {
+  const server = net.createServer();
+  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
+  const number = server.address().port;
+  await new Promise((resolve) => server.close(resolve));
+  return number;
+}
+async function until(predicate) {
+  for (let attempt = 0; attempt < 100; attempt++) {
+    if (await predicate()) {
+      return;
+    }
+    await new Promise((resolve) => setTimeout(resolve, 50));
+  }
+  throw new Error('browser condition did not settle');
+}
+
+for (const engine of ['playwright', 'puppeteer']) {
+  it(
+    `records bounded, private mutations and interrupted goto: ${engine}`,
+    options,
+    async () => {
+      const output = await fs.mkdtemp(path.join(os.tmpdir(), 'bc-160-e2e-'));
+      const launched = await launchE2EBrowser({ engine });
+      let trace;
+      let slowStarted;
+      const started = new Promise((resolve) => {
+        slowStarted = resolve;
+      });
+      const server = await startFixtureHost((route, req, res) => {
+        if (route === '/private') {
+          res.setHeader('content-type', 'application/json');
+          res.setHeader('X-Xsrftoken', 'PRIVATE_HEADER');
+          res.end(
+            JSON.stringify({ token: 'PRIVATE_RESPONSE', public: 'visible' })
+          );
+        } else if (route === '/slow') {
+          slowStarted();
+          req.once('close', () => res.destroy());
+        } else {
+          sendHtml(
+            res,
+            '<div id="app">public</div><input type="hidden" value="PRIVATE_HIDDEN"><meta name="csrf-token" content="PRIVATE_META"><script>window.xsrfToken="PRIVATE_SCRIPT";</script>'
+          );
+        }
+      });
+      try {
+        await launched.page.goto(server.baseUrl);
+        trace = await startTrace({
+          page: launched.page,
+          engine,
+          output,
+          mode: 'continuous',
+          screenshots: false,
+          network: { bodies: true, har: true },
+          gzip: engine === 'puppeteer',
+          privacy: { redactPatterns: [/PRIVATE_DYNAMIC/g] },
+          links: { dom: 'full', output: path.join(output, 'trace.lino') },
+        });
+        await launched.page.evaluate(async () => {
+          await fetch('/private?xsrf=PRIVATE_QUERY', {
+            method: 'POST',
+            headers: {
+              'content-type': 'application/json',
+              'X-Xsrftoken': 'PRIVATE_HEADER',
+            },
+            body: JSON.stringify({
+              password: 'PRIVATE_PASSWORD',
+              _xsrf: 'PRIVATE_BODY',
+              public: 'visible',
+            }),
+          }).then((response) => response.text());
+        });
+        await launched.page.evaluate(() => {
+          const app = document.querySelector('#app');
+          app.innerHTML = `<section>${'public'.repeat(1000)}</section>`;
+          for (let i = 0; i < 50; i++) {
+            const node = document.createElement('span');
+            node.textContent = `public-${i} PRIVATE_DYNAMIC`;
+            app.appendChild(node);
+          }
+          document.querySelector('meta').content = 'PRIVATE_CHANGED';
+        });
+        await trace.checkpoint('after');
+        await trace.stop();
+        const opened = await readTrace(output);
+        const batches = await opened.mutations(opened.checkpoints[0].index);
+        const data = JSON.stringify(batches);
+        assert.ok(data.length < 60000, `mutation bytes: ${data.length}`);
+        for (const batch of batches) {
+          for (const record of batch.records ?? []) {
+            assert.equal(record.target?.html, undefined);
+          }
+        }
+        const contents = [
+          data,
+          JSON.stringify(opened.events),
+          await fs.readFile(path.join(output, 'trace.lino'), 'utf8'),
+          await fs.readFile(path.join(output, 'network.har'), 'utf8'),
+          ...(await Promise.all(
+            opened.checkpoints.map((cp) => opened.html(cp.index))
+          )),
+        ];
+        for (const text of contents) {
+          assert.doesNotMatch(text, /PRIVATE_/);
+          assert.match(text, /public|visible/);
+        }
+        const interrupted = launched.page
+          .goto(`${server.baseUrl}/slow`)
+          .catch((error) => error);
+        await started;
+        await launched.page.goto(server.baseUrl);
+        assert.equal(isNavigationError(await interrupted), true);
+      } finally {
+        await trace?.stop();
+        await launched.cleanup();
+        await server.close();
+        await fs.rm(output, { recursive: true, force: true });
+      }
+    }
+  );
+}
+
+it(
+  'adopts a verified profile, heartbeats an idle controller and closes later tabs',
+  options,
+  async () => {
+    const userDataDir = await fs.mkdtemp(
+      path.join(os.tmpdir(), 'bc-160-session-')
+    );
+    const remoteDebuggingPort = await port();
+    const settings = {
+      engine: 'playwright',
+      userDataDir,
+      remoteDebuggingPort,
+      idleTimeoutMs: 2500,
+      headless: true,
+      ...CHROME_LAUNCH_OPTIONS,
+    };
+    let session, originalOwner;
+    try {
+      session = await connectOrLaunch(settings);
+      originalOwner = JSON.parse(
+        await fs.readFile(path.join(userDataDir, SESSION_METADATA), 'utf8')
+      );
+      const originalUrl = session.page.url();
+      const preferred = await session.browser.contexts()[0].newPage();
+      await preferred.goto('data:text/html,<p>preferred tab</p>');
+      const preferredUrl = preferred.url();
+      assert.ok(
+        (await session.reusePage({
+          urlMatchers: [preferredUrl, originalUrl],
+        })) === preferred,
+        'ranked URL matchers must override the remembered target'
+      );
+      await session.detach();
+      session = await connectOrLaunch({
+        ...settings,
+        urlMatchers: [originalUrl, preferredUrl],
+      });
+      assert.equal(session.page.url(), originalUrl);
+      await session.detach();
+      await fs.rm(path.join(userDataDir, SESSION_METADATA));
+      session = await connectOrLaunch({
+        ...settings,
+        adoptExisting: true,
+        closeNewTabs: true,
+        singleTab: true,
+      });
+      assert.equal(session.adopted, true);
+      const selected = session.page;
+      await session.browser
+        .contexts()[0]
+        .newPage()
+        .catch(() => {});
+      await until(() => session.browser.contexts()[0].pages().length === 1);
+      assert.equal(session.page, selected);
+      await selected.close();
+      const replacement = await session.reusePage();
+      assert.equal(replacement.isClosed(), false);
+      await new Promise((resolve) => setTimeout(resolve, 4000));
+      assert.ok(await probeSession(`http://127.0.0.1:${remoteDebuggingPort}`));
+      await session.detach();
+      await closeOwnedSession(
+        path.join(userDataDir, SESSION_METADATA),
+        JSON.parse(
+          await fs.readFile(path.join(userDataDir, SESSION_METADATA), 'utf8')
+        )
+      );
+    } finally {
+      await session?.detach();
+      const metadataFile = path.join(userDataDir, SESSION_METADATA);
+      const metadata = await fs
+        .readFile(metadataFile, 'utf8')
+        .then(JSON.parse, () => null);
+      if (metadata) {
+        await closeOwnedSession(metadataFile, metadata);
+      } else if (originalOwner) {
+        const version = await probeSession(
+          `http://127.0.0.1:${remoteDebuggingPort}`
+        );
+        if (
+          version?.webSocketDebuggerUrl === originalOwner.webSocketDebuggerUrl
+        ) {
+          await closeRemoteBrowser(version.webSocketDebuggerUrl);
+        }
+      }
+      await removeProfile(userDataDir);
+    }
+  }
+);
+
+it(
+  'reconnects to headed Chrome with no page targets in its default context',
+  options,
+  async () => {
+    const userDataDir = await fs.mkdtemp(
+      path.join(os.tmpdir(), 'bc-160-zero-pages-')
+    );
+    const remoteDebuggingPort = await port();
+    const endpoint = `http://127.0.0.1:${remoteDebuggingPort}`;
+    const process = startProcess(PARITY_CHROME, [
+      ...(CHROME_LAUNCH_OPTIONS.args ?? []),
+      `--user-data-dir=${userDataDir}`,
+      `--remote-debugging-port=${remoteDebuggingPort}`,
+      '--no-startup-window',
+      '--keep-alive-for-test',
+      '--no-first-run',
+    ]);
+    let connection;
+    try {
+      await until(() => probeSession(endpoint));
+      const targets = await fetch(`${endpoint}/json/list`).then((response) =>
+        response.json()
+      );
+      assert.equal(
+        targets.filter((target) => target.type === 'page').length,
+        0
+      );
+      connection = await connectBrowser({
+        engine: 'playwright',
+        cdpEndpoint: endpoint,
+      });
+      assert.ok(connection.page);
+      await connection.page.goto('data:text/html,<p>public profile page</p>');
+      await connection.detach();
+      assert.ok(await probeSession(endpoint));
+    } finally {
+      await connection?.detach();
+      process.kill();
+      await process.exited;
+      await removeProfile(userDataDir);
+    }
+  }
+);

package/tests/fixtures/traces/conformance/expected/bundle/checkpoints/0002.html

@@ -1,2 +1,2 @@
 <!DOCTYPE html>
-<html><body><form><input name="user" value="bob"><textarea>sk-zzz9 note</textarea></form></body></html>
+<html><body><form><input name="user" value="bob"><textarea>[redacted] note</textarea></form></body></html>

package/tests/fixtures/traces/conformance/expected/bundle/checkpoints/0003.html

@@ -1,2 +1,2 @@
 <!DOCTYPE html>
-<html><body><form><input name="user" value="bob"><textarea>sk-zzz9 note</textarea></form></body></html>
+<html><body><form><input name="user" value="bob"><textarea>[redacted] note</textarea></form></body></html>