browser-commander 0.17.1 → 0.18.0

package/tests/unit/tests/tracing.test.js

@@ -0,0 +1,299 @@
+import { describe, it } from 'node:test';
+import assert from 'node:assert';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import {
+  finishScenarioTrace,
+  normalizeTestScreenshots,
+  normalizeTestTrace,
+  resolveTestDownloads,
+  resolveTraceSetting,
+  startScenarioTrace,
+  TEST_DOWNLOADS_DIRNAME,
+  TRACE_BUNDLE_SUFFIX,
+  traceOutputPath,
+} from '../../../src/tests/tracing.js';
+import { readTrace } from '../../../src/traces/reader.js';
+import { TRACE_FILES, TRACE_MODE } from '../../../src/traces/schema.js';
+import {
+  createFakeCommander,
+  createFakePage,
+  useTempTraceDirectory,
+} from '../../helpers/trace-fixtures.js';
+
+describe('test runner tracing (issues #87 and #88)', () => {
+  const directory = useTempTraceDirectory();
+
+  describe('normalizing what a scenario asked for', () => {
+    it('should treat no trace option as off', () => {
+      assert.strictEqual(normalizeTestTrace(undefined), 'off');
+      assert.strictEqual(normalizeTestTrace(null), 'off');
+      assert.strictEqual(normalizeTestTrace(false), 'off');
+    });
+
+    it('should read true as recording the whole run', () => {
+      assert.strictEqual(normalizeTestTrace(true), 'on');
+    });
+
+    it('should keep the named modes', () => {
+      assert.strictEqual(
+        normalizeTestTrace('retain-on-failure'),
+        'retain-on-failure'
+      );
+      assert.strictEqual(
+        normalizeTestTrace('on-first-retry'),
+        'on-first-retry'
+      );
+    });
+
+    it('should refuse a mode it does not know', () => {
+      assert.throws(
+        () => normalizeTestTrace('retain-on-tuesday'),
+        /trace must be one of off, on, retain-on-failure, on-first-retry/
+      );
+    });
+
+    it('should default screenshots to only-on-failure', () => {
+      assert.strictEqual(
+        normalizeTestScreenshots(undefined),
+        'only-on-failure'
+      );
+      assert.strictEqual(normalizeTestScreenshots(true), 'on');
+      assert.strictEqual(normalizeTestScreenshots(false), 'off');
+      assert.strictEqual(normalizeTestScreenshots('off'), 'off');
+    });
+
+    it('should refuse a screenshot mode it does not know', () => {
+      assert.throws(
+        () => normalizeTestScreenshots('sometimes'),
+        /screenshots must be one of/
+      );
+    });
+  });
+
+  describe('deciding whether an attempt records', () => {
+    it('should not record when tracing is off', () => {
+      assert.strictEqual(resolveTraceSetting('off'), null);
+      assert.strictEqual(resolveTraceSetting('off', 3), null);
+    });
+
+    it('should keep every bundle when tracing is on', () => {
+      assert.deepStrictEqual(resolveTraceSetting('on'), {
+        recorderMode: TRACE_MODE.CONTINUOUS,
+        retainOnFailure: false,
+      });
+    });
+
+    it('should record but only keep a failure otherwise', () => {
+      assert.deepStrictEqual(resolveTraceSetting('retain-on-failure'), {
+        recorderMode: TRACE_MODE.RETAIN_ON_FAILURE,
+        retainOnFailure: true,
+      });
+    });
+
+    it('should wait for the retry when asked to', () => {
+      assert.strictEqual(resolveTraceSetting('on-first-retry', 1), null);
+      assert.deepStrictEqual(resolveTraceSetting('on-first-retry', 2), {
+        recorderMode: TRACE_MODE.RETAIN_ON_FAILURE,
+        retainOnFailure: true,
+      });
+    });
+  });
+
+  // The expectations resolve their paths the way the code under test does.
+  // A POSIX literal is not a path on Windows: `path.resolve('/a', 'x')` there
+  // is `D:\\a\\x`, because a rooted path with no drive means the current one.
+  describe('where artifacts go', () => {
+    it('should name a bundle after the scenario', () => {
+      assert.strictEqual(
+        traceOutputPath({ artifactsDir: '/a', safeName: 'checkout' }),
+        path.resolve('/a', `checkout${TRACE_BUNDLE_SUFFIX}`)
+      );
+    });
+
+    it('should keep each retry in its own bundle', () => {
+      assert.strictEqual(
+        traceOutputPath({
+          artifactsDir: '/a',
+          safeName: 'checkout',
+          attempt: 2,
+        }),
+        path.resolve('/a', `checkout.attempt-2${TRACE_BUNDLE_SUFFIX}`)
+      );
+    });
+
+    it('should keep a scenario downloads in its own artifact directory', () => {
+      assert.deepStrictEqual(
+        resolveTestDownloads(true, {
+          artifactsDir: '/a',
+          safeName: 'checkout',
+        }),
+        { directory: path.resolve('/a', 'checkout', TEST_DOWNLOADS_DIRNAME) }
+      );
+    });
+
+    it('should keep the rest of the download options', () => {
+      assert.deepStrictEqual(
+        resolveTestDownloads(
+          { timeout: 5000, allowOverwrite: true },
+          { artifactsDir: '/a', safeName: 'checkout' }
+        ),
+        {
+          timeout: 5000,
+          allowOverwrite: true,
+          directory: path.resolve('/a', 'checkout', TEST_DOWNLOADS_DIRNAME),
+        }
+      );
+    });
+
+    it('should not move downloads the caller placed itself', () => {
+      assert.deepStrictEqual(
+        resolveTestDownloads(
+          { directory: '/somewhere/else' },
+          { artifactsDir: '/a', safeName: 'checkout' }
+        ),
+        { directory: '/somewhere/else' }
+      );
+    });
+
+    it('should leave downloads alone when they are not enabled', () => {
+      assert.strictEqual(
+        resolveTestDownloads(false, { artifactsDir: '/a', safeName: 'x' }),
+        false
+      );
+      assert.strictEqual(
+        resolveTestDownloads(undefined, { artifactsDir: '/a', safeName: 'x' }),
+        undefined
+      );
+    });
+  });
+
+  describe('recording one scenario', () => {
+    /**
+     * Start a scenario trace over a fake page.
+     *
+     * @param {Object} [options] - Overrides for `startScenarioTrace`
+     * @returns {Promise<Object>} `{started, page}`
+     */
+    const start = async (options = {}) => {
+      const { page = createFakePage(), ...overrides } = options;
+      const started = await startScenarioTrace({
+        commander: createFakeCommander(page),
+        trace: 'retain-on-failure',
+        artifactsDir: directory.path,
+        safeName: 'checkout',
+        ...overrides,
+        page,
+      });
+      return { started, page };
+    };
+
+    it('should not record when the scenario did not ask for a trace', async () => {
+      const { started } = await start({ trace: 'off' });
+
+      assert.strictEqual(started, null);
+      assert.deepStrictEqual(await fs.readdir(directory.path), []);
+    });
+
+    it('should not record when the runner has nowhere to write', async () => {
+      const { started } = await start({ trace: 'on', artifactsDir: undefined });
+
+      assert.strictEqual(started, null);
+    });
+
+    it('should record DOM mutations whenever it records at all', async () => {
+      const { started, page } = await start({ trace: 'retain-on-failure' });
+
+      assert.ok(
+        page.state.evaluated.some(
+          (call) => call.name === 'installMutationRecorderInPage'
+        )
+      );
+      await started.trace.stop({ discard: true });
+    });
+
+    it('should throw away a bundle a passing test left behind', async () => {
+      const { started } = await start({ trace: 'retain-on-failure' });
+
+      const stopped = await finishScenarioTrace(started);
+
+      assert.strictEqual(stopped.discarded, true);
+      assert.deepStrictEqual(await fs.readdir(directory.path), []);
+    });
+
+    it('should keep a failing test bundle, with the error on the timeline', async () => {
+      const { started } = await start({ trace: 'retain-on-failure' });
+
+      const stopped = await finishScenarioTrace(started, {
+        error: new Error('expected 2 items, found 1'),
+      });
+
+      assert.ok(!stopped.discarded);
+      const reader = await readTrace(stopped.path);
+      const fatal = reader.events.find((event) => event.fatal);
+      assert.strictEqual(fatal.message, 'expected 2 items, found 1');
+      assert.strictEqual(reader.checkpoints.at(-1).name, 'failure');
+      assert.strictEqual(reader.checkpoints.at(-1).actor, 'runner');
+    });
+
+    it('should keep a passing test bundle when tracing is simply on', async () => {
+      const { started } = await start({ trace: 'on' });
+
+      const stopped = await finishScenarioTrace(started);
+
+      assert.ok(!stopped.discarded);
+      assert.strictEqual(
+        (await readTrace(stopped.path)).checkpoints.at(-1).name,
+        'final'
+      );
+    });
+
+    it('should leave a viewer beside every bundle it keeps', async () => {
+      const { started } = await start({ trace: 'on' });
+
+      const stopped = await finishScenarioTrace(started);
+
+      const viewer = await fs.readFile(
+        path.join(stopped.path, TRACE_FILES.VIEWER),
+        'utf8'
+      );
+      assert.match(viewer, /Browser Commander trace/);
+    });
+
+    it('should take a failure screenshot even when passing runs take none', async () => {
+      const { started, page } = await start({
+        trace: 'retain-on-failure',
+        screenshots: 'only-on-failure',
+      });
+
+      await finishScenarioTrace(started, { error: new Error('nope') });
+
+      assert.strictEqual(page.state.screenshots, 1);
+    });
+
+    it('should still stop when the final checkpoint cannot be captured', async () => {
+      const page = createFakePage();
+      const { started } = await start({ page, trace: 'on' });
+      page.failEvaluate(
+        'captureSnapshotInPage',
+        'Target page, context or browser has been closed'
+      );
+      const noted = [];
+
+      const stopped = await finishScenarioTrace(started, {
+        log: { debug: (message) => noted.push(message) },
+      });
+
+      assert.ok(stopped.path);
+      assert.ok((await readTrace(stopped.path)).events.length > 0);
+      // The page closing is a gap in the trace, not a second failure on top of
+      // whatever the test was already reporting.
+      assert.ok(noted.every((message) => message.startsWith('[trace]')));
+    });
+
+    it('should do nothing when there was no trace to finish', async () => {
+      assert.strictEqual(await finishScenarioTrace(null), null);
+    });
+  });
+});

package/tests/unit/traces/bundle.test.js

@@ -0,0 +1,260 @@
+import { describe, it } from 'node:test';
+import assert from 'node:assert';
+import crypto from 'node:crypto';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import {
+  openTraceBundle,
+  TRACE_FILE_MODE,
+} from '../../../src/traces/bundle.js';
+import {
+  createManifest,
+  TRACE_DROP_REASON,
+  TRACE_EVENT,
+  TRACE_FILES,
+  TRACE_MODE,
+  TRACE_OUTCOME,
+} from '../../../src/traces/schema.js';
+import { useTempTraceDirectory } from '../../helpers/trace-fixtures.js';
+
+describe('trace bundle writer (issue #87)', () => {
+  const directory = useTempTraceDirectory();
+
+  /**
+   * Open a bundle inside the suite's temporary directory.
+   *
+   * @param {Object} [options] - Options for `openTraceBundle`
+   * @returns {Promise<Object>} The bundle writer
+   */
+  const open = (options = {}) =>
+    openTraceBundle({ output: path.join(directory.path, 'run'), ...options });
+
+  /**
+   * List a bundle's members, relative to its root.
+   *
+   * @param {string} root - Bundle directory
+   * @returns {Promise<string[]>} Sorted member names
+   */
+  const members = async (root) =>
+    (await fs.readdir(root, { recursive: true }))
+      .map((entry) => entry.split(path.sep).join('/'))
+      .sort();
+
+  /**
+   * Read the timeline back.
+   *
+   * @param {string} root - Bundle directory
+   * @returns {Promise<Object[]>} Parsed events
+   */
+  const timeline = async (root) =>
+    (await fs.readFile(path.join(root, TRACE_FILES.EVENTS), 'utf8'))
+      .split('\n')
+      .filter(Boolean)
+      .map((line) => JSON.parse(line));
+
+  /**
+   * The first gap the bundle recorded.
+   *
+   * @param {string} root - Bundle directory
+   * @returns {Promise<Object|undefined>} The dropped event
+   */
+  const firstDrop = async (root) =>
+    (await timeline(root)).find((event) => event.kind === TRACE_EVENT.DROPPED);
+
+  it('should refuse to open without an output path', async () => {
+    await assert.rejects(openTraceBundle({}), /trace output must be a path/);
+  });
+
+  it('should write the layout the format documents', async () => {
+    const bundle = await open();
+
+    await bundle.writeCheckpoint({
+      index: 1,
+      html: '<html></html>',
+      state: { url: 'https://example.com' },
+      screenshot: Buffer.from('png'),
+    });
+    await bundle.writeMutations(1, [{ records: [] }]);
+    await bundle.writeArtifact('report body', '.txt');
+    await bundle.close(createManifest({ mode: TRACE_MODE.CHECKPOINTS }));
+
+    assert.deepStrictEqual(await members(bundle.root), [
+      'artifacts',
+      `artifacts/${crypto.createHash('sha256').update('report body').digest('hex')}.txt`,
+      'checkpoints',
+      'checkpoints/0001.html',
+      'checkpoints/0001.png',
+      'checkpoints/0001.state.json',
+      'events.ndjson',
+      'manifest.json',
+      'mutations',
+      'mutations/0001.ndjson',
+    ]);
+  });
+
+  it('should number events and stamp them with both clocks', async () => {
+    const bundle = await open();
+
+    const first = await bundle.appendEvent({ kind: TRACE_EVENT.CHECKPOINT });
+    const second = await bundle.appendEvent({ kind: TRACE_EVENT.CONSOLE });
+
+    assert.strictEqual(first.sequence, 1);
+    assert.strictEqual(second.sequence, 2);
+    assert.ok(Date.parse(first.at) > 0);
+    assert.ok(second.monotonicMs >= first.monotonicMs);
+  });
+
+  it('should keep a trace readable for its owner only', async () => {
+    if (process.platform === 'win32') {
+      return;
+    }
+    const bundle = await open();
+
+    await bundle.appendEvent({ kind: TRACE_EVENT.CHECKPOINT });
+    await bundle.close(createManifest({ mode: TRACE_MODE.CHECKPOINTS }));
+
+    const mode = (await fs.stat(path.join(bundle.root, TRACE_FILES.MANIFEST)))
+      .mode;
+    assert.strictEqual(mode & 0o777, TRACE_FILE_MODE);
+  });
+
+  it('should store one resource once, however often it is written', async () => {
+    const bundle = await open();
+
+    const first = await bundle.writeArtifact('same bytes', '.txt');
+    const second = await bundle.writeArtifact('same bytes', '.txt');
+
+    assert.strictEqual(first.member, second.member);
+    assert.strictEqual(first.deduplicated, false);
+    assert.strictEqual(second.deduplicated, true);
+    assert.deepStrictEqual(
+      (await fs.readdir(path.join(bundle.root, TRACE_FILES.ARTIFACTS_DIR)))
+        .length,
+      1
+    );
+  });
+
+  it('should drop a member over the per-resource limit and say so', async () => {
+    const bundle = await open({ limits: { maxResourceBytes: 16 } });
+
+    const result = await bundle.writeMember(
+      'checkpoints/0001.html',
+      'x'.repeat(64)
+    );
+    await bundle.close(createManifest({ mode: TRACE_MODE.CHECKPOINTS }));
+
+    assert.strictEqual(result, null);
+    assert.strictEqual(bundle.dropped, 1);
+    const dropped = await firstDrop(bundle.root);
+    assert.strictEqual(dropped.reason, TRACE_DROP_REASON.SIZE_LIMIT);
+    assert.strictEqual(dropped.member, 'checkpoints/0001.html');
+  });
+
+  it('should report a partial trace once anything was dropped', async () => {
+    const bundle = await open({ limits: { maxResourceBytes: 4 } });
+
+    await bundle.writeMember('checkpoints/0001.html', 'over the limit');
+    const manifest = await bundle.close(
+      createManifest({ mode: TRACE_MODE.CHECKPOINTS })
+    );
+
+    assert.strictEqual(manifest.outcome, TRACE_OUTCOME.PARTIAL);
+    assert.strictEqual(manifest.dropped, 1);
+  });
+
+  it('should raise the first problem in strict mode instead of recording it', async () => {
+    const bundle = await open({
+      strict: true,
+      limits: { maxResourceBytes: 4 },
+    });
+
+    await assert.rejects(
+      bundle.writeMember('checkpoints/0001.html', 'over the limit'),
+      /dropped: size-limit/
+    );
+  });
+
+  it('should record a write it could not perform', async () => {
+    const bundle = await open();
+    // A member whose parent is a file, not a directory, cannot be written.
+    await bundle.writeMember('blocked', 'i am a file');
+
+    const result = await bundle.writeMember('blocked/0001.html', 'content');
+
+    assert.strictEqual(result, null);
+    assert.strictEqual(
+      bundle.problems[0].reason,
+      TRACE_DROP_REASON.WRITE_FAILED
+    );
+  });
+
+  it('should write nothing for an empty batch of mutations', async () => {
+    const bundle = await open();
+
+    assert.strictEqual(await bundle.writeMutations(1, []), null);
+    assert.strictEqual(await bundle.writeMutations(1, null), null);
+    assert.strictEqual(bundle.counts.mutationBatches, 0);
+  });
+
+  it('should count what it wrote in the manifest', async () => {
+    const bundle = await open();
+
+    await bundle.writeCheckpoint({ index: 1, html: '<html></html>' });
+    await bundle.writeMutations(1, [{ records: [1] }, { records: [2] }]);
+    const manifest = await bundle.close(
+      createManifest({ mode: TRACE_MODE.CONTINUOUS })
+    );
+
+    assert.strictEqual(manifest.counts.checkpoints, 1);
+    assert.strictEqual(manifest.counts.mutationBatches, 2);
+    assert.strictEqual(manifest.outcome, TRACE_OUTCOME.COMPLETE);
+  });
+
+  it('should record a dropped member even under a tiny resource limit', async () => {
+    // The record of a gap must not be the next thing that falls through it.
+    const bundle = await open({ limits: { maxResourceBytes: 16 } });
+
+    await bundle.writeMember('checkpoints/0001.html', 'x'.repeat(64));
+
+    const dropped = await firstDrop(bundle.root);
+    assert.strictEqual(dropped.reason, TRACE_DROP_REASON.SIZE_LIMIT);
+  });
+
+  it('should write the timeline in the order the events happened', async () => {
+    const bundle = await open();
+
+    // Observers report what happened without awaiting the record, so the
+    // writes overlap; the file must still read as the story in order.
+    await Promise.all(
+      Array.from({ length: 50 }, (_unused, index) =>
+        bundle.appendEvent({ kind: TRACE_EVENT.CONSOLE, index })
+      )
+    );
+
+    const written = await timeline(bundle.root);
+    assert.deepStrictEqual(
+      written.map((event) => event.index),
+      Array.from({ length: 50 }, (_unused, index) => index)
+    );
+    assert.deepStrictEqual(
+      written.map((event) => event.sequence),
+      Array.from({ length: 50 }, (_unused, index) => index + 1)
+    );
+  });
+
+  it('should stop appending once the bundle limit is reached', async () => {
+    const bundle = await open({ limits: { maxBundleBytes: 220 } });
+
+    let accepted = 0;
+    for (let index = 0; index < 20; index++) {
+      if (await bundle.appendEvent({ kind: TRACE_EVENT.CONSOLE, index })) {
+        accepted += 1;
+      }
+    }
+
+    assert.ok(accepted > 0, 'the first events still fit');
+    assert.ok(accepted < 20, 'the limit stopped the rest');
+    assert.ok(bundle.bytesWritten <= 220);
+  });
+});