browser-commander 0.20.0 → 0.21.1

package/tests/unit/parity/harness.test.js

@@ -0,0 +1,138 @@
+/**
+ * The reference capture starts a browser nothing automates and waits for the
+ * probe page to POST its report. A browser that cannot start at all (no usable
+ * sandbox, a rejected switch) never reports, so the capture has to fail with
+ * the browser's own exit code and stderr rather than a bare 60 s timeout.
+ */
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+
+import { captureReferenceReport } from '../../../src/parity/harness.js';
+
+function fakeProcess({ exitCode, stderr = '' }) {
+  const listeners = new Set();
+  let resolveExit;
+  const child = {
+    exited: new Promise((resolve) => {
+      resolveExit = resolve;
+    }),
+    stderr: {
+      on(event, listener) {
+        listeners.add(listener);
+      },
+    },
+    kill() {
+      resolveExit(143);
+    },
+  };
+  setTimeout(() => {
+    for (const listener of listeners) {
+      listener(stderr);
+    }
+    if (exitCode !== null) {
+      resolveExit(exitCode);
+    }
+  }, 0);
+  return child;
+}
+
+function fakeServer({ report = null } = {}) {
+  return {
+    url: (token) => `http://127.0.0.1:1/probe/${token}`,
+    waitForReport: (token, timeoutMs) =>
+      report
+        ? new Promise((resolve) => setTimeout(() => resolve(report), 20))
+        : new Promise((resolve, reject) =>
+            setTimeout(
+              () => reject(new Error(`timed out waiting for report ${token}`)),
+              timeoutMs
+            )
+          ),
+  };
+}
+
+describe('captureReferenceReport', () => {
+  it('fails fast with the exit code and stderr of a browser that died', async () => {
+    const started = Date.now();
+    await assert.rejects(
+      captureReferenceReport({
+        executablePath: '/usr/bin/chromium',
+        server: fakeServer(),
+        token: 't1',
+        timeoutMs: 3000,
+        start: () =>
+          fakeProcess({
+            exitCode: 1,
+            stderr: 'FATAL: No usable sandbox!',
+          }),
+      }),
+      (error) => {
+        assert.match(error.message, /\/usr\/bin\/chromium exited with code 1/);
+        assert.match(error.message, /No usable sandbox/);
+        return true;
+      }
+    );
+    assert.ok(Date.now() - started < 5000);
+  });
+
+  it('includes stderr when the report times out', async () => {
+    await assert.rejects(
+      captureReferenceReport({
+        server: fakeServer(),
+        token: 't2',
+        timeoutMs: 50,
+        start: () => fakeProcess({ exitCode: null, stderr: 'GPU init failed' }),
+      }),
+      /timed out waiting for report t2\nbrowser stderr:\nGPU init failed/
+    );
+  });
+
+  it('retries a fresh profile after the browser network service crashes', async () => {
+    const tokens = [];
+    let starts = 0;
+    const report = await captureReferenceReport({
+      server: {
+        url: (token) => {
+          tokens.push(token);
+          return `http://127.0.0.1:1/probe/${token}`;
+        },
+        waitForReport: (token) =>
+          token === 't4'
+            ? new Promise((resolve, reject) =>
+                setTimeout(
+                  () =>
+                    reject(new Error(`timed out waiting for report ${token}`)),
+                  20
+                )
+              )
+            : Promise.resolve({ navigator: { webdriver: false } }),
+      },
+      token: 't4',
+      timeoutMs: 50,
+      start: () => {
+        starts += 1;
+        return fakeProcess({
+          exitCode: null,
+          stderr:
+            starts === 1
+              ? 'Network service crashed or was terminated, restarting service.'
+              : '',
+        });
+      },
+    });
+    assert.deepEqual(report, { navigator: { webdriver: false } });
+    assert.equal(starts, 2);
+    assert.equal(tokens.length, 2);
+    assert.notEqual(tokens[0], tokens[1]);
+  });
+
+  it('keeps waiting when a launcher script exits cleanly', async () => {
+    const report = await captureReferenceReport({
+      server: fakeServer({ report: { navigator: { webdriver: false } } }),
+      token: 't3',
+      start: () => fakeProcess({ exitCode: 0 }),
+    });
+    assert.deepEqual(report, { navigator: { webdriver: false } });
+    assert.ok(Array.isArray(report.commandLine));
+  });
+});

package/tests/unit/playwright-protocol-coverage.test.js

@@ -0,0 +1,231 @@
+/**
+ * Coverage of the typed Rust Playwright bindings against `protocol.yml`
+ * (issue #108).
+ *
+ * The Rust `playwright` engine talks to the official driver through
+ * `rust/src/playwright/protocol/`, generated from the vendored protocol spec in
+ * `rust/protocol/playwright/`. These tests read the spec on their own and fail
+ * when:
+ *
+ * - the committed bindings differ from what the generator writes;
+ * - any interface, command (own or inherited) or event in the spec has no typed
+ *   Rust counterpart;
+ * - the vendored spec disagrees with the driver that is installed, which is
+ *   the one the Rust tests and CI start.
+ */
+
+import assert from 'node:assert/strict';
+import * as fs from 'node:fs';
+import { createRequire } from 'node:module';
+import path from 'node:path';
+import { describe, it } from 'node:test';
+import { pathToFileURL } from 'node:url';
+
+import { repoPath } from '../helpers/repo.js';
+
+const SPEC_DIR = repoPath('rust/protocol/playwright');
+const OUT_DIR = repoPath('rust/src/playwright/protocol');
+
+const { generateProtocol } = await import(
+  pathToFileURL(repoPath('scripts/generate-playwright-protocol.mjs'))
+);
+
+const require = createRequire(repoPath('js/package.json'));
+const YAML = require('yaml');
+
+function loadDefinitions() {
+  const definitions = {};
+  for (const file of fs
+    .readdirSync(SPEC_DIR)
+    .filter((f) => f.endsWith('.yml'))) {
+    const parsed = YAML.parse(
+      fs.readFileSync(path.join(SPEC_DIR, file), 'utf8')
+    );
+    Object.assign(definitions, parsed ?? {});
+  }
+  return definitions;
+}
+
+const definitions = loadDefinitions();
+const version = fs.readFileSync(path.join(SPEC_DIR, 'VERSION'), 'utf8').trim();
+const interfaces = Object.keys(definitions).filter(
+  (name) => definitions[name].type === 'interface'
+);
+
+/** `goto` -> `Goto`, `setViewportSize` -> `SetViewportSize`. */
+function upperFirst(name) {
+  return name[0].toUpperCase() + name.slice(1);
+}
+
+/** Own and inherited members, with the interface that declares each one. */
+function members(name, kind) {
+  const definition = definitions[name];
+  const inherited = definition.extends ? members(definition.extends, kind) : [];
+  const own = Object.entries(definition[kind] ?? {}).map(([member, spec]) => ({
+    owner: name,
+    name: member,
+    spec: spec ?? {},
+  }));
+  const ownNames = new Set(own.map((member) => member.name));
+  return [...inherited.filter((m) => !ownNames.has(m.name)), ...own];
+}
+
+function hasEntries(value) {
+  return Boolean(value) && Object.keys(value).length > 0;
+}
+
+/**
+ * Every top-level `impl … {` block in the generated sources keyed by its
+ * header (`impl Page`, `impl ChannelType for Page`), and every struct or enum
+ * body keyed by `struct Name` or `enum Name`.
+ */
+function rustBlocks() {
+  const blocks = new Map();
+  for (const file of fs.readdirSync(OUT_DIR).filter((f) => f.endsWith('.rs'))) {
+    const source = fs.readFileSync(path.join(OUT_DIR, file), 'utf8');
+    for (const match of source.matchAll(
+      /^(impl [^\n{]+?|pub (?:struct|enum) \w+) \{(?:\}|\n([\s\S]*?)^\})/gmu
+    )) {
+      const key = match[1].replace(/^pub /u, '');
+      blocks.set(key, `${blocks.get(key) ?? ''}${match[2] ?? ''}`);
+    }
+  }
+  return blocks;
+}
+
+function escape(text) {
+  return text.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
+}
+
+describe('Rust Playwright protocol bindings', () => {
+  it('are what the generator writes from the vendored spec', () => {
+    const files = generateProtocol();
+    const committed = fs
+      .readdirSync(OUT_DIR)
+      .filter((file) => file.endsWith('.rs'))
+      .sort();
+    assert.deepEqual(committed, [...files.keys()].sort());
+    for (const [file, contents] of files) {
+      assert.equal(
+        fs.readFileSync(path.join(OUT_DIR, file), 'utf8'),
+        contents,
+        `${file} is stale; run node scripts/generate-playwright-protocol.mjs`
+      );
+    }
+  });
+
+  it('pin the spec version they were generated from', () => {
+    const module = fs.readFileSync(path.join(OUT_DIR, 'mod.rs'), 'utf8');
+    assert.match(
+      module,
+      new RegExp(
+        `pub const PROTOCOL_VERSION: &str = "${escape(version)}";`,
+        'u'
+      )
+    );
+  });
+
+  it('cover every interface, command and event in protocol.yml', () => {
+    const blocks = rustBlocks();
+    const missing = [];
+    let commands = 0;
+    let events = 0;
+    assert.ok(interfaces.length >= 30, `only ${interfaces.length} interfaces`);
+    for (const name of interfaces) {
+      const channel = blocks.get(`impl ChannelType for ${name}`) ?? '';
+      if (!channel.includes(`const INTERFACE: &'static str = "${name}";`)) {
+        missing.push(`${name}: channel type`);
+      }
+      if (!blocks.has(`struct ${name}Initializer`)) {
+        missing.push(`${name}: initializer`);
+      }
+
+      const methods = blocks.get(`impl ${name}`) ?? '';
+      for (const command of members(name, 'commands')) {
+        commands += 1;
+        const base = `${command.owner}${upperFirst(command.name)}`;
+        const params = hasEntries(command.spec.parameters)
+          ? `, params: ${base}Params\\)`
+          : '\\)';
+        const result = hasEntries(command.spec.returns)
+          ? `${base}Result`
+          : '\\(\\)';
+        const send = hasEntries(command.spec.returns)
+          ? 'send'
+          : 'send_no_result';
+        const method = new RegExp(
+          `/// \`${escape(command.owner)}\\.${escape(command.name)}\`[^\\n]*\\n` +
+            `\\s*pub async fn (?:r#)?\\w+\\(&self${params} -> Result<${result}, ProtocolError> \\{\\n` +
+            `\\s*self\\.channel\\.${send}\\("${escape(command.name)}", `,
+          'u'
+        );
+        if (!method.test(methods)) {
+          missing.push(`${name}.${command.name}`);
+        }
+      }
+
+      const variants = blocks.get(`enum ${name}Event`) ?? '';
+      const parse = blocks.get(`impl ProtocolEvent for ${name}Event`) ?? '';
+      for (const event of members(name, 'events')) {
+        events += 1;
+        const variant = upperFirst(event.name);
+        const declared = hasEntries(event.spec.parameters)
+          ? `${variant}(${event.owner}${variant}EventParams),`
+          : `${variant},`;
+        if (
+          !variants.includes(
+            `/// \`${event.owner}.${event.name}\`\n    ${declared}`
+          ) ||
+          !parse.includes(`"${event.name}" => Ok(Self::${variant}`)
+        ) {
+          missing.push(`${name} event ${event.name}`);
+        }
+      }
+    }
+    assert.deepEqual(missing, []);
+    assert.ok(commands > 300, `only ${commands} commands`);
+    assert.ok(events > 50, `only ${events} events`);
+  });
+
+  it('match the protocol of the installed driver', (t) => {
+    let root;
+    try {
+      root = path.dirname(require.resolve('playwright-core/package.json'));
+    } catch {
+      t.skip('playwright-core is not installed');
+      return;
+    }
+    const installed = JSON.parse(
+      fs.readFileSync(path.join(root, 'package.json'), 'utf8')
+    ).version;
+    if (installed !== version) {
+      t.skip(
+        `playwright-core ${installed} is installed, the spec is ${version}`
+      );
+      return;
+    }
+    // The driver validates every message against these schemes; one exists per
+    // command (`<Interface><Command>Params`) and event (`<Interface><Event>Event`).
+    const bundle = fs.readFileSync(
+      path.join(root, 'lib/coreBundle.js'),
+      'utf8'
+    );
+    const driver = new Set(
+      [...bundle.matchAll(/scheme\.(\w+(?:Params|Event)) = /gu)]
+        .map((match) => match[1])
+        .filter((scheme) => !definitions[scheme])
+    );
+    const spec = new Set();
+    for (const name of interfaces) {
+      for (const command of members(name, 'commands')) {
+        spec.add(`${name}${upperFirst(command.name)}Params`);
+      }
+      for (const event of members(name, 'events')) {
+        spec.add(`${name}${upperFirst(event.name)}Event`);
+      }
+    }
+    assert.ok(driver.size > 300, `found ${driver.size} driver schemes`);
+    assert.deepEqual([...spec].filter((s) => !driver.has(s)).sort(), []);
+    assert.deepEqual([...driver].filter((s) => !spec.has(s)).sort(), []);
+  });
+});

package/tests/unit/puppeteer-bindings-coverage.test.js

@@ -0,0 +1,214 @@
+/**
+ * Coverage of the typed Rust and Python Puppeteer wrappers (issue #108).
+ *
+ * `rust/src/puppeteer/api/` and `python/src/browser_commander/puppeteer/api/`
+ * are generated by `scripts/generate-puppeteer-bindings.mjs` from
+ * `rust/protocol/puppeteer/api.json`, which `scripts/puppeteer-api.mjs`
+ * extracts from puppeteer-core's `lib/types.d.ts`. These tests fail when:
+ *
+ * - the committed wrappers differ from what the generator writes;
+ * - the committed manifest differs from the installed puppeteer-core;
+ * - a public method or getter declared in `lib/types.d.ts` (own or inherited)
+ *   has no typed entry point in Rust or Python.
+ */
+
+import assert from 'node:assert/strict';
+import * as fs from 'node:fs';
+import path from 'node:path';
+import { describe, it } from 'node:test';
+import { pathToFileURL } from 'node:url';
+
+import { puppeteerDeclarations } from '../helpers/declared-api.js';
+import { repoPath } from '../helpers/repo.js';
+
+const generator = await import(
+  pathToFileURL(repoPath('scripts/generate-puppeteer-bindings.mjs'))
+);
+const extractor = await import(
+  pathToFileURL(repoPath('scripts/puppeteer-api.mjs'))
+);
+
+const {
+  MANIFEST_PATH,
+  RUST_DIR,
+  PYTHON_DIR,
+  formatManifest,
+  generatePuppeteerBindings,
+  loadManifest,
+  pythonMethodName,
+  rustMethodName,
+} = generator;
+
+const manifest = loadManifest();
+const types = new Map(manifest.types.map((type) => [type.name, type]));
+
+function readDir(dir, extension) {
+  return new Map(
+    fs
+      .readdirSync(dir)
+      .filter((file) => file.endsWith(extension))
+      .map((file) => [file, fs.readFileSync(path.join(dir, file), 'utf8')])
+  );
+}
+
+/** The body of every `impl Name {` block, keyed by type name. */
+function rustImpls(files) {
+  const impls = new Map();
+  for (const source of files.values()) {
+    for (const match of source.matchAll(/^impl (\w+) \{\n([\s\S]*?)^\}/gmu)) {
+      impls.set(match[1], (impls.get(match[1]) ?? '') + match[2]);
+    }
+  }
+  return impls;
+}
+
+/** The body of every top-level `class Name(…):`, keyed by class name. */
+function pythonClasses(files) {
+  const classes = new Map();
+  for (const source of files.values()) {
+    for (const match of source.matchAll(
+      /^class (\w+)\([^)]*\):\n([\s\S]*?)(?=^\S|(?![\s\S]))/gmu
+    )) {
+      classes.set(match[1], match[2]);
+    }
+  }
+  return classes;
+}
+
+const rust = rustImpls(readDir(RUST_DIR, '.rs'));
+const python = pythonClasses(readDir(PYTHON_DIR, '.py'));
+
+/** Python inherits: a method is reachable when it or an ancestor defines it. */
+function pythonHas(typeName, method) {
+  const pattern = new RegExp(
+    `^ {4}async def ${pythonMethodName(method)}\\(`,
+    'mu'
+  );
+  for (let name = typeName; name; name = types.get(name)?.extends ?? null) {
+    if (pattern.test(python.get(name) ?? '')) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/** Rust has no inheritance: the method must be on the type's own `impl`. */
+function rustHas(typeName, owner, method) {
+  const doc = `/// \`${owner}.${method}\``;
+  const name = rustMethodName(method);
+  const body = rust.get(typeName) ?? '';
+  return body.includes(doc) && body.includes(`pub async fn ${name}(`);
+}
+
+describe('Puppeteer bindings (Rust and Python)', () => {
+  it('are what the generator writes from the committed manifest', () => {
+    const generated = generatePuppeteerBindings(manifest);
+    for (const [dir, files, extension] of [
+      [RUST_DIR, generated.rust, '.rs'],
+      [PYTHON_DIR, generated.python, '.py'],
+    ]) {
+      const committed = readDir(dir, extension);
+      assert.deepEqual(
+        [...committed.keys()].sort(),
+        [...files.keys()].sort(),
+        `${dir} has stale or missing files; run node scripts/generate-puppeteer-bindings.mjs`
+      );
+      for (const [file, text] of files) {
+        assert.equal(
+          committed.get(file),
+          text,
+          `${path.join(dir, file)} is out of date; run node scripts/generate-puppeteer-bindings.mjs`
+        );
+      }
+    }
+  });
+
+  it('come from the installed puppeteer-core', (t) => {
+    let installed;
+    try {
+      installed = extractor.installedPuppeteer();
+    } catch {
+      t.skip('puppeteer-core is not installed');
+      return;
+    }
+    if (installed.version !== manifest.version) {
+      t.skip(
+        `puppeteer-core ${installed.version} is installed, the manifest is ${manifest.version}`
+      );
+      return;
+    }
+    const fresh = {
+      package: 'puppeteer-core',
+      version: installed.version,
+      ...extractor.extractPuppeteerApi(installed.source),
+    };
+    assert.equal(
+      fs.readFileSync(MANIFEST_PATH, 'utf8'),
+      formatManifest(fresh),
+      'run node scripts/generate-puppeteer-bindings.mjs --update-api'
+    );
+  });
+
+  it('give every declared Puppeteer method a typed entry point', () => {
+    const missing = [];
+    let checked = 0;
+    for (const [typeName, methods] of puppeteerDeclarations()) {
+      for (const method of methods) {
+        checked += 1;
+        const member = types
+          .get(typeName)
+          ?.methods.find((candidate) => candidate.name === method);
+        if (!member) {
+          missing.push(`${typeName}.${method}: not in the manifest`);
+          continue;
+        }
+        if (!rustHas(typeName, member.owner, method)) {
+          missing.push(`${typeName}.${method}: no Rust method`);
+        }
+        if (!pythonHas(typeName, method)) {
+          missing.push(`${typeName}.${method}: no Python method`);
+        }
+      }
+    }
+    assert.deepEqual(missing, []);
+    assert.ok(checked > 250, `only ${checked} declared methods were checked`);
+  });
+
+  it('give every manifest member, own or inherited, a typed entry point', () => {
+    const missing = [];
+    let members = 0;
+    for (const type of manifest.types) {
+      for (const member of type.methods) {
+        members += 1;
+        if (!rustHas(type.name, member.owner, member.name)) {
+          missing.push(`Rust ${type.name}.${member.name}`);
+        }
+        if (!pythonHas(type.name, member.name)) {
+          missing.push(`Python ${type.name}.${member.name}`);
+        }
+      }
+    }
+    assert.deepEqual(missing, []);
+    assert.ok(types.size > 40, `only ${types.size} types`);
+    assert.ok(members > 450, `only ${members} members`);
+  });
+
+  it('keep Python classes in the Puppeteer class hierarchy', () => {
+    for (const type of manifest.types) {
+      const source = fs.readFileSync(
+        path.join(PYTHON_DIR, `${generator.snakeCase(type.name)}.py`),
+        'utf8'
+      );
+      const parent =
+        type.extends &&
+        fs.existsSync(
+          path.join(PYTHON_DIR, `${generator.snakeCase(type.extends)}.py`)
+        );
+      const base = parent ? type.extends : 'RemoteObject';
+      assert.match(
+        source,
+        new RegExp(`^class ${type.name}\\(${base}\\):`, 'mu')
+      );
+    }
+  });
+});

package/tests/unit/scripts/ci-timeout-budgets.test.js

@@ -196,7 +196,14 @@ describe('CI execution budgets', () => {
     assert.deepEqual(budgeted, [
       'docs.yml build-docs Rust API docs',
       'js.yml test Node.js test suite',
+      'parity.yml cli CLI and API coverage suites',
+      'parity.yml cli Cross-language CLI contract',
+      'parity.yml cli Rust CLI build',
       'parity.yml parity Fingerprint parity suite',
+      'parity.yml parity WebDriver suite',
+      'parity.yml snapshots Python native snapshot launches',
+      'parity.yml snapshots Rust native snapshots and parity',
+      'parity.yml storage-state Rust storage state transfer',
       'python.yml test pytest suite',
       'rust.yml coverage Rust code coverage',
       'rust.yml test Rust doc tests',
@@ -241,6 +248,26 @@ describe('CI execution budgets', () => {
 });
 
 describe('pipeline status gate', () => {
+  it('resolves WebDriver executables in the job that runs native WebDriver tests', () => {
+    const workflow = readWorkflow('parity.yml');
+    const jobs = listWorkflowJobs(workflow).filter((job) =>
+      getJobBlock(workflow, job).includes('--test webdriver')
+    );
+    assert.ok(
+      jobs.length > 0,
+      'native WebDriver integration tests are missing'
+    );
+    for (const job of jobs) {
+      const block = getJobBlock(workflow, job);
+      const resolver = block.indexOf('resolve-webdriver.mjs');
+      const test = block.indexOf('--test webdriver');
+      assert.ok(
+        resolver >= 0 && resolver < test,
+        `${job}: resolve WebDriver before running its native tests`
+      );
+    }
+  });
+
   it('is present in every workflow and needs every other job', () => {
     for (const { fileName, text } of WORKFLOWS) {
       const jobs = listWorkflowJobs(text);