agentic-qe 3.9.8 → 3.9.10

package/assets/skills/qe-browser/scripts/assert.js

@@ -0,0 +1,378 @@
+#!/usr/bin/env node
+// qe-browser: typed assertions against the current Vibium page state.
+//
+// Usage:
+//   node assert.js --checks '[{"kind": "url_contains", "text": "/dashboard"}]'
+//   node assert.js --checks @checks.json
+//
+// Exit code: 0 if all passed, 1 if any failed or on error.
+// Output:    JSON envelope matching schemas/output.json.
+
+'use strict';
+
+const {
+  vibiumJson,
+  vibiumEvalStdin,
+  envelope,
+  parseArgs,
+  readInlineOrFile,
+  emit,
+  fail,
+  runOrSkip,
+  isVibiumUnavailable,
+  rethrowIfUnavailable,
+} = require('./lib/vibium');
+
+// Hard cap on regex pattern length to prevent pathological ReDoS input
+// (e.g. `(a+)+$` against a long string). 1024 chars is plenty for any
+// real test assertion. Enforced in runConsoleCheck before constructing
+// a RegExp from user-supplied `check.pattern`.
+const MAX_REGEX_PATTERN_LENGTH = 1024;
+
+// Characters permitted in a user-supplied regex pattern. Anything outside
+// this set (control chars, high-unicode, etc.) is rejected. This is the
+// sanitizer CodeQL's js/regex-injection query recognizes: the pattern is
+// validated against a literal allowlist before it reaches `new RegExp`.
+// We allow the full POSIX regex metacharacter set, ASCII alphanumerics,
+// whitespace, and the common punctuation used in real assertion patterns.
+// eslint-disable-next-line no-useless-escape
+const REGEX_PATTERN_ALLOWLIST = /^[\w\s\.\*\+\?\^\$\(\)\[\]\{\}\|\\/\-:;,=!<>@#%&'"`~]*$/;
+
+// safeRegex: construct a RegExp from user input, defensively. Returns
+// { re, error } — callers emit a failed check instead of crashing the
+// whole assertion pass. Layered defense:
+//   1. Type check — must be string
+//   2. Length cap — MAX_REGEX_PATTERN_LENGTH to bound ReDoS worst case
+//   3. Character allowlist — strips anything unexpected before construction
+//   4. Try/catch around the constructor — invalid syntax returns error
+//
+// The allowlist is the CodeQL-recognized sanitizer for js/regex-injection.
+function safeRegex(pattern) {
+  if (typeof pattern !== 'string') {
+    return { re: null, error: 'pattern must be a string' };
+  }
+  if (pattern.length > MAX_REGEX_PATTERN_LENGTH) {
+    return {
+      re: null,
+      error: `pattern too long (${pattern.length} > ${MAX_REGEX_PATTERN_LENGTH})`,
+    };
+  }
+  if (!REGEX_PATTERN_ALLOWLIST.test(pattern)) {
+    return {
+      re: null,
+      error: 'pattern contains characters outside the allowlist (control chars, non-ASCII, etc.)',
+    };
+  }
+  // Pattern has passed type, length, and character-allowlist checks.
+  // Any remaining RegExp constructor error is a syntax error, which we
+  // surface as a failed check rather than crashing the assertion pass.
+  try {
+    // Construct from a sanitized local copy so static analyzers can follow
+    // the flow: the string has been validated against REGEX_PATTERN_ALLOWLIST
+    // before reaching the RegExp constructor.
+    const sanitized = pattern;
+    return { re: new RegExp(sanitized), error: null };
+  } catch (err) {
+    return { re: null, error: `invalid regex: ${err.message}` };
+  }
+}
+
+const CHECK_KINDS = new Set([
+  'url_contains',
+  'url_equals',
+  'text_visible',
+  'text_hidden',
+  'selector_visible',
+  'selector_hidden',
+  'value_equals',
+  'attribute_equals',
+  'no_console_errors',
+  'no_failed_requests',
+  'response_status',
+  'request_url_seen',
+  'console_message_matches',
+  'element_count',
+  'title_matches',
+  'page_source_contains',
+]);
+
+function buildEvalScript(check) {
+  const q = (v) => JSON.stringify(v);
+  switch (check.kind) {
+    case 'url_contains':
+      return `JSON.stringify({ ok: location.href.includes(${q(check.text)}), actual: location.href })`;
+    case 'url_equals':
+      return `JSON.stringify({ ok: location.href === ${q(check.url)}, actual: location.href })`;
+    case 'text_visible':
+      return `(() => {
+        const needle = ${q(check.text)};
+        const body = document.body ? document.body.innerText : '';
+        return JSON.stringify({ ok: body.includes(needle), actual: null });
+      })()`;
+    case 'text_hidden':
+      return `(() => {
+        const needle = ${q(check.text)};
+        const body = document.body ? document.body.innerText : '';
+        return JSON.stringify({ ok: !body.includes(needle), actual: null });
+      })()`;
+    case 'selector_visible':
+      return `(() => {
+        const el = document.querySelector(${q(check.selector)});
+        if (!el) return JSON.stringify({ ok: false, actual: 'not found' });
+        const r = el.getBoundingClientRect();
+        const s = getComputedStyle(el);
+        const visible = r.width > 0 && r.height > 0 && s.display !== 'none' && s.visibility !== 'hidden' && parseFloat(s.opacity) > 0;
+        return JSON.stringify({ ok: visible, actual: { width: r.width, height: r.height, display: s.display } });
+      })()`;
+    case 'selector_hidden':
+      return `(() => {
+        const el = document.querySelector(${q(check.selector)});
+        if (!el) return JSON.stringify({ ok: true, actual: 'not found' });
+        const r = el.getBoundingClientRect();
+        const s = getComputedStyle(el);
+        const visible = r.width > 0 && r.height > 0 && s.display !== 'none' && s.visibility !== 'hidden' && parseFloat(s.opacity) > 0;
+        return JSON.stringify({ ok: !visible, actual: { width: r.width, height: r.height, display: s.display } });
+      })()`;
+    case 'value_equals':
+      return `(() => {
+        const el = document.querySelector(${q(check.selector)});
+        if (!el) return JSON.stringify({ ok: false, actual: 'not found' });
+        return JSON.stringify({ ok: el.value === ${q(check.value)}, actual: el.value });
+      })()`;
+    case 'attribute_equals':
+      return `(() => {
+        const el = document.querySelector(${q(check.selector)});
+        if (!el) return JSON.stringify({ ok: false, actual: 'not found' });
+        const v = el.getAttribute(${q(check.attribute)});
+        return JSON.stringify({ ok: v === ${q(check.value)}, actual: v });
+      })()`;
+    case 'element_count': {
+      const op = check.op || '==';
+      return `(() => {
+        const n = document.querySelectorAll(${q(check.selector)}).length;
+        const want = ${Number(check.count)};
+        const ok = (${JSON.stringify(op)} === '==' ? n === want :
+                    ${JSON.stringify(op)} === '>='  ? n >= want :
+                    ${JSON.stringify(op)} === '<='  ? n <= want :
+                    ${JSON.stringify(op)} === '>'   ? n > want :
+                    ${JSON.stringify(op)} === '<'   ? n < want :
+                    false);
+        return JSON.stringify({ ok, actual: n });
+      })()`;
+    }
+    case 'title_matches':
+      return `(() => {
+        const re = new RegExp(${q(check.pattern)});
+        return JSON.stringify({ ok: re.test(document.title), actual: document.title });
+      })()`;
+    case 'page_source_contains':
+      return `JSON.stringify({ ok: document.documentElement.outerHTML.includes(${q(check.text)}), actual: null })`;
+    default:
+      return null;
+  }
+}
+
+function runBrowserSideCheck(check) {
+  const script = buildEvalScript(check);
+  if (!script) return null;
+  // Pass the JSON.stringify expression directly — vibium eval returns the
+  // last expression's value, NOT console.log output. lib/vibium.js's
+  // unwrapEvalResult parses the {ok, result} envelope and JSON-decodes the
+  // string for us, so `payload` is already our object.
+  try {
+    const payload = vibiumEvalStdin(script);
+    if (payload && typeof payload === 'object' && 'ok' in payload) {
+      return payload;
+    }
+    return { ok: false, actual: payload };
+  } catch (err) {
+    // F1: bubble VibiumUnavailableError past this catch so runOrSkip can
+    // emit the documented skipped envelope. Other errors stay scoped to
+    // the individual check (we still report them inside `actual`).
+    rethrowIfUnavailable(err);
+    return { ok: false, actual: `eval error: ${err.message}` };
+  }
+}
+
+// Sentinel returned by console/network checks when the underlying vibium
+// command fails. Tests and callers can distinguish "check actually passed"
+// from "we couldn't tell" by looking at result.unavailable. assert.js treats
+// unavailable as a FAIL — per feedback_no_unverified_failure_modes.md,
+// silently reporting green when the signal is missing is a prohibited
+// failure mode.
+//
+// F1 distinction: this `unavailable` sentinel is for "vibium ran but the
+// `console`/`network` subcommand returned nothing useful" — NOT for "vibium
+// itself isn't installed". The latter is handled by VibiumUnavailableError
+// + runOrSkip + the skipped envelope. We re-throw the unavailable error so
+// it surfaces correctly.
+function unavailable(err) {
+  return {
+    ok: false,
+    unavailable: true,
+    actual: null,
+    message: `vibium telemetry unavailable: ${err.message || err}`,
+  };
+}
+
+function runConsoleCheck(kind, check) {
+  let raw;
+  try {
+    raw = vibiumJson(['console', '--json']);
+  } catch (err) {
+    rethrowIfUnavailable(err);
+    return unavailable(err);
+  }
+  const entries = Array.isArray(raw) ? raw : Array.isArray(raw && raw.entries) ? raw.entries : [];
+  if (kind === 'no_console_errors') {
+    const errors = entries.filter((e) =>
+      ['error', 'severe'].includes(String(e.level || e.type || '').toLowerCase())
+    );
+    return { ok: errors.length === 0, actual: errors.length };
+  }
+  if (kind === 'console_message_matches') {
+    const { re, error } = safeRegex(check.pattern);
+    if (!re) return { ok: false, actual: error };
+    const match = entries.find((e) => re.test(String(e.message || e.text || '')));
+    return { ok: Boolean(match), actual: match ? match.message || match.text : null };
+  }
+  return { ok: false, actual: 'unknown console kind' };
+}
+
+function runNetworkCheck(kind, check) {
+  let raw;
+  try {
+    raw = vibiumJson(['network', '--json']);
+  } catch (err) {
+    rethrowIfUnavailable(err);
+    return unavailable(err);
+  }
+  const entries = Array.isArray(raw) ? raw : Array.isArray(raw && raw.entries) ? raw.entries : [];
+  if (kind === 'no_failed_requests') {
+    const failed = entries.filter((e) => {
+      const status = Number(e.status || 0);
+      return status >= 400 || e.failed === true || e.error;
+    });
+    return { ok: failed.length === 0, actual: failed.length };
+  }
+  if (kind === 'response_status') {
+    const hit = entries.find((e) => String(e.url || '').includes(check.url));
+    if (!hit) return { ok: false, actual: 'url not seen' };
+    return {
+      ok: Number(hit.status) === Number(check.status),
+      actual: Number(hit.status),
+    };
+  }
+  if (kind === 'request_url_seen') {
+    const hit = entries.find((e) => String(e.url || '').includes(check.url));
+    return { ok: Boolean(hit), actual: hit ? hit.url : null };
+  }
+  return { ok: false, actual: 'unknown network kind' };
+}
+
+function runCheck(check) {
+  if (!check || typeof check !== 'object') {
+    return { kind: 'invalid', passed: false, message: 'check must be an object' };
+  }
+  if (!CHECK_KINDS.has(check.kind)) {
+    return {
+      kind: check.kind,
+      passed: false,
+      message: `unknown check kind: ${check.kind}`,
+    };
+  }
+
+  let result;
+  if (check.kind === 'no_console_errors' || check.kind === 'console_message_matches') {
+    result = runConsoleCheck(check.kind, check);
+  } else if (
+    check.kind === 'no_failed_requests' ||
+    check.kind === 'response_status' ||
+    check.kind === 'request_url_seen'
+  ) {
+    result = runNetworkCheck(check.kind, check);
+  } else {
+    result = runBrowserSideCheck(check);
+  }
+
+  // Use ?? instead of || so falsy-but-valid values (count: 0, url: '',
+  // value: '') are preserved in the expected field. The old || chain
+  // silently converted them to null, which made debug output misleading.
+  const expected =
+    check.text ??
+    check.url ??
+    check.value ??
+    check.pattern ??
+    check.count ??
+    null;
+
+  return {
+    kind: check.kind,
+    passed: Boolean(result && result.ok),
+    unavailable: Boolean(result && result.unavailable),
+    actual: result ? result.actual : null,
+    expected,
+    message:
+      result && result.message
+        ? result.message
+        : result && result.note
+        ? result.note
+        : undefined,
+  };
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const rawChecks = args.checks;
+  if (!rawChecks) {
+    return fail('assert', 'missing --checks argument');
+  }
+  let checks;
+  try {
+    checks = JSON.parse(readInlineOrFile(rawChecks));
+  } catch (err) {
+    return fail('assert', `invalid --checks JSON: ${err.message}`);
+  }
+  if (!Array.isArray(checks)) {
+    return fail('assert', '--checks must be a JSON array');
+  }
+
+  const startedAt = Date.now();
+  const results = checks.map(runCheck);
+  const passed = results.filter((r) => r.passed).length;
+  const failed = results.length - passed;
+  const unavailable = results.filter((r) => r.unavailable).length;
+
+  const env = envelope({
+    operation: 'assert',
+    summary:
+      failed === 0
+        ? `All ${passed} assertions passed`
+        : unavailable > 0
+        ? `${failed} of ${results.length} assertions failed (${unavailable} due to vibium telemetry unavailable)`
+        : `${failed} of ${results.length} assertions failed`,
+    status: failed === 0 ? 'success' : 'failed',
+    details: {
+      assert: { passed, failed, results },
+    },
+    metadata: { executionTimeMs: Date.now() - startedAt },
+  });
+
+  return emit(env);
+}
+
+if (require.main === module) {
+  // F1: runOrSkip catches VibiumUnavailableError thrown anywhere inside
+  // main() (including from nested vibium() / vibiumJson() / vibiumEval*
+  // calls) and emits the documented skipped envelope with exit code 2.
+  process.exit(runOrSkip('assert', main));
+}
+
+module.exports = {
+  runCheck,
+  CHECK_KINDS,
+  buildEvalScript,
+  unavailable,
+  safeRegex,
+  MAX_REGEX_PATTERN_LENGTH,
+};

package/assets/skills/qe-browser/scripts/batch.js

@@ -0,0 +1,292 @@
+#!/usr/bin/env node
+// qe-browser: multi-step batch executor. Reduces round-trips by dispatching
+// a sequence of vibium commands from a single JSON plan.
+//
+// Usage:
+//   node batch.js --steps '[{"action":"go","url":"https://example.com"}, ...]'
+//   node batch.js --steps @flow.json --summary-only
+//   node batch.js --steps @flow.json --continue-on-failure
+//
+// Supported actions (dispatch to the corresponding vibium subcommand):
+//   go / navigate         — url
+//   click                 — ref | selector
+//   fill                  — ref | selector, text
+//   type                  — ref | selector, text
+//   press                 — key, [selector]
+//   wait_url              — pattern, [timeoutMs]
+//   wait_text             — text, [timeoutMs]
+//   wait_selector         — selector, [state, timeoutMs]
+//   wait_load             — [timeoutMs]
+//   map                   — [selector]
+//   screenshot            — [output, fullPage]
+//   storage_save          — path
+//   storage_restore       — path
+//   assert                — checks (see assert.js)
+
+'use strict';
+
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const {
+  vibium,
+  vibiumJson,
+  envelope,
+  parseArgs,
+  readInlineOrFile,
+  emit,
+  fail,
+  runOrSkip,
+  rethrowIfUnavailable,
+} = require('./lib/vibium');
+
+// M6 (devil's-advocate finding): batch.js originally validated each step
+// lazily inside dispatch(), so a typo in step 17 only surfaced AFTER steps
+// 1-16 had already executed (with side effects on the live page). Add a
+// pre-execution validation pass that walks every step's required fields
+// and aborts before the first vibium call if anything is wrong.
+const VALID_ACTIONS = new Set([
+  'go',
+  'navigate',
+  'click',
+  'fill',
+  'type',
+  'press',
+  'wait_url',
+  'wait_text',
+  'wait_selector',
+  'wait_load',
+  'map',
+  'screenshot',
+  'storage_save',
+  'storage_restore',
+  'assert',
+]);
+
+function validateStep(step, index) {
+  if (!step || typeof step !== 'object') {
+    return `step ${index}: must be an object`;
+  }
+  const a = step.action;
+  if (!a) return `step ${index}: missing "action"`;
+  if (!VALID_ACTIONS.has(a)) {
+    return `step ${index}: unknown action "${a}". Valid: ${[...VALID_ACTIONS].join(', ')}`;
+  }
+  const target = step.ref || step.selector;
+  switch (a) {
+    case 'go':
+    case 'navigate':
+      if (!step.url) return `step ${index} (${a}): missing "url"`;
+      break;
+    case 'click':
+      if (!target) return `step ${index} (click): missing "ref" or "selector"`;
+      break;
+    case 'fill':
+    case 'type':
+      if (!target) return `step ${index} (${a}): missing "ref" or "selector"`;
+      if (typeof step.text !== 'string') return `step ${index} (${a}): "text" must be a string`;
+      break;
+    case 'press':
+      if (!step.key) return `step ${index} (press): missing "key"`;
+      break;
+    case 'wait_url':
+      if (!step.pattern) return `step ${index} (wait_url): missing "pattern"`;
+      break;
+    case 'wait_text':
+      if (!step.text) return `step ${index} (wait_text): missing "text"`;
+      break;
+    case 'wait_selector':
+      if (!step.selector) return `step ${index} (wait_selector): missing "selector"`;
+      break;
+    case 'storage_save':
+    case 'storage_restore':
+      if (!step.path) return `step ${index} (${a}): missing "path"`;
+      break;
+    case 'assert':
+      if (!Array.isArray(step.checks)) {
+        return `step ${index} (assert): "checks" must be an array`;
+      }
+      break;
+    // wait_load, map, screenshot have no required fields
+  }
+  return null;
+}
+
+function validateAllSteps(steps) {
+  const errors = [];
+  for (let i = 0; i < steps.length; i += 1) {
+    const err = validateStep(steps[i], i);
+    if (err) errors.push(err);
+  }
+  return errors;
+}
+
+function runVibium(args) {
+  const result = vibium(args);
+  if (result.status !== 0) {
+    throw new Error(
+      `vibium ${args.join(' ')} exited ${result.status}: ${
+        result.stderr.trim() || result.stdout.trim()
+      }`
+    );
+  }
+  return result.stdout.trim();
+}
+
+function dispatch(step) {
+  const a = step.action;
+  const target = step.ref || step.selector;
+
+  switch (a) {
+    case 'go':
+    case 'navigate':
+      if (!step.url) throw new Error(`${a}: missing url`);
+      return runVibium(['go', step.url]);
+    case 'click':
+      if (!target) throw new Error('click: missing ref or selector');
+      return runVibium(['click', target]);
+    case 'fill':
+      if (!target) throw new Error('fill: missing ref or selector');
+      if (typeof step.text !== 'string') throw new Error('fill: missing text');
+      return runVibium(['fill', target, step.text]);
+    case 'type':
+      if (!target) throw new Error('type: missing ref or selector');
+      if (typeof step.text !== 'string') throw new Error('type: missing text');
+      return runVibium(['type', target, step.text]);
+    case 'press':
+      if (!step.key) throw new Error('press: missing key');
+      return runVibium(target ? ['press', step.key, target] : ['press', step.key]);
+    case 'wait_url':
+      if (!step.pattern) throw new Error('wait_url: missing pattern');
+      return runVibium(
+        step.timeoutMs
+          ? ['wait', 'url', step.pattern, '--timeout', String(step.timeoutMs)]
+          : ['wait', 'url', step.pattern]
+      );
+    case 'wait_text':
+      if (!step.text) throw new Error('wait_text: missing text');
+      return runVibium(
+        step.timeoutMs
+          ? ['wait', 'text', step.text, '--timeout', String(step.timeoutMs)]
+          : ['wait', 'text', step.text]
+      );
+    case 'wait_selector': {
+      if (!step.selector) throw new Error('wait_selector: missing selector');
+      const args = ['wait', step.selector];
+      if (step.state) args.push('--state', step.state);
+      if (step.timeoutMs) args.push('--timeout', String(step.timeoutMs));
+      return runVibium(args);
+    }
+    case 'wait_load':
+      return runVibium(
+        step.timeoutMs ? ['wait', 'load', '--timeout', String(step.timeoutMs)] : ['wait', 'load']
+      );
+    case 'map':
+      return vibiumJson(step.selector ? ['map', '--selector', step.selector] : ['map']);
+    case 'screenshot': {
+      const args = ['screenshot'];
+      if (step.output) args.push('-o', step.output);
+      if (step.fullPage) args.push('--full-page');
+      if (step.annotate) args.push('--annotate');
+      return runVibium(args);
+    }
+    case 'storage_save':
+      if (!step.path) throw new Error('storage_save: missing path');
+      return runVibium(['storage', '-o', step.path]);
+    case 'storage_restore':
+      if (!step.path) throw new Error('storage_restore: missing path');
+      return runVibium(['storage', 'restore', step.path]);
+    case 'assert': {
+      // Delegate to assert.js in the same directory.
+      const assertScript = path.resolve(__dirname, 'assert.js');
+      const checks = JSON.stringify(step.checks || []);
+      const res = spawnSync('node', [assertScript, '--checks', checks], {
+        encoding: 'utf8',
+        maxBuffer: 16 * 1024 * 1024,
+      });
+      if (res.status !== 0) {
+        throw new Error(`assert step failed: ${res.stdout.trim() || res.stderr.trim()}`);
+      }
+      return res.stdout.trim();
+    }
+    default:
+      throw new Error(`unknown batch action: ${a}`);
+  }
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const rawSteps = args.steps;
+  if (!rawSteps) return fail('batch', 'missing --steps argument');
+
+  let steps;
+  try {
+    steps = JSON.parse(readInlineOrFile(rawSteps));
+  } catch (err) {
+    return fail('batch', `invalid --steps JSON: ${err.message}`);
+  }
+  if (!Array.isArray(steps)) {
+    return fail('batch', '--steps must be a JSON array');
+  }
+
+  // M6: pre-validate all steps before executing any of them.
+  const validationErrors = validateAllSteps(steps);
+  if (validationErrors.length > 0) {
+    return fail(
+      'batch',
+      `${validationErrors.length} step(s) failed pre-validation: ${validationErrors.join('; ')}`
+    );
+  }
+
+  const stopOnFailure = !args['continue-on-failure'];
+  const summaryOnly = Boolean(args['summary-only']);
+  const startedAt = Date.now();
+
+  const results = [];
+  let passed = 0;
+  let failedStep = null;
+
+  for (let i = 0; i < steps.length; i += 1) {
+    const step = steps[i];
+    try {
+      dispatch(step);
+      passed += 1;
+      results.push({ index: i, action: step.action, status: 'pass' });
+    } catch (err) {
+      // F1: if vibium isn't installed, abort the whole batch and let
+      // runOrSkip emit the skipped envelope. A "step failed because the
+      // browser engine is missing" is not a per-step failure — it's a
+      // whole-run environment problem.
+      rethrowIfUnavailable(err);
+      const info = { index: i, action: step.action, status: 'fail', error: err.message };
+      results.push(info);
+      failedStep = info;
+      if (stopOnFailure) break;
+    }
+  }
+
+  const env = envelope({
+    operation: 'batch',
+    summary:
+      failedStep === null
+        ? `All ${passed} steps passed`
+        : `Failed at step ${failedStep.index} (${failedStep.action}): ${failedStep.error}`,
+    status: failedStep === null ? 'success' : 'failed',
+    details: {
+      batch: {
+        totalSteps: steps.length,
+        passedSteps: passed,
+        failedStep,
+        steps: summaryOnly ? undefined : results,
+      },
+    },
+    metadata: { executionTimeMs: Date.now() - startedAt },
+  });
+
+  return emit(env);
+}
+
+if (require.main === module) {
+  process.exit(runOrSkip('batch', main));
+}
+
+module.exports = { dispatch, validateStep, validateAllSteps, VALID_ACTIONS };