@yemi33/minions 0.1.2071 → 0.1.2073

package/engine/qa-runners.js

@@ -0,0 +1,323 @@
+/**
+ * engine/qa-runners.js — P-b8e1d4a6
+ *
+ * Pluggable test-runner registry for QA Sessions. Mirrors the
+ * engine/watches.js target-type registry pattern (registerTargetType /
+ * getTargetType / listTargetTypes / plugin folder discovery).
+ *
+ * A runner adapter teaches QA Sessions how to translate a natural-language
+ * QA flow into a runner-native test file, execute it against a managed-spawn
+ * target, and surface artifacts. Each runner is registered with five hooks:
+ *
+ *   detect(target, project)          → boolean. Inspect the target/project
+ *                                      and return true when this runner is
+ *                                      the right choice (e.g. a Playwright
+ *                                      adapter returns true when the project
+ *                                      has playwright.config.* in its repo).
+ *   generateBrief(opts)              → string|object. Produces the
+ *                                      instructions handed to the DRAFT
+ *                                      agent so it emits a runner-native
+ *                                      test file under engine/qa-tests/<id>/.
+ *   executeBrief(opts)               → result. Produces the instructions /
+ *                                      command the EXECUTE agent runs to
+ *                                      invoke the drafted test against the
+ *                                      live managed-spawn target.
+ *   validateOutputDir(dir)           → { ok: boolean, errors: string[] }.
+ *                                      Called after DRAFT lands a test file
+ *                                      to confirm the runner-native artifact
+ *                                      is present before allowing EXECUTE
+ *                                      to schedule.
+ *   installHint                      → string. Human-readable instructions
+ *                                      shown to the user when detect()
+ *                                      returns true but the runner CLI is
+ *                                      missing (e.g. "Run `npm i -D
+ *                                      @playwright/test` and `npx playwright
+ *                                      install` to enable this runner.").
+ *
+ * Resolution order (detectRunner):
+ *   1. Explicit override: when the caller passes a registered runner name,
+ *      return that runner immediately — no detect() call. This is what the
+ *      `qa-session` POST payload's `runner` field flows into.
+ *   2. Priority-desc iteration: registered runners are scanned in descending
+ *      `priority` order (ties broken by name asc for determinism). The first
+ *      runner whose `detect(target, project)` returns truthy wins.
+ *   3. No match: return null. Callers surface this as a "no QA runner
+ *      detected — pass runner=… explicitly or install one" error.
+ *
+ * Plugin folder discovery:
+ *   At module load, every `*.js` file under `<MINIONS_DIR>/qa-runners.d/`
+ *   is required inside its own try/catch. Each file exports either:
+ *
+ *     module.exports = { name: 'playwright', spec: { ... } };
+ *
+ *   or an array of those objects for multi-runner plugins. Bad plugin files
+ *   log WARN with the file path and do NOT block sibling plugins or built-in
+ *   runners from registering. A missing qa-runners.d/ directory is a silent
+ *   no-op (matches watches.d/ behaviour).
+ *
+ * Security: plugins are user-trusted JS (same trust level as `playbooks/`
+ * and `watches.d/`); no sandboxing. Hot-reload is not supported at module
+ * level — restart the engine to pick up new plugin files, or POST
+ * /api/qa/runners/reload (implemented in P-d2f5a8c9) which re-invokes
+ * loadRunnerPlugins().
+ *
+ * Built-in runner adapters (Maestro + Playwright) ship in P-c4a9e7f3.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const shared = require('./shared');
+const { log } = shared;
+
+const _KEBAB_RE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
+const RUNNER_NAME_MAX = 64;
+const REQUIRED_HOOK_FNS = ['detect', 'generateBrief', 'executeBrief', 'validateOutputDir'];
+
+// Registry. Keyed by runner name (kebab-case). The whole object is the
+// allowlist — listRunners / detectRunner / `POST /api/qa/runners/reload`
+// derive their answers from it.
+const RUNNERS = {};
+
+function _isNonEmptyString(v) {
+  return typeof v === 'string' && v.length > 0;
+}
+
+/**
+ * Validate a runner spec without registering it. Returns { ok, errors }.
+ * Never throws — `registerQaRunner` is the throwing wrapper.
+ */
+function validateRunnerSpec(name, spec) {
+  const errors = [];
+
+  if (!_isNonEmptyString(name)) {
+    errors.push('name is required (non-empty string)');
+  } else {
+    if (name.length > RUNNER_NAME_MAX) {
+      errors.push(`name exceeds ${RUNNER_NAME_MAX} chars`);
+    }
+    if (!_KEBAB_RE.test(name)) {
+      errors.push('name must be kebab-case (a-z, 0-9, hyphens; no leading/trailing hyphen, no double hyphen)');
+    }
+  }
+
+  if (!spec || typeof spec !== 'object' || Array.isArray(spec)) {
+    return { ok: false, errors: errors.concat(['spec must be a plain object']) };
+  }
+
+  if (typeof spec.priority !== 'number' || !Number.isFinite(spec.priority)) {
+    errors.push('spec.priority is required (finite number; higher = preferred)');
+  }
+
+  for (const hook of REQUIRED_HOOK_FNS) {
+    if (typeof spec[hook] !== 'function') {
+      errors.push(`spec.${hook} is required (function)`);
+    }
+  }
+
+  if (!_isNonEmptyString(spec.installHint)) {
+    errors.push('spec.installHint is required (non-empty string)');
+  }
+
+  return { ok: errors.length === 0, errors };
+}
+
+/**
+ * Register a QA runner adapter. Throws on validation failure so a bad
+ * registration is loud — the engine prints the file path + reason at boot
+ * via the per-plugin try/catch in _loadRunnerPlugins.
+ *
+ * Last-write-wins: re-registering an existing name replaces the previous
+ * spec. This matches watches.js so plugins can override built-ins.
+ */
+function registerQaRunner(name, spec) {
+  const v = validateRunnerSpec(name, spec);
+  if (!v.ok) {
+    const err = new Error(`registerQaRunner(${name}): ${v.errors.join('; ')}`);
+    err.validationErrors = v.errors;
+    throw err;
+  }
+  // Freeze the registry entry so callers can't mutate hooks after register-time.
+  // installHint stays as a string, priority stays as a number.
+  RUNNERS[name] = Object.freeze({
+    name,
+    priority: spec.priority,
+    detect: spec.detect,
+    generateBrief: spec.generateBrief,
+    executeBrief: spec.executeBrief,
+    validateOutputDir: spec.validateOutputDir,
+    installHint: spec.installHint,
+    // Optional human-readable label/description for dashboard pickers.
+    label: _isNonEmptyString(spec.label) ? spec.label : name,
+    description: _isNonEmptyString(spec.description) ? spec.description : '',
+  });
+}
+
+/** Returns the registered spec for a runner name, or null. */
+function getRunner(name) {
+  if (!_isNonEmptyString(name)) return null;
+  return RUNNERS[name] || null;
+}
+
+/**
+ * Returns the registered runners as a serializable array, sorted priority
+ * desc, then name asc for determinism. Hooks are NOT included because they
+ * are functions — only metadata (name, priority, label, description,
+ * installHint) is returned, suitable for `GET /api/qa/runners` and
+ * dashboard pickers.
+ */
+function listRunners() {
+  return Object.keys(RUNNERS)
+    .sort()
+    .map(k => RUNNERS[k])
+    .sort((a, b) => (b.priority - a.priority) || a.name.localeCompare(b.name))
+    .map(r => ({
+      name: r.name,
+      priority: r.priority,
+      label: r.label,
+      description: r.description,
+      installHint: r.installHint,
+    }));
+}
+
+/**
+ * Resolve which runner should handle a target. Resolution order:
+ *
+ *   1. If `explicitRunner` names a registered runner, return it (no detect
+ *      call). Unknown explicit names return null — the caller decides
+ *      whether to fall through to auto-detect or surface an error.
+ *   2. Iterate registered runners by priority desc (ties broken by name
+ *      asc) and return the first whose `detect(target, project)` returns
+ *      truthy. Per-runner detect() errors are caught + logged at WARN so
+ *      one broken adapter can't poison the iteration.
+ *   3. No match: return null.
+ *
+ * @param {object} target           - target object from the session spec
+ *                                    (e.g. { kind: 'pr', prId: 1234 }).
+ * @param {object|null} project     - optional project config (or null when
+ *                                    the session is central / project-less).
+ * @param {string|null} [explicitRunner] - explicit runner name from the
+ *                                    session spec; takes precedence.
+ * @returns {object|null}           - the frozen registry entry, or null.
+ */
+function detectRunner(target, project, explicitRunner) {
+  // Explicit-runner-wins: when the caller specifies a runner by name and it
+  // exists, use it immediately. detect() is intentionally NOT consulted —
+  // the user said this runner, so honour it. Unknown explicit names fall
+  // through to null (auto-detect is suppressed to avoid surprising the user).
+  if (explicitRunner !== undefined && explicitRunner !== null && explicitRunner !== '') {
+    if (!_isNonEmptyString(explicitRunner)) return null;
+    return RUNNERS[explicitRunner] || null;
+  }
+
+  const ordered = Object.keys(RUNNERS)
+    .sort()
+    .map(k => RUNNERS[k])
+    .sort((a, b) => (b.priority - a.priority) || a.name.localeCompare(b.name));
+
+  for (const r of ordered) {
+    let hit = false;
+    try {
+      hit = !!r.detect(target, project);
+    } catch (err) {
+      log('warn', `qa-runners: detect() threw for runner '${r.name}': ${err.message}`);
+      continue;
+    }
+    if (hit) return r;
+  }
+  return null;
+}
+
+// ── Plugin folder discovery (qa-runners.d/) ─────────────────────────────────
+// Mirrors engine/watches.js:_loadPluginTargetTypes (W-mp7hg58e000b5212).
+// Auto-loaded at module load time below. Tests bust the require cache via
+// test/_helpers.js ISOLATED_MODULES so MINIONS_TEST_DIR/qa-runners.d/ takes
+// effect on every createTestMinionsDir() boundary.
+function _loadRunnerPlugins() {
+  const dir = path.join(shared.MINIONS_DIR, 'qa-runners.d');
+  let entries;
+  try {
+    entries = fs.readdirSync(dir);
+  } catch (err) {
+    if (err && err.code === 'ENOENT') return; // silent no-op
+    log('warn', `qa-runners.d/ readdir failed: ${err.message}`);
+    return;
+  }
+  for (const fname of entries.sort()) {
+    if (!fname.endsWith('.js')) continue;
+    const fp = path.join(dir, fname);
+    try {
+      // Bust cache so re-loads (tests, /api/qa/runners/reload) pick up edits.
+      try { delete require.cache[require.resolve(fp)]; } catch {}
+      const mod = require(fp);
+      const list = Array.isArray(mod) ? mod : [mod];
+      for (const entry of list) {
+        if (!entry || !entry.name || !entry.spec) {
+          log('warn', `qa-runners.d/${fname}: export missing { name, spec } — skipping entry`);
+          continue;
+        }
+        try {
+          registerQaRunner(entry.name, entry.spec);
+          log('info', `Loaded QA runner '${entry.name}' from qa-runners.d/${fname}`);
+        } catch (regErr) {
+          log('warn', `qa-runners.d/${fname}: registerQaRunner('${entry.name}') failed: ${regErr.message}`);
+        }
+      }
+    } catch (err) {
+      log('warn', `qa-runners.d/${fname}: load failed: ${err.message}`);
+    }
+  }
+}
+
+// Clear the in-process registry. Exposed for tests + `POST
+// /api/qa/runners/reload` (P-d2f5a8c9), which calls _clearRunners() →
+// _registerBuiltins() → _loadRunnerPlugins() to re-scan disk without
+// restarting the engine.
+function _clearRunners() {
+  for (const k of Object.keys(RUNNERS)) delete RUNNERS[k];
+}
+
+// Register the built-in adapters that ship with the engine. Plugins under
+// <MINIONS_DIR>/qa-runners.d/ run AFTER this and may override built-ins by
+// name (last-write-wins, mirrors watches.js). Surface registration
+// failures as WARN so a broken built-in does not block other adapters or
+// disk plugins. Built-ins are required relative to this file (not
+// MINIONS_DIR) so test isolation can't strand them on a stale module path.
+function _registerBuiltins() {
+  const BUILTIN_FILES = ['./qa-runners/maestro', './qa-runners/playwright'];
+  for (const rel of BUILTIN_FILES) {
+    try {
+      const resolved = require.resolve(rel);
+      delete require.cache[resolved];
+      const mod = require(rel);
+      if (!mod || !mod.name || !mod.spec) {
+        log('warn', `qa-runners built-in ${rel}: export missing { name, spec } — skipping`);
+        continue;
+      }
+      try {
+        registerQaRunner(mod.name, mod.spec);
+      } catch (regErr) {
+        log('warn', `qa-runners built-in ${rel}: registerQaRunner('${mod.name}') failed: ${regErr.message}`);
+      }
+    } catch (err) {
+      log('warn', `qa-runners built-in ${rel}: load failed: ${err.message}`);
+    }
+  }
+}
+
+_registerBuiltins();
+_loadRunnerPlugins();
+
+module.exports = {
+  RUNNER_NAME_MAX,
+  REQUIRED_HOOK_FNS,
+  validateRunnerSpec,
+  registerQaRunner,
+  getRunner,
+  listRunners,
+  detectRunner,
+  // Exposed for tests + the reload endpoint (P-d2f5a8c9).
+  _loadRunnerPlugins,
+  _registerBuiltins,
+  _clearRunners,
+  _RUNNERS: RUNNERS,
+};