@atlashub/smartstack-cli 4.80.0 → 5.0.0

package/templates/skills/development/testing/ui-test/cli/run-ui-test/execute.ts

@@ -0,0 +1,136 @@
+/**
+ * cli:run-ui-test — execute.ts
+ *
+ * Reads manifest + users, runs each test via dev-browser-driver, returns
+ * an aggregated report. NO persistence here — the Studio runner consumes
+ * the envelope and calls uiTestService for DB rows.
+ *
+ * Auto-correction is also out of scope for this CLI — it's the Studio
+ * runner's responsibility (Vague 7). The CLI runs each test EXACTLY ONCE.
+ */
+import path from 'node:path';
+import { promises as fs } from 'node:fs';
+import {
+  runDevBrowserTest,
+  type DriverTestSpec,
+  type DriverTestUser,
+} from './lib/dev-browser-driver.js';
+import type { RunUiTestInput, RunUiTestResult, RunUiTestReport } from './types.js';
+
+interface ManifestFile {
+  generatedAt: string;
+  module: string;
+  appCode: string;
+  tests: Array<{
+    id: string;
+    scenario: DriverTestSpec['scenario'];
+    page: string;
+    role: string;
+    fixture?: Record<string, unknown>;
+    expectations: DriverTestSpec['expectations'];
+  }>;
+}
+
+interface UsersFile {
+  generatedAt: string;
+  module: string;
+  appCode: string;
+  users: Array<{
+    role: string;
+    email: string;
+    password: string;
+    applicationCode: string;
+    roleName: string;
+  }>;
+}
+
+export async function execute(spec: RunUiTestInput): Promise<RunUiTestReport> {
+  const manifestAbs = path.resolve(spec.projectPath, spec.manifestPath);
+  const usersAbs = path.resolve(spec.projectPath, spec.usersPath);
+
+  const manifest: ManifestFile = JSON.parse(await fs.readFile(manifestAbs, 'utf8'));
+  const usersFile: UsersFile = JSON.parse(await fs.readFile(usersAbs, 'utf8'));
+
+  const usersByRole = new Map<string, DriverTestUser>();
+  for (const u of usersFile.users) {
+    usersByRole.set(u.role, { role: u.role, email: u.email, password: u.password });
+  }
+
+  // Filter tests by --test-id and --module if provided.
+  let tests = manifest.tests;
+  if (spec.testId) tests = tests.filter((t) => t.id === spec.testId);
+  if (spec.module && spec.module !== manifest.module) {
+    return {
+      module: manifest.module,
+      appCode: manifest.appCode,
+      totalTests: 0,
+      passed: 0,
+      failed: 0,
+      durationMs: 0,
+      results: [],
+    };
+  }
+
+  const startedAt = Date.now();
+  const results: RunUiTestResult[] = [];
+
+  for (const t of tests) {
+    const user = usersByRole.get(t.role);
+    if (!user) {
+      results.push({
+        testId: t.id,
+        scenario: t.scenario,
+        page: t.page,
+        role: t.role,
+        status: 'fail',
+        reason: `driver-error: no test user seeded for role "${t.role}"`,
+        durationMs: 0,
+        capture: { exitCode: null },
+      });
+      continue;
+    }
+
+    const driverResult = await runDevBrowserTest(
+      {
+        id: t.id,
+        scenario: t.scenario,
+        page: t.page,
+        role: t.role,
+        fixture: t.fixture,
+        expectations: t.expectations,
+      },
+      user,
+      {
+        frontendUrl: spec.frontendUrl,
+        apiUrl: spec.apiUrl,
+        captureDir: spec.captureDir,
+        timeoutMs: spec.timeoutMs,
+        cwd: spec.projectPath,
+      },
+    );
+
+    results.push({
+      testId: t.id,
+      scenario: t.scenario,
+      page: t.page,
+      role: t.role,
+      status: driverResult.status,
+      reason: driverResult.reason,
+      durationMs: driverResult.durationMs,
+      capture: driverResult.capture,
+    });
+  }
+
+  const passed = results.filter((r) => r.status === 'pass').length;
+  const failed = results.length - passed;
+
+  return {
+    module: manifest.module,
+    appCode: manifest.appCode,
+    totalTests: results.length,
+    passed,
+    failed,
+    durationMs: Date.now() - startedAt,
+    results,
+  };
+}

package/templates/skills/development/testing/ui-test/cli/run-ui-test/index.ts

@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+/**
+ * cli:run-ui-test — index.ts
+ *
+ * Args:
+ *   --project-path <abs>      Required. Project root.
+ *   --frontend-url <url>      Required. e.g. http://localhost:5173
+ *   --api-url <url>           Required. e.g. http://localhost:5142
+ *   --module <name>           Optional filter.
+ *   --test-id <id>            Optional. Run only one test.
+ *   --manifest-path <rel>     Default: tests/ui-test/manifest.json
+ *   --users-path <rel>        Default: tests/ui-test/test-users.json
+ *   --capture-dir <rel>       Default: tests/ui-test/captures
+ *   --timeout-ms <int>        Default: 60000
+ */
+import { parseArgs } from 'node:util';
+import { validate } from './validate.js';
+import { execute } from './execute.js';
+import { executeEnvelope, failExecute, printEnvelope } from '../../../../../lib/output.js';
+
+const COMMAND = 'run-ui-test';
+
+async function main(): Promise<void> {
+  const args = parseArgs({
+    options: {
+      'project-path': { type: 'string' },
+      'frontend-url': { type: 'string' },
+      'api-url': { type: 'string' },
+      module: { type: 'string' },
+      'test-id': { type: 'string' },
+      'manifest-path': { type: 'string', default: 'tests/ui-test/manifest.json' },
+      'users-path': { type: 'string', default: 'tests/ui-test/test-users.json' },
+      'capture-dir': { type: 'string', default: 'tests/ui-test/captures' },
+      'timeout-ms': { type: 'string', default: '60000' },
+    },
+    allowPositionals: false,
+  });
+
+  const raw = {
+    projectPath: args.values['project-path'],
+    frontendUrl: args.values['frontend-url'],
+    apiUrl: args.values['api-url'],
+    module: args.values.module,
+    testId: args.values['test-id'],
+    manifestPath: args.values['manifest-path'],
+    usersPath: args.values['users-path'],
+    captureDir: args.values['capture-dir'],
+    timeoutMs: Number.parseInt(args.values['timeout-ms'] ?? '60000', 10),
+  };
+
+  const v = await validate(raw);
+  if (!v.valid || !v.data) {
+    printEnvelope(failExecute(COMMAND, v.errors));
+    process.exit(2);
+  }
+
+  try {
+    const report = await execute(v.data);
+    const envelope = executeEnvelope(COMMAND, {
+      success: report.failed === 0,
+      report: report as unknown as Record<string, unknown>,
+      warnings: v.warnings,
+      nextSteps: report.failed > 0
+        ? [`${report.failed}/${report.totalTests} test(s) failed. The Studio runner will spawn debug-bug fix loops (max 50 iter per test).`]
+        : ['All tests passed.'],
+    });
+    printEnvelope(envelope);
+    process.exit(report.failed === 0 ? 0 : 1);
+  } catch (err) {
+    printEnvelope(failExecute(COMMAND, [err instanceof Error ? err.message : String(err)]));
+    process.exit(1);
+  }
+}
+
+main();

package/templates/skills/development/testing/ui-test/cli/run-ui-test/lib/dev-browser-driver.ts

@@ -0,0 +1,250 @@
+/**
+ * dev-browser-driver — Wrapper around the upstream `dev-browser --headless` CLI.
+ *
+ * Responsibilities:
+ *  - Spawn `dev-browser --headless` (or `npx dev-browser --headless` if not on PATH)
+ *  - Pipe a rendered JS script via stdin (the upstream CLI has no --script flag)
+ *  - Capture stdout (single-line JSON result) + stderr (trace) + exit code
+ *  - Enforce a wall-clock timeout via AbortController (upstream has no --timeout flag)
+ *  - Parse the final JSON object from stdout
+ *
+ * The script contract: the rendered template MUST emit exactly one
+ * `console.log(JSON.stringify({ status, reason, capture }))` on stdout, with
+ * any trace output going to console.warn/error. Driver ignores everything but
+ * the LAST valid JSON object on stdout (so partial logs don't confuse parsing).
+ *
+ * Upstream: https://github.com/SawyerHood/dev-browser (MIT, v0.2.7).
+ * Pinned in the consumer project's package.json devDependencies.
+ */
+import { spawn, type ChildProcess } from 'node:child_process';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { loadTemplate } from '../../../../../../lib/template-loader.js';
+
+export interface DriverTestSpec {
+  id: string;
+  scenario: 'list' | 'detail' | 'form-submit' | 'edit' | 'delete' | 'permission-negative';
+  page: string;
+  role: string;
+  fixture?: Record<string, unknown>;
+  expectations: {
+    httpStatus?: number[];
+    noConsoleError?: boolean;
+    rendersTestId?: string;
+    redirectsTo?: string;
+  };
+}
+
+export interface DriverTestUser {
+  role: string;
+  email: string;
+  password: string;
+}
+
+export interface DriverContext {
+  /** Absolute URL of the running app's frontend (e.g. http://localhost:5173). */
+  frontendUrl: string;
+  /** Absolute URL of the running app's API (e.g. http://localhost:5142). */
+  apiUrl: string;
+  /** Filesystem dir (relative to project) where screenshots are written. */
+  captureDir: string;
+  /** Wall-clock cap per test (default 60_000 ms). */
+  timeoutMs?: number;
+  /** Override CWD for the dev-browser spawn (defaults to process.cwd()). */
+  cwd?: string;
+}
+
+export interface DriverHttpRequest {
+  url: string;
+  method: string;
+  status: number;
+}
+
+export interface DriverFailureCapture {
+  httpRequests?: DriverHttpRequest[];
+  consoleErrors?: string[];
+  screenshotRelPath?: string;
+  /** Driver-level diagnostic when the script itself crashed. */
+  stderrTail?: string;
+  exitCode?: number | null;
+}
+
+export interface DriverResult {
+  status: 'pass' | 'fail';
+  reason: string;
+  capture: DriverFailureCapture;
+  /** Wall-clock duration for the dev-browser invocation (ms). */
+  durationMs: number;
+}
+
+const DEFAULT_TIMEOUT_MS = 60_000;
+
+/**
+ * Run a single UI test via the dev-browser CLI. Returns a typed result.
+ *
+ * Templates live in ../templates/{scenario}.js.hbs and receive the test spec +
+ * user credentials + driver context as Handlebars data.
+ */
+export async function runDevBrowserTest(
+  test: DriverTestSpec,
+  user: DriverTestUser,
+  ctx: DriverContext,
+): Promise<DriverResult> {
+  const startedAt = Date.now();
+  const timeoutMs = ctx.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+
+  let script: string;
+  try {
+    // loadTemplate appends `templates/` to the dir hint. The driver lives at
+    // run-ui-test/lib/dev-browser-driver.ts, so we walk up one dir to point
+    // at run-ui-test/, where templates/ sits.
+    const driverDir = path.dirname(fileURLToPath(import.meta.url));
+    const cliRoot = path.resolve(driverDir, '..');
+    const render = await loadTemplate(`${test.scenario}.js.hbs`, cliRoot);
+    script = render({
+      test,
+      user,
+      frontendUrl: ctx.frontendUrl,
+      apiUrl: ctx.apiUrl,
+      captureDir: ctx.captureDir,
+      // Convenience aliases — Handlebars template can reference {{page}} directly.
+      testId: test.id,
+      page: test.page,
+      expectations: test.expectations,
+      // Pre-stringified so templates can write `const fixture = {{{fixtureJson}}};`
+      // without needing a `json` Handlebars helper.
+      fixture: test.fixture ?? {},
+      fixtureJson: JSON.stringify(test.fixture ?? {}),
+    });
+  } catch (err) {
+    return {
+      status: 'fail',
+      reason: `template-error: ${err instanceof Error ? err.message : String(err)}`,
+      capture: { exitCode: null },
+      durationMs: Date.now() - startedAt,
+    };
+  }
+
+  return await spawnAndWait(script, ctx.cwd ?? process.cwd(), timeoutMs, startedAt);
+}
+
+function spawnAndWait(
+  script: string,
+  cwd: string,
+  timeoutMs: number,
+  startedAt: number,
+): Promise<DriverResult> {
+  return new Promise((resolve) => {
+    // Prefer `npx dev-browser` so the consumer project's devDependencies pin
+    // wins over any global install (which may diverge from v0.2.7).
+    const proc: ChildProcess = spawn('npx', ['--yes', 'dev-browser', '--headless'], {
+      cwd,
+      stdio: ['pipe', 'pipe', 'pipe'],
+      shell: process.platform === 'win32',
+    });
+
+    let stdout = '';
+    let stderr = '';
+    let killed = false;
+
+    const watchdog = setTimeout(() => {
+      killed = true;
+      try { if (!proc.killed) proc.kill('SIGTERM'); } catch { /* ignore */ }
+    }, timeoutMs);
+
+    proc.stdout?.on('data', (chunk: Buffer) => { stdout += chunk.toString(); });
+    proc.stderr?.on('data', (chunk: Buffer) => { stderr += chunk.toString(); });
+
+    proc.on('error', (err) => {
+      clearTimeout(watchdog);
+      resolve({
+        status: 'fail',
+        reason: `driver-error: spawn failed (${err.message})`,
+        capture: { stderrTail: tail(stderr, 2000), exitCode: null },
+        durationMs: Date.now() - startedAt,
+      });
+    });
+
+    proc.on('close', (code) => {
+      clearTimeout(watchdog);
+
+      if (killed) {
+        resolve({
+          status: 'fail',
+          reason: `driver-error: timeout after ${timeoutMs}ms`,
+          capture: { stderrTail: tail(stderr, 2000), exitCode: code },
+          durationMs: Date.now() - startedAt,
+        });
+        return;
+      }
+
+      const parsed = extractLastJsonObject(stdout);
+      if (!parsed) {
+        resolve({
+          status: 'fail',
+          reason: `driver-error: no JSON result on stdout (exit ${code ?? '?'})`,
+          capture: { stderrTail: tail(stderr, 2000), exitCode: code },
+          durationMs: Date.now() - startedAt,
+        });
+        return;
+      }
+
+      const status = parsed.status === 'pass' ? 'pass' : 'fail';
+      resolve({
+        status,
+        reason: typeof parsed.reason === 'string' ? parsed.reason : (status === 'pass' ? 'ok' : 'unknown'),
+        capture: (parsed.capture as DriverFailureCapture) ?? { exitCode: code },
+        durationMs: Date.now() - startedAt,
+      });
+    });
+
+    // Pipe the script via stdin. dev-browser reads to EOF before executing.
+    proc.stdin?.write(script);
+    proc.stdin?.end();
+  });
+}
+
+function tail(text: string, maxBytes: number): string {
+  if (text.length <= maxBytes) return text;
+  return `…${text.slice(-maxBytes)}`;
+}
+
+/**
+ * Extract the LAST balanced top-level JSON object from a string. Tolerates
+ * preceding logs / ANSI codes / partial fragments. Returns null if no
+ * parseable JSON object is found at any depth.
+ */
+function extractLastJsonObject(text: string): Record<string, unknown> | null {
+  // Walk from the end, find the last '}' then walk backwards counting braces
+  // (with naive string-skipping) to find the matching '{'. Then JSON.parse.
+  // Tries multiple candidates: each '}' from end is a potential close.
+  for (let end = text.lastIndexOf('}'); end >= 0; end = text.lastIndexOf('}', end - 1)) {
+    let depth = 0;
+    let inString = false;
+    let escape = false;
+    for (let i = end; i >= 0; i--) {
+      const ch = text[i];
+      if (escape) { escape = false; continue; }
+      if (ch === '\\') { escape = true; continue; }
+      if (ch === '"') { inString = !inString; continue; }
+      if (inString) continue;
+      if (ch === '}') depth++;
+      else if (ch === '{') {
+        depth--;
+        if (depth === 0) {
+          const candidate = text.slice(i, end + 1);
+          try {
+            const parsed = JSON.parse(candidate);
+            if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+              return parsed as Record<string, unknown>;
+            }
+          } catch {
+            // fall through, try next end position
+          }
+          break;
+        }
+      }
+    }
+  }
+  return null;
+}

package/templates/skills/development/testing/ui-test/cli/run-ui-test/templates/delete.js.hbs

@@ -0,0 +1,83 @@
+// Phase 5 UI test — scenario: delete
+// Test ID: {{testId}}
+// Loads list, finds the first row's delete affordance, confirms, expects 204/200.
+
+const page = await browser.getPage("main");
+const httpRequests = [];
+const consoleErrors = [];
+
+page.on("response", (r) => {
+  try { httpRequests.push({ url: r.url(), status: r.status(), method: r.request().method() }); } catch (e) {}
+});
+page.on("console", (m) => {
+  try { if (m.type() === "error") consoleErrors.push(m.text()); } catch (e) {}
+});
+
+let result;
+
+try {
+  await page.goto("{{frontendUrl}}/login", { waitUntil: "domcontentloaded" });
+  await page.locator('[data-testid="login-email"]').fill("{{user.email}}");
+  await page.locator('[data-testid="login-password"]').fill("{{user.password}}");
+  await page.locator('[data-testid="login-submit"]').click();
+  await page.waitForURL((u) => !u.pathname.endsWith("/login"), { timeout: 10000 });
+
+  const listPath = "{{page}}".replace(/\/__detail_id__\/?$/, "");
+  await page.goto("{{frontendUrl}}" + listPath, { waitUntil: "domcontentloaded" });
+  await page.waitForTimeout(1500);
+
+  const deleteBtn = page.locator('[data-testid$="-list-row"] [data-testid="row-delete"]').first();
+  const hasDelete = await deleteBtn.count() > 0;
+  if (!hasDelete) {
+    result = {
+      status: "fail",
+      reason: "no delete affordance on list — missing data-testid=row-delete",
+      capture: { httpRequests, consoleErrors },
+    };
+    console.log(JSON.stringify(result));
+    return;
+  }
+
+  // Some apps confirm via a dialog/modal. Auto-accept any browser confirm.
+  page.on("dialog", (d) => { try { d.accept(); } catch (e) {} });
+
+  const beforeDelete = httpRequests.length;
+  await deleteBtn.click();
+  await page.waitForTimeout(500);
+
+  // If a confirm modal exists, click its data-testid="confirm-delete" affordance.
+  const confirmBtn = page.locator('[data-testid="confirm-delete"]');
+  if (await confirmBtn.count() > 0) {
+    await confirmBtn.click();
+  }
+  await page.waitForTimeout(2000);
+
+  const deleteResp = httpRequests.slice(beforeDelete).find((r) => r.method === "DELETE");
+  const expected = [{{#each expectations.httpStatus}}{{this}}{{#unless @last}}, {{/unless}}{{/each}}];
+
+  let status = "pass";
+  let reason = "ok";
+  if (!deleteResp) {
+    status = "fail";
+    reason = "no DELETE request fired";
+  } else if (expected.indexOf(deleteResp.status) === -1) {
+    status = "fail";
+    reason = "delete returned HTTP " + deleteResp.status + ", expected one of [" + expected.join(",") + "]";
+  }
+
+  const screenshotRelPath = "{{captureDir}}/{{testId}}.png";
+  try {
+    const buf = await page.screenshot({ fullPage: false });
+    await saveScreenshot(buf, screenshotRelPath);
+  } catch (e) {}
+
+  result = { status, reason, capture: { httpRequests, consoleErrors, screenshotRelPath } };
+} catch (err) {
+  result = {
+    status: "fail",
+    reason: "script-exception: " + (err && err.message ? err.message : String(err)),
+    capture: { httpRequests, consoleErrors },
+  };
+}
+
+console.log(JSON.stringify(result));

package/templates/skills/development/testing/ui-test/cli/run-ui-test/templates/detail.js.hbs

@@ -0,0 +1,87 @@
+// Phase 5 UI test — scenario: detail
+// Test ID: {{testId}}
+// Loads the list page first, picks the first row's link, navigates to its
+// detail page, asserts no 4xx/5xx + expected detail testId renders.
+
+const page = await browser.getPage("main");
+const httpRequests = [];
+const consoleErrors = [];
+
+page.on("response", (r) => {
+  try { httpRequests.push({ url: r.url(), status: r.status(), method: r.request().method() }); } catch (e) {}
+});
+page.on("console", (m) => {
+  try { if (m.type() === "error") consoleErrors.push(m.text()); } catch (e) {}
+});
+
+let result;
+
+try {
+  console.warn("[ui-test] login as " + "{{user.email}}");
+  await page.goto("{{frontendUrl}}/login", { waitUntil: "domcontentloaded" });
+  await page.locator('[data-testid="login-email"]').fill("{{user.email}}");
+  await page.locator('[data-testid="login-password"]').fill("{{user.password}}");
+  await page.locator('[data-testid="login-submit"]').click();
+  await page.waitForURL((u) => !u.pathname.endsWith("/login"), { timeout: 10000 });
+
+  // Step 1: navigate to the LIST page, pick first row.
+  const listPage = "{{page}}".replace(/\/__detail_id__\/?$/, "");
+  console.warn("[ui-test] navigate to list at " + listPage);
+  await page.goto("{{frontendUrl}}" + listPage, { waitUntil: "domcontentloaded" });
+  await page.waitForTimeout(1500);
+
+  // Find first row's link or use href containing /:id pattern.
+  const firstRowLink = page.locator('[data-testid$="-list-row"] a').first();
+  const hasRow = await firstRowLink.count() > 0;
+  if (!hasRow) {
+    result = {
+      status: "fail",
+      reason: "no detail row available — list is empty (run a create test first)",
+      capture: { httpRequests, consoleErrors },
+    };
+    console.log(JSON.stringify(result));
+    return;
+  }
+
+  await firstRowLink.click();
+  await page.waitForLoadState("domcontentloaded");
+  await page.waitForTimeout(1000);
+
+  const bad = httpRequests.find((r) =>
+    [400, 401, 403, 404, 500, 502, 503].indexOf(r.status) !== -1,
+  );
+
+  let hasExpected = true;
+  {{#if expectations.rendersTestId}}
+  hasExpected = (await page.locator('[data-testid="{{expectations.rendersTestId}}"]').count()) > 0;
+  {{/if}}
+
+  let status = "pass";
+  let reason = "ok";
+  if (bad) {
+    status = "fail";
+    reason = "HTTP " + bad.status + " on " + bad.method + " " + bad.url;
+  } else if (!hasExpected) {
+    status = "fail";
+    reason = 'expected testId "{{expectations.rendersTestId}}" not rendered';
+  } else if ({{#if expectations.noConsoleError}}consoleErrors.length > 0{{else}}false{{/if}}) {
+    status = "fail";
+    reason = "console errors: " + consoleErrors[0];
+  }
+
+  const screenshotRelPath = "{{captureDir}}/{{testId}}.png";
+  try {
+    const buf = await page.screenshot({ fullPage: false });
+    await saveScreenshot(buf, screenshotRelPath);
+  } catch (e) { console.warn("screenshot failed:", e && e.message); }
+
+  result = { status, reason, capture: { httpRequests, consoleErrors, screenshotRelPath } };
+} catch (err) {
+  result = {
+    status: "fail",
+    reason: "script-exception: " + (err && err.message ? err.message : String(err)),
+    capture: { httpRequests, consoleErrors },
+  };
+}
+
+console.log(JSON.stringify(result));