@sudobility/testomniac_runner 0.0.128

package/src/config/index.test.ts

@@ -0,0 +1,23 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { loadConfig } from "./index";
+
+describe("config", () => {
+  beforeEach(() => {
+    process.env.TESTOMNIAC_API_URL = "http://localhost:9999";
+    process.env.SCANNER_API_KEY = "test-key";
+    process.env.DEEP_LINK_SECRET = "test-secret";
+  });
+
+  it("loads required config from env", () => {
+    const config = loadConfig();
+    expect(config.apiUrl).toBe("http://localhost:9999");
+    expect(config.scannerApiKey).toBe("test-key");
+    expect(config.deepLinkSecret).toBe("test-secret");
+  });
+
+  it("uses defaults for optional values", () => {
+    const config = loadConfig();
+    expect(config.artifactDir).toBe("./artifacts");
+    expect(config.userDataDir).toBe("./browser-profile");
+  });
+});

package/src/config/index.ts

@@ -0,0 +1,35 @@
+export interface Config {
+  apiUrl: string;
+  scannerApiKey: string;
+  openaiApiKey: string;
+  postmarkServerToken: string;
+  postmarkFromEmail: string;
+  deepLinkSecret: string;
+  appBaseUrl: string;
+  signicIndexerUrl: string;
+  signicWildduckUrl: string;
+  chromiumPath: string;
+  userDataDir: string;
+  artifactDir: string;
+  maxConcurrentRunners: number;
+}
+
+export function loadConfig(): Config {
+  return {
+    apiUrl: process.env.TESTOMNIAC_API_URL || "http://localhost:8027",
+    scannerApiKey: process.env.SCANNER_API_KEY || "",
+    openaiApiKey: process.env.OPENAI_API_KEY || "",
+    postmarkServerToken: process.env.POSTMARK_SERVER_TOKEN || "",
+    postmarkFromEmail: process.env.POSTMARK_FROM_EMAIL || "",
+    deepLinkSecret: process.env.DEEP_LINK_SECRET || "",
+    appBaseUrl: process.env.APP_BASE_URL || "http://localhost:3000",
+    signicIndexerUrl:
+      process.env.SIGNIC_INDEXER_URL || "https://api.signic.email/idx",
+    signicWildduckUrl:
+      process.env.SIGNIC_WILDDUCK_URL || "https://api.signic.email/api",
+    chromiumPath: process.env.CHROMIUM_PATH || "/usr/bin/chromium",
+    userDataDir: process.env.USER_DATA_DIR || "./browser-profile",
+    artifactDir: process.env.ARTIFACT_DIR || "./artifacts",
+    maxConcurrentRunners: Number(process.env.MAX_CONCURRENT_RUNNERS ?? 5),
+  };
+}

package/src/email/deep-link.test.ts

@@ -0,0 +1,17 @@
+import { describe, it, expect } from "vitest";
+import { generateToken, verifyToken } from "./deep-link";
+
+describe("deep-link", () => {
+  const secret = "test-secret-key-32-chars-long!!!";
+
+  it("generates and verifies a token", async () => {
+    const token = await generateToken(42, secret);
+    expect(typeof token).toBe("string");
+    const runId = await verifyToken(token, secret);
+    expect(runId).toBe(42);
+  });
+
+  it("rejects invalid token", async () => {
+    await expect(verifyToken("garbage", secret)).rejects.toThrow();
+  });
+});

package/src/email/deep-link.ts

@@ -0,0 +1,23 @@
+import { SignJWT, jwtVerify } from "jose";
+
+const encoder = new TextEncoder();
+
+export async function generateToken(
+  runId: number,
+  secret: string
+): Promise<string> {
+  const key = encoder.encode(secret);
+  return new SignJWT({ runId })
+    .setProtectedHeader({ alg: "HS256" })
+    .setExpirationTime("30d")
+    .sign(key);
+}
+
+export async function verifyToken(
+  token: string,
+  secret: string
+): Promise<number> {
+  const key = encoder.encode(secret);
+  const { payload } = await jwtVerify(token, key);
+  return payload.runId as number;
+}

package/src/email/sender.ts

@@ -0,0 +1,35 @@
+import { ServerClient } from "postmark";
+import pino from "pino";
+import { loadConfig } from "../config/index";
+import type { RunSummary } from "../runner/reporter";
+import { buildHtmlEmail, buildTextEmail } from "./templates";
+import { generateToken } from "./deep-link";
+
+const logger = pino({ name: "email-sender" });
+
+export async function sendReportEmail(
+  userEmail: string,
+  runnerName: string,
+  targetUrl: string,
+  runId: number,
+  summary: RunSummary
+): Promise<string> {
+  const config = loadConfig();
+  const token = await generateToken(runId, config.deepLinkSecret);
+  const deepLinkUrl = `${config.appBaseUrl}/results/${token}`;
+
+  const client = new ServerClient(config.postmarkServerToken);
+  const html = buildHtmlEmail(runnerName, targetUrl, summary, deepLinkUrl);
+  const text = buildTextEmail(runnerName, targetUrl, summary, deepLinkUrl);
+
+  await client.sendEmail({
+    From: config.postmarkFromEmail,
+    To: userEmail,
+    Subject: `Test Report: ${runnerName} — ${summary.passed}/${summary.total} passed`,
+    HtmlBody: html,
+    TextBody: text,
+  });
+
+  logger.info({ userEmail, runId }, "report email sent");
+  return token;
+}

package/src/email/templates.ts

@@ -0,0 +1,34 @@
+import type { RunSummary } from "../runner/reporter";
+
+export function buildHtmlEmail(
+  runnerName: string,
+  targetUrl: string,
+  summary: RunSummary,
+  deepLinkUrl: string
+): string {
+  return `
+<html><body>
+<h2>Test Report: ${runnerName}</h2>
+<p>Target: <a href="${targetUrl}">${targetUrl}</a></p>
+<table>
+<tr><td>Total</td><td>${summary.total}</td></tr>
+<tr><td>Passed</td><td>${summary.passed}</td></tr>
+<tr><td>Failed</td><td>${summary.failed}</td></tr>
+<tr><td>Duration</td><td>${(summary.durationMs / 1000).toFixed(1)}s</td></tr>
+</table>
+<p><a href="${deepLinkUrl}">View Full Results</a></p>
+</body></html>`.trim();
+}
+
+export function buildTextEmail(
+  runnerName: string,
+  targetUrl: string,
+  summary: RunSummary,
+  deepLinkUrl: string
+): string {
+  return `Test Report: ${runnerName}
+Target: ${targetUrl}
+Total: ${summary.total} | Passed: ${summary.passed} | Failed: ${summary.failed}
+Duration: ${(summary.durationMs / 1000).toFixed(1)}s
+View results: ${deepLinkUrl}`;
+}

package/src/index.test.ts

@@ -0,0 +1,17 @@
+import server from "./index";
+
+describe("scanner service", () => {
+  it("responds to /health", async () => {
+    const response = await server.fetch(new Request("http://localhost/health"));
+    const body = await response.json();
+
+    expect(response.status).toBe(200);
+    expect(body).toEqual({
+      ok: true,
+      service: "testomniac_runner",
+      pollIntervalMs: 10000,
+      maxConcurrentRunners: 5,
+      activeRuns: 0,
+    });
+  });
+});

package/src/index.ts

@@ -0,0 +1,110 @@
+import { Hono } from "hono";
+import { parseArgs } from "util";
+import pino from "pino";
+import { loadConfig } from "./config/index";
+import { RunnerManager } from "./runner-manager";
+import { runFullScan, runSequenceScan } from "./orchestrator";
+
+const logger = pino({
+  name: "testomniac_runner",
+  level: process.env.LOG_LEVEL ?? "info",
+});
+
+const port = Number(process.env.PORT ?? 8030);
+const config = loadConfig();
+const pollIntervalMs = Number(process.env.SCAN_POLL_INTERVAL_MS ?? 10_000);
+const maxConcurrentRunners = config.maxConcurrentRunners;
+const runnerManager = new RunnerManager(pollIntervalMs, maxConcurrentRunners);
+
+const app = new Hono();
+
+app.get("/", c => {
+  return c.json({
+    service: "testomniac_runner",
+    status: "ok",
+  });
+});
+
+app.get("/health", c => {
+  return c.json({
+    ok: true,
+    service: "testomniac_runner",
+    pollIntervalMs,
+    maxConcurrentRunners,
+    activeRuns: runnerManager.getActiveRunCount(),
+  });
+});
+
+if (import.meta.main) {
+  const { values: args } = parseArgs({
+    args: process.argv.slice(2),
+    options: {
+      "run-id": { type: "string" },
+      "sequence-run-id": { type: "string" },
+      "runner-id": { type: "string" },
+      "base-url": { type: "string" },
+      "size-class": { type: "string", default: "desktop" },
+    },
+    strict: false,
+  });
+
+  if (args["run-id"]) {
+    // One-shot mode: execute a specific test run and exit
+    const scanId = Number(args["run-id"]);
+    const runnerId = Number(args["runner-id"]);
+    const baseUrl = String(args["base-url"] ?? "");
+    const sizeClass = String(args["size-class"] ?? "desktop");
+
+    logger.info({ scanId, runnerId }, "one-shot mode: executing test run");
+    try {
+      await runFullScan({
+        runnerId,
+        scanId,
+        scanUrl: baseUrl,
+        baseUrl,
+        sizeClass,
+        runnerInstanceId: crypto.randomUUID(),
+        runnerInstanceName: "mcp-runner",
+      });
+      logger.info({ scanId }, "one-shot run completed");
+      process.exit(0);
+    } catch (err) {
+      logger.error({ err, scanId }, "one-shot run failed");
+      process.exit(1);
+    }
+  } else if (args["sequence-run-id"]) {
+    // One-shot mode: execute a specific sequence run and exit
+    const sequenceRunId = Number(args["sequence-run-id"]);
+    const runnerId = Number(args["runner-id"]);
+
+    logger.info(
+      { sequenceRunId, runnerId },
+      "one-shot mode: executing sequence run"
+    );
+    try {
+      await runSequenceScan({ sequenceRunId, runnerId });
+      logger.info({ sequenceRunId }, "one-shot sequence run completed");
+      process.exit(0);
+    } catch (err) {
+      logger.error({ err, sequenceRunId }, "one-shot sequence run failed");
+      process.exit(1);
+    }
+  } else {
+    // Default: polling mode
+    setInterval(() => {
+      void runnerManager.tick();
+    }, pollIntervalMs);
+
+    void runnerManager.tick();
+
+    logger.warn(
+      { port, pollIntervalMs, maxConcurrentRunners },
+      "starting scanner service"
+    );
+  }
+}
+
+export default {
+  port,
+  fetch: app.fetch,
+};

package/src/orchestrator.ts

@@ -0,0 +1,220 @@
+import pino from "pino";
+import { ChromiumManager } from "./browser/chromium";
+import { loadConfig } from "./config/index";
+import { computeSummary } from "./runner/reporter";
+import { sendReportEmail } from "./email/sender";
+import { PuppeteerAdapter } from "./adapters/PuppeteerAdapter";
+import {
+  getApiClient,
+  runTestRun,
+  runSequenceRun,
+  createDefaultExpertises,
+  type ScanEventHandler,
+  SizeClass,
+  DESKTOP_SCREENS,
+  MOBILE_SCREENS,
+} from "@sudobility/testomniac_runner_service";
+import type {
+  Credentials,
+  Screen,
+} from "@sudobility/testomniac_runner_service";
+
+const logger = pino({ name: "orchestrator" });
+
+function elapsed(start: number): number {
+  return Date.now() - start;
+}
+
+export interface RunOptions {
+  runnerId: number;
+  scanId: number;
+  scanUrl: string;
+  baseUrl: string;
+  sizeClass: string;
+  runnerName?: string;
+  runnerInstanceId?: string;
+  runnerInstanceName?: string;
+  credentials?: Credentials;
+  userEmail?: string;
+  scanScopePath?: string;
+  loginUrl?: string;
+  entityCredentialId?: number;
+  quickScan?: boolean;
+}
+
+export async function runFullScan(options: RunOptions): Promise<void> {
+  const config = loadConfig();
+  const api = getApiClient(
+    config.apiUrl + "/api/v1/scanner",
+    config.scannerApiKey
+  );
+  const { scanUrl, baseUrl, userEmail } = options;
+  const runnerName = options.runnerName || new URL(scanUrl).hostname;
+  const sizeClass = (options.sizeClass as SizeClass) || SizeClass.Desktop;
+
+  const runStart = Date.now();
+
+  const chromium = new ChromiumManager(config);
+  await chromium.launch();
+  const runnerId = options.runnerId;
+  const scanId = options.scanId;
+
+  try {
+    const defaultScreen: Screen =
+      sizeClass === SizeClass.Desktop ? DESKTOP_SCREENS[0] : MOBILE_SCREENS[0];
+    const page = await chromium.newPage(defaultScreen);
+    const adapter = new PuppeteerAdapter(page);
+
+    const eventHandler: ScanEventHandler = {
+      onPageFound: p =>
+        logger.info(
+          { relativePath: p.relativePath, pageId: p.pageId },
+          "page discovered"
+        ),
+      onPageStateCreated: s =>
+        logger.info({ pageStateId: s.pageStateId }, "page state created"),
+      onTestSurfaceCreated: s =>
+        logger.info(
+          { surfaceId: s.surfaceId, title: s.title },
+          "test surface created"
+        ),
+      onTestInteractionRunCompleted: r =>
+        logger.info(
+          { testInteractionRunId: r.testInteractionRunId, passed: r.passed },
+          "test element run completed"
+        ),
+      onTestRunCompleted: r =>
+        logger.info(
+          { testRunId: r.testRunId, passed: r.passed },
+          "test run completed"
+        ),
+      onFindingCreated: f =>
+        logger.warn({ type: f.type, title: f.title }, "finding created"),
+      onStatsUpdated: stats => logger.debug(stats, "stats updated"),
+      onScreenshotCaptured: () => {},
+      onScanComplete: summary => logger.info(summary, "scan complete"),
+      onError: err => logger.error(err, "scan error"),
+    };
+
+    const runnerInstanceId = options.runnerInstanceId ?? crypto.randomUUID();
+    const runnerInstanceName = options.runnerInstanceName ?? runnerName;
+    const expertises = createDefaultExpertises();
+
+    await runTestRun(
+      adapter,
+      {
+        testRunId: scanId,
+        runnerId,
+        testEnvironmentId: undefined,
+        baseUrl,
+        sizeClass,
+        runnerInstanceId,
+        runnerInstanceName,
+        scanScopePath: options.scanScopePath,
+        loginUrl: options.loginUrl,
+        entityCredentialId: options.entityCredentialId,
+        credentials: options.credentials,
+        quickScan: options.quickScan,
+      },
+      api,
+      expertises,
+      eventHandler
+    );
+
+    // Send email report after scan completes
+    if (userEmail) {
+      const summary = computeSummary([]);
+      const token = await sendReportEmail(
+        userEmail,
+        runnerName,
+        scanUrl,
+        scanId,
+        summary
+      );
+      await api.createReportEmail({
+        rootTestRunId: scanId,
+        userEmail,
+        deepLinkToken: token,
+      });
+    }
+
+    await page.close();
+
+    logger.info(
+      { scanId, sizeClass, totalDurationMs: elapsed(runStart) },
+      "run complete"
+    );
+  } finally {
+    await chromium.close();
+  }
+}
+
+export interface SequenceRunOptions {
+  sequenceRunId: number;
+  runnerId: number;
+  sizeClass?: string;
+}
+
+export async function runSequenceScan(
+  options: SequenceRunOptions
+): Promise<void> {
+  const config = loadConfig();
+  const api = getApiClient(
+    config.apiUrl + "/api/v1/scanner",
+    config.scannerApiKey
+  );
+
+  const runStart = Date.now();
+  const sizeClass = (options.sizeClass as SizeClass) || SizeClass.Desktop;
+
+  const chromium = new ChromiumManager(config);
+  await chromium.launch();
+
+  try {
+    const defaultScreen =
+      sizeClass === SizeClass.Desktop ? DESKTOP_SCREENS[0] : MOBILE_SCREENS[0];
+    const page = await chromium.newPage(defaultScreen);
+    const adapter = new PuppeteerAdapter(page);
+    const expertises = createDefaultExpertises();
+
+    const eventHandler: ScanEventHandler = {
+      onPageFound: p => logger.info(p, "page found"),
+      onPageStateCreated: s => logger.info(s, "page state created"),
+      onTestSurfaceCreated: s => logger.info(s, "surface created"),
+      onTestInteractionRunCompleted: r =>
+        logger.info(r, "interaction completed"),
+      onTestRunCompleted: r => logger.info(r, "run completed"),
+      onFindingCreated: f => logger.warn(f, "finding created"),
+      onStatsUpdated: s => logger.debug(s, "stats"),
+      onScreenshotCaptured: () => {},
+      onScanComplete: s => logger.info(s, "complete"),
+      onError: e => logger.error(e, "error"),
+    };
+
+    await runSequenceRun(
+      adapter,
+      {
+        sequenceRunId: options.sequenceRunId,
+        runnerId: options.runnerId,
+        sizeClass,
+        runnerInstanceId: crypto.randomUUID(),
+        runnerInstanceName: "mcp-runner",
+      },
+      api,
+      expertises,
+      eventHandler
+    );
+
+    await page.close();
+
+    logger.info(
+      {
+        sequenceRunId: options.sequenceRunId,
+        totalDurationMs: elapsed(runStart),
+      },
+      "sequence run complete"
+    );
+  } finally {
+    await chromium.close();
+  }
+}

package/src/plugins/content/ai-checks.ts

@@ -0,0 +1,115 @@
+import pino from "pino";
+import type { OpenAI } from "openai";
+import type { PluginIssue } from "../types";
+
+const logger = pino({ name: "ai-checks" });
+
+export async function checkSpellingAndGrammar(
+  client: OpenAI,
+  text: string,
+  pageUrl: string
+): Promise<PluginIssue[]> {
+  if (text.trim().length < 20) {
+    return [];
+  }
+
+  const truncated = text.substring(0, 3000);
+
+  const response = await client.chat.completions.create({
+    model: "gpt-4o-mini",
+    messages: [
+      {
+        role: "system",
+        content:
+          "You are a proofreader. Find spelling and grammar errors in the text. Return a JSON array of objects with 'error' (the mistake) and 'suggestion' (the correction). Return an empty array if no errors are found. Only output valid JSON.",
+      },
+      {
+        role: "user",
+        content: truncated,
+      },
+    ],
+    temperature: 0,
+    response_format: { type: "json_object" },
+  });
+
+  const content = response.choices[0]?.message?.content;
+  if (!content) return [];
+
+  try {
+    const parsed = JSON.parse(content);
+    const errors: { error: string; suggestion: string }[] =
+      parsed.errors ?? parsed;
+
+    if (!Array.isArray(errors)) return [];
+
+    return errors.map(err => ({
+      type: "content-spelling-grammar",
+      severity: "info" as const,
+      description: `Spelling/grammar: "${err.error}" -> "${err.suggestion}"`,
+      pageUrl,
+      details: { error: err.error, suggestion: err.suggestion },
+    }));
+  } catch (err) {
+    logger.warn({ err }, "failed to parse LLM spelling/grammar response");
+    return [];
+  }
+}
+
+export async function checkTerminologyConsistency(
+  client: OpenAI,
+  pageTexts: { url: string; text: string }[]
+): Promise<PluginIssue[]> {
+  if (pageTexts.length < 2) {
+    return [];
+  }
+
+  const summaries = pageTexts
+    .map((p, i) => `Page ${i + 1} (${p.url}):\n${p.text.substring(0, 1000)}`)
+    .join("\n\n---\n\n");
+
+  const response = await client.chat.completions.create({
+    model: "gpt-4o-mini",
+    messages: [
+      {
+        role: "system",
+        content:
+          "You check for inconsistent terminology across web pages. Find cases where different pages use different terms for the same concept (e.g., 'sign in' vs 'log in'). Return a JSON object with an 'inconsistencies' array of objects with 'term1', 'term2', 'pages' (array of page numbers). Return empty array if consistent.",
+      },
+      {
+        role: "user",
+        content: summaries,
+      },
+    ],
+    temperature: 0,
+    response_format: { type: "json_object" },
+  });
+
+  const content = response.choices[0]?.message?.content;
+  if (!content) return [];
+
+  try {
+    const parsed = JSON.parse(content);
+    const inconsistencies: {
+      term1: string;
+      term2: string;
+      pages: number[];
+    }[] = parsed.inconsistencies ?? [];
+
+    if (!Array.isArray(inconsistencies)) return [];
+
+    return inconsistencies.map(item => ({
+      type: "content-terminology-inconsistency",
+      severity: "info" as const,
+      description: `Inconsistent terminology: "${item.term1}" vs "${item.term2}"`,
+      pageUrl: pageTexts[0].url,
+      details: {
+        term1: item.term1,
+        term2: item.term2,
+        pages: item.pages.map(i => pageTexts[i - 1]?.url ?? `Page ${i}`),
+      },
+    }));
+  } catch (err) {
+    logger.warn({ err }, "failed to parse LLM terminology response");
+    return [];
+  }
+}

package/src/plugins/content/checks.test.ts

@@ -0,0 +1,49 @@
+import { describe, it, expect } from "vitest";
+import {
+  checkPlaceholderContent,
+  checkReadability,
+  checkCopyrightYear,
+} from "./checks";
+
+const PAGE_URL = "https://example.com";
+
+describe("content checks", () => {
+  it("flags lorem ipsum placeholder text", () => {
+    const text = "Welcome to our site. Lorem ipsum dolor sit amet.";
+    const issues = checkPlaceholderContent(text, PAGE_URL);
+    expect(issues.length).toBeGreaterThanOrEqual(1);
+    expect(issues[0].type).toBe("content-placeholder");
+    expect(issues[0].details?.match).toBe("Lorem ipsum");
+  });
+
+  it("flags TODO placeholder text", () => {
+    const text = "This section is TODO and needs updating.";
+    const issues = checkPlaceholderContent(text, PAGE_URL);
+    expect(issues.length).toBeGreaterThanOrEqual(1);
+    expect(issues.some(i => i.details?.match === "TODO")).toBe(true);
+  });
+
+  it("calculates readability score for long text", () => {
+    // Generate text with 30+ words that is simple enough not to trigger high grade level
+    const simpleText =
+      "The cat sat on the mat. The dog ran in the park. The bird sang in the tree. The fish swam in the sea. The sun was warm and bright today. The children played outside all day long.";
+    const issue = checkReadability(simpleText, PAGE_URL);
+    // Simple text should not flag as too hard to read
+    expect(issue).toBeNull();
+  });
+
+  it("flags outdated copyright year", () => {
+    const text = "© 2020 Example Corp. All rights reserved.";
+    const issue = checkCopyrightYear(text, PAGE_URL);
+    expect(issue).not.toBeNull();
+    expect(issue!.type).toBe("content-outdated-copyright");
+    expect(issue!.details?.year).toBe(2020);
+  });
+
+  it("passes current copyright year", () => {
+    const currentYear = new Date().getFullYear();
+    const text = `© ${currentYear} Example Corp. All rights reserved.`;
+    const issue = checkCopyrightYear(text, PAGE_URL);
+    expect(issue).toBeNull();
+  });
+});