@monster-orange/den 0.0.0 → 0.1.0

package/src/shared/reporting/jsonl-reporter.ts

@@ -0,0 +1,131 @@
+/**
+ * JSONL Reporter for Playwright
+ * Writes test results to JSONL format for historical tracking
+ */
+
+import type {
+  Reporter,
+  TestCase,
+  TestResult as PlaywrightTestResult,
+  FullConfig,
+  Suite,
+} from "@playwright/test/reporter";
+import { resultWriter } from "../results/result-writer.js";
+import type { Pillar, Environment } from "../types/index.js";
+import * as os from "os";
+import * as path from "path";
+
+export default class JSONLReporter implements Reporter {
+  onBegin(_config: FullConfig, _suite: Suite) {
+    console.log("[JSONL Reporter] Initialized");
+  }
+
+  async onTestEnd(test: TestCase, result: PlaywrightTestResult) {
+    console.log("[JSONL Reporter] onTestEnd called");
+    try {
+      // Detect pillar from test file path
+      const pillar = this.detectPillar(test.location.file);
+      console.log(`[JSONL Reporter] Pillar detected: ${pillar}`);
+
+      // Get environment from env var or default to local
+      const environment = (process.env.TEST_ENV || "local") as Environment;
+
+      // Get test title path (e.g., "Smoke Tests - App Health › web app is accessible")
+      const testTitle = test.titlePath().slice(1).join(" › ");
+
+      // Get relative test file path for better readability
+      const testFile = this.getRelativeTestPath(test.location.file, pillar);
+
+      // Determine status
+      let status: "passed" | "failed" | "skipped";
+      if (result.status === "passed") {
+        status = "passed";
+      } else if (result.status === "skipped") {
+        status = "skipped";
+      } else {
+        status = "failed";
+      }
+
+      // Get screenshot and trace paths if they exist
+      const screenshot = result.attachments.find(
+        (a) => a.name === "screenshot" || a.contentType?.startsWith("image/"),
+      )?.path;
+
+      const trace = result.attachments.find(
+        (a) => a.name === "trace" || a.contentType?.includes("zip"),
+      )?.path;
+
+      // Write result to JSONL
+      await resultWriter.writeResult({
+        timestamp: new Date().toISOString(),
+        pillar,
+        environment,
+        test: `${testFile} › ${testTitle}`,
+        status,
+        duration: result.duration,
+        user: process.env.USER || os.userInfo().username || "unknown",
+        error: result.error?.message,
+        screenshot,
+        trace,
+        metadata: {
+          browser: test.parent.project()?.name,
+          retries: result.retry,
+          workerIndex: result.workerIndex,
+        },
+      });
+    } catch (error) {
+      // Don't let reporter errors break tests
+      console.warn(
+        `[JSONL Reporter] Failed to write result: ${(error as Error).message}`,
+      );
+    }
+  }
+
+  /**
+   * Detect pillar from test file path
+   */
+  private detectPillar(filePath: string): Pillar {
+    if (
+      filePath.includes("/.synth-run/") ||
+      filePath.includes("/pillars/browser/")
+    ) {
+      return "synthetic";
+    } else if (
+      filePath.includes("/pillars/api/") ||
+      filePath.includes("/integration/")
+    ) {
+      return "integration";
+    } else if (
+      filePath.includes("/pillars/perf/") ||
+      filePath.includes("/performance/")
+    ) {
+      return "performance";
+    } else if (
+      filePath.includes("/pillars/smoke/") ||
+      filePath.includes("/.smoke-run/")
+    ) {
+      return "smoke";
+    } else if (filePath.includes("/pillars/journeys/")) {
+      // Journeys aren't Playwright-run (they have their own JSONL writer in
+      // src/pillars/journeys/report/); branch kept for union completeness.
+      return "journeys";
+    }
+
+    // Default to synthetic
+    return "synthetic";
+  }
+
+  /**
+   * Get relative test path for display
+   */
+  private getRelativeTestPath(filePath: string, pillar: Pillar): string {
+    // Extract path relative to pillar directory
+    const parts = filePath.split(`/${pillar}/tests/`);
+    if (parts.length > 1) {
+      return parts[1].replace(/\.spec\.ts$/, "");
+    }
+
+    // Fallback to just filename
+    return path.basename(filePath, ".spec.ts");
+  }
+}

package/src/shared/reporting/slack-reporter.ts

@@ -0,0 +1,340 @@
+/**
+ * Slack Reporter
+ * Sends test results to Slack via webhook
+ */
+
+import { readBrand } from "../brand.js";
+import { IncomingWebhook, IncomingWebhookSendArguments } from "@slack/webhook";
+import { readFile } from "fs/promises";
+import { resolve } from "path";
+import type { TestSummary, Pillar } from "../types/index.js";
+
+export interface SlackConfig {
+  webhookUrl: string;
+  enabled: boolean;
+  notifyOn: {
+    testComplete: boolean;
+    testFailure: boolean;
+    thresholdExceeded: boolean;
+  };
+  thresholds: {
+    failurePercentage: number;
+  };
+  channel: string;
+}
+
+export class SlackReporter {
+  private webhook: IncomingWebhook | null = null;
+  private config: SlackConfig | null = null;
+
+  /**
+   * Load Slack configuration
+   */
+  async loadConfig(): Promise<void> {
+    // cwd-derived path is READ-ONLY: optional notify config lookup; nothing is
+    // ever written here (and the unit test pins cwd via jest).
+    const configPath = resolve(process.cwd(), "config/slack.json");
+
+    try {
+      const content = await readFile(configPath, "utf-8");
+      this.config = JSON.parse(content);
+
+      if (
+        this.config?.enabled &&
+        this.config.webhookUrl !== "REPLACE_WITH_SLACK_WEBHOOK_URL"
+      ) {
+        this.webhook = new IncomingWebhook(this.config.webhookUrl);
+      }
+    } catch (error) {
+      console.warn("Failed to load Slack config:", (error as Error).message);
+    }
+  }
+
+  /**
+   * Check if Slack reporting is enabled and configured
+   */
+  isEnabled(): boolean {
+    return this.webhook !== null && this.config !== null && this.config.enabled;
+  }
+
+  /**
+   * Send test summary to Slack
+   */
+  async sendTestSummary(
+    pillar: Pillar,
+    environment: string,
+    summary: TestSummary,
+    reportUrl?: string,
+  ): Promise<void> {
+    if (!this.isEnabled() || !this.config) {
+      return;
+    }
+
+    const failurePercentage =
+      summary.total > 0 ? (summary.failed / summary.total) * 100 : 0;
+    const shouldNotify = this.shouldNotify(summary, failurePercentage);
+
+    if (!shouldNotify) {
+      return;
+    }
+
+    const message = this.formatTestSummary(
+      pillar,
+      environment,
+      summary,
+      reportUrl,
+    );
+
+    try {
+      await this.webhook!.send(message);
+    } catch (error) {
+      console.error(
+        "Failed to send Slack notification:",
+        (error as Error).message,
+      );
+    }
+  }
+
+  /**
+   * Send test failure notification
+   */
+  async sendTestFailure(
+    pillar: Pillar,
+    environment: string,
+    testName: string,
+    errorMessage: string,
+    reportUrl?: string,
+  ): Promise<void> {
+    if (!this.isEnabled() || !this.config?.notifyOn.testFailure) {
+      return;
+    }
+
+    const message = this.formatTestFailure(
+      pillar,
+      environment,
+      testName,
+      errorMessage,
+      reportUrl,
+    );
+
+    try {
+      await this.webhook!.send(message);
+    } catch (error) {
+      console.error(
+        "Failed to send Slack notification:",
+        (error as Error).message,
+      );
+    }
+  }
+
+  /**
+   * Determine if we should send notification based on config
+   */
+  private shouldNotify(
+    summary: TestSummary,
+    failurePercentage: number,
+  ): boolean {
+    if (!this.config) {
+      return false;
+    }
+
+    // Always notify on threshold exceeded
+    if (
+      this.config.notifyOn.thresholdExceeded &&
+      failurePercentage >= this.config.thresholds.failurePercentage
+    ) {
+      return true;
+    }
+
+    // Notify on test complete if enabled
+    if (this.config.notifyOn.testComplete) {
+      return true;
+    }
+
+    // Notify on any failure if enabled
+    if (this.config.notifyOn.testFailure && summary.failed > 0) {
+      return true;
+    }
+
+    return false;
+  }
+
+  /**
+   * Format test summary as Slack message
+   */
+  private formatTestSummary(
+    pillar: Pillar,
+    environment: string,
+    summary: TestSummary,
+    reportUrl?: string,
+  ): IncomingWebhookSendArguments {
+    const passPercentage = (
+      summary.total > 0 ? (summary.passed / summary.total) * 100 : 0
+    ).toFixed(1);
+    const emoji = summary.failed === 0 ? ":white_check_mark:" : ":warning:";
+    const color = summary.failed === 0 ? "#36a64f" : "#ff9800";
+
+    return {
+      username: `${readBrand().name} E2E`,
+      icon_emoji: ":robot_face:",
+      channel: this.config?.channel,
+      attachments: [
+        {
+          color,
+          title: `${emoji} ${this.formatPillar(pillar)} Tests - ${environment}`,
+          fields: [
+            {
+              title: "Status",
+              value:
+                summary.failed === 0
+                  ? "All Passed"
+                  : `${summary.failed} Failed`,
+              short: true,
+            },
+            {
+              title: "Pass Rate",
+              value: `${passPercentage}%`,
+              short: true,
+            },
+            {
+              title: "Total Tests",
+              value: summary.total.toString(),
+              short: true,
+            },
+            {
+              title: "Duration",
+              value: this.formatDuration(summary.duration),
+              short: true,
+            },
+            {
+              title: "Passed",
+              value: summary.passed.toString(),
+              short: true,
+            },
+            {
+              title: "Failed",
+              value: summary.failed.toString(),
+              short: true,
+            },
+            {
+              title: "Skipped",
+              value: summary.skipped.toString(),
+              short: true,
+            },
+            {
+              title: "Environment",
+              value: environment,
+              short: true,
+            },
+          ],
+          footer: `${readBrand().name} E2E`,
+          ts: Math.floor(Date.now() / 1000).toString(),
+        },
+      ],
+      ...(reportUrl && {
+        text: `<${reportUrl}|View Full Report>`,
+      }),
+    };
+  }
+
+  /**
+   * Format test failure as Slack message
+   */
+  private formatTestFailure(
+    pillar: Pillar,
+    environment: string,
+    testName: string,
+    errorMessage: string,
+    reportUrl?: string,
+  ): IncomingWebhookSendArguments {
+    return {
+      username: `${readBrand().name} E2E`,
+      icon_emoji: ":robot_face:",
+      channel: this.config?.channel,
+      attachments: [
+        {
+          color: "#d32f2f",
+          title: `:x: Test Failure - ${this.formatPillar(pillar)}`,
+          fields: [
+            {
+              title: "Environment",
+              value: environment,
+              short: true,
+            },
+            {
+              title: "Test",
+              value: testName,
+              short: true,
+            },
+            {
+              title: "Error",
+              value: this.truncateError(errorMessage),
+              short: false,
+            },
+          ],
+          footer: `${readBrand().name} E2E`,
+          ts: Math.floor(Date.now() / 1000).toString(),
+        },
+      ],
+      ...(reportUrl && {
+        text: `<${reportUrl}|View Full Report>`,
+      }),
+    };
+  }
+
+  /**
+   * Format pillar name for display
+   */
+  private formatPillar(pillar: Pillar): string {
+    const pillarNames: Record<Pillar, string> = {
+      synthetic: "Synthetic (Browser)",
+      integration: "Integration (API)",
+      performance: "Performance (Load)",
+      smoke: "Smoke",
+      journeys: "Journeys (Agent E2E)",
+      security: "Security (OWASP ZAP)",
+      suites: "Repo suites",
+    };
+
+    return pillarNames[pillar] || pillar;
+  }
+
+  /**
+   * Format duration in human-readable format
+   */
+  private formatDuration(ms: number): string {
+    if (ms < 1000) {
+      return `${ms}ms`;
+    }
+
+    const seconds = Math.floor(ms / 1000);
+    const minutes = Math.floor(seconds / 60);
+    const hours = Math.floor(minutes / 60);
+
+    if (hours > 0) {
+      const remainingMinutes = minutes % 60;
+      return `${hours}h ${remainingMinutes}m`;
+    }
+
+    if (minutes > 0) {
+      const remainingSeconds = seconds % 60;
+      return `${minutes}m ${remainingSeconds}s`;
+    }
+
+    return `${seconds}s`;
+  }
+
+  /**
+   * Truncate error message if too long
+   */
+  private truncateError(error: string, maxLength: number = 500): string {
+    if (error.length <= maxLength) {
+      return error;
+    }
+
+    return error.substring(0, maxLength) + "...";
+  }
+}
+
+// Export singleton instance
+export const slackReporter = new SlackReporter();