@monster-orange/den 0.0.0 → 0.2.0

package/src/shared/reporting/slack-reporter.ts

@@ -0,0 +1,340 @@
+/**
+ * Slack Reporter
+ * Sends test results to Slack via webhook
+ */
+
+import { readBrand } from "../brand.js";
+import { IncomingWebhook, IncomingWebhookSendArguments } from "@slack/webhook";
+import { readFile } from "fs/promises";
+import { resolve } from "path";
+import type { TestSummary, Pillar } from "../types/index.js";
+
+export interface SlackConfig {
+  webhookUrl: string;
+  enabled: boolean;
+  notifyOn: {
+    testComplete: boolean;
+    testFailure: boolean;
+    thresholdExceeded: boolean;
+  };
+  thresholds: {
+    failurePercentage: number;
+  };
+  channel: string;
+}
+
+export class SlackReporter {
+  private webhook: IncomingWebhook | null = null;
+  private config: SlackConfig | null = null;
+
+  /**
+   * Load Slack configuration
+   */
+  async loadConfig(): Promise<void> {
+    // cwd-derived path is READ-ONLY: optional notify config lookup; nothing is
+    // ever written here (and the unit test pins cwd via jest).
+    const configPath = resolve(process.cwd(), "config/slack.json");
+
+    try {
+      const content = await readFile(configPath, "utf-8");
+      this.config = JSON.parse(content);
+
+      if (
+        this.config?.enabled &&
+        this.config.webhookUrl !== "REPLACE_WITH_SLACK_WEBHOOK_URL"
+      ) {
+        this.webhook = new IncomingWebhook(this.config.webhookUrl);
+      }
+    } catch (error) {
+      console.warn("Failed to load Slack config:", (error as Error).message);
+    }
+  }
+
+  /**
+   * Check if Slack reporting is enabled and configured
+   */
+  isEnabled(): boolean {
+    return this.webhook !== null && this.config !== null && this.config.enabled;
+  }
+
+  /**
+   * Send test summary to Slack
+   */
+  async sendTestSummary(
+    pillar: Pillar,
+    environment: string,
+    summary: TestSummary,
+    reportUrl?: string,
+  ): Promise<void> {
+    if (!this.isEnabled() || !this.config) {
+      return;
+    }
+
+    const failurePercentage =
+      summary.total > 0 ? (summary.failed / summary.total) * 100 : 0;
+    const shouldNotify = this.shouldNotify(summary, failurePercentage);
+
+    if (!shouldNotify) {
+      return;
+    }
+
+    const message = this.formatTestSummary(
+      pillar,
+      environment,
+      summary,
+      reportUrl,
+    );
+
+    try {
+      await this.webhook!.send(message);
+    } catch (error) {
+      console.error(
+        "Failed to send Slack notification:",
+        (error as Error).message,
+      );
+    }
+  }
+
+  /**
+   * Send test failure notification
+   */
+  async sendTestFailure(
+    pillar: Pillar,
+    environment: string,
+    testName: string,
+    errorMessage: string,
+    reportUrl?: string,
+  ): Promise<void> {
+    if (!this.isEnabled() || !this.config?.notifyOn.testFailure) {
+      return;
+    }
+
+    const message = this.formatTestFailure(
+      pillar,
+      environment,
+      testName,
+      errorMessage,
+      reportUrl,
+    );
+
+    try {
+      await this.webhook!.send(message);
+    } catch (error) {
+      console.error(
+        "Failed to send Slack notification:",
+        (error as Error).message,
+      );
+    }
+  }
+
+  /**
+   * Determine if we should send notification based on config
+   */
+  private shouldNotify(
+    summary: TestSummary,
+    failurePercentage: number,
+  ): boolean {
+    if (!this.config) {
+      return false;
+    }
+
+    // Always notify on threshold exceeded
+    if (
+      this.config.notifyOn.thresholdExceeded &&
+      failurePercentage >= this.config.thresholds.failurePercentage
+    ) {
+      return true;
+    }
+
+    // Notify on test complete if enabled
+    if (this.config.notifyOn.testComplete) {
+      return true;
+    }
+
+    // Notify on any failure if enabled
+    if (this.config.notifyOn.testFailure && summary.failed > 0) {
+      return true;
+    }
+
+    return false;
+  }
+
+  /**
+   * Format test summary as Slack message
+   */
+  private formatTestSummary(
+    pillar: Pillar,
+    environment: string,
+    summary: TestSummary,
+    reportUrl?: string,
+  ): IncomingWebhookSendArguments {
+    const passPercentage = (
+      summary.total > 0 ? (summary.passed / summary.total) * 100 : 0
+    ).toFixed(1);
+    const emoji = summary.failed === 0 ? ":white_check_mark:" : ":warning:";
+    const color = summary.failed === 0 ? "#36a64f" : "#ff9800";
+
+    return {
+      username: `${readBrand().name} E2E`,
+      icon_emoji: ":robot_face:",
+      channel: this.config?.channel,
+      attachments: [
+        {
+          color,
+          title: `${emoji} ${this.formatPillar(pillar)} Tests - ${environment}`,
+          fields: [
+            {
+              title: "Status",
+              value:
+                summary.failed === 0
+                  ? "All Passed"
+                  : `${summary.failed} Failed`,
+              short: true,
+            },
+            {
+              title: "Pass Rate",
+              value: `${passPercentage}%`,
+              short: true,
+            },
+            {
+              title: "Total Tests",
+              value: summary.total.toString(),
+              short: true,
+            },
+            {
+              title: "Duration",
+              value: this.formatDuration(summary.duration),
+              short: true,
+            },
+            {
+              title: "Passed",
+              value: summary.passed.toString(),
+              short: true,
+            },
+            {
+              title: "Failed",
+              value: summary.failed.toString(),
+              short: true,
+            },
+            {
+              title: "Skipped",
+              value: summary.skipped.toString(),
+              short: true,
+            },
+            {
+              title: "Environment",
+              value: environment,
+              short: true,
+            },
+          ],
+          footer: `${readBrand().name} E2E`,
+          ts: Math.floor(Date.now() / 1000).toString(),
+        },
+      ],
+      ...(reportUrl && {
+        text: `<${reportUrl}|View Full Report>`,
+      }),
+    };
+  }
+
+  /**
+   * Format test failure as Slack message
+   */
+  private formatTestFailure(
+    pillar: Pillar,
+    environment: string,
+    testName: string,
+    errorMessage: string,
+    reportUrl?: string,
+  ): IncomingWebhookSendArguments {
+    return {
+      username: `${readBrand().name} E2E`,
+      icon_emoji: ":robot_face:",
+      channel: this.config?.channel,
+      attachments: [
+        {
+          color: "#d32f2f",
+          title: `:x: Test Failure - ${this.formatPillar(pillar)}`,
+          fields: [
+            {
+              title: "Environment",
+              value: environment,
+              short: true,
+            },
+            {
+              title: "Test",
+              value: testName,
+              short: true,
+            },
+            {
+              title: "Error",
+              value: this.truncateError(errorMessage),
+              short: false,
+            },
+          ],
+          footer: `${readBrand().name} E2E`,
+          ts: Math.floor(Date.now() / 1000).toString(),
+        },
+      ],
+      ...(reportUrl && {
+        text: `<${reportUrl}|View Full Report>`,
+      }),
+    };
+  }
+
+  /**
+   * Format pillar name for display
+   */
+  private formatPillar(pillar: Pillar): string {
+    const pillarNames: Record<Pillar, string> = {
+      synthetic: "Synthetic (Browser)",
+      integration: "Integration (API)",
+      performance: "Performance (Load)",
+      smoke: "Smoke",
+      journeys: "Journeys (Agent E2E)",
+      security: "Security (OWASP ZAP)",
+      suites: "Repo suites",
+    };
+
+    return pillarNames[pillar] || pillar;
+  }
+
+  /**
+   * Format duration in human-readable format
+   */
+  private formatDuration(ms: number): string {
+    if (ms < 1000) {
+      return `${ms}ms`;
+    }
+
+    const seconds = Math.floor(ms / 1000);
+    const minutes = Math.floor(seconds / 60);
+    const hours = Math.floor(minutes / 60);
+
+    if (hours > 0) {
+      const remainingMinutes = minutes % 60;
+      return `${hours}h ${remainingMinutes}m`;
+    }
+
+    if (minutes > 0) {
+      const remainingSeconds = seconds % 60;
+      return `${minutes}m ${remainingSeconds}s`;
+    }
+
+    return `${seconds}s`;
+  }
+
+  /**
+   * Truncate error message if too long
+   */
+  private truncateError(error: string, maxLength: number = 500): string {
+    if (error.length <= maxLength) {
+      return error;
+    }
+
+    return error.substring(0, maxLength) + "...";
+  }
+}
+
+// Export singleton instance
+export const slackReporter = new SlackReporter();

package/src/shared/results/result-reader.ts

@@ -0,0 +1,266 @@
+/**
+ * Result Reader
+ * Reads and queries test results from JSONL files
+ */
+
+import { readFile, readdir } from "fs/promises";
+import { resolve, join } from "path";
+import type {
+  TestResult,
+  TestSummary,
+  FlakyTest,
+  TrendData,
+  Pillar,
+} from "../types/index.js";
+
+export interface QueryFilters {
+  pillar?: Pillar;
+  environment?: string;
+  dateFrom?: Date;
+  dateTo?: Date;
+  status?: "passed" | "failed" | "skipped";
+  user?: string;
+}
+
+export class ResultReader {
+  private resultsDir: string;
+
+  constructor(resultsDir: string = "./results") {
+    this.resultsDir = resolve(resultsDir);
+  }
+
+  /**
+   * Query results with filters
+   */
+  async query(filters: QueryFilters = {}): Promise<TestResult[]> {
+    const files = await this.getRelevantFiles(filters);
+    const results: TestResult[] = [];
+
+    for (const file of files) {
+      try {
+        const fileResults = await this.readJSONLFile(file);
+        const filtered = fileResults.filter((r) =>
+          this.matchesFilters(r, filters),
+        );
+        results.push(...filtered);
+      } catch (error) {
+        console.warn(
+          `Failed to read file ${file}: ${(error as Error).message}`,
+        );
+      }
+    }
+
+    // Sort by timestamp (newest first)
+    return results.sort(
+      (a, b) =>
+        new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime(),
+    );
+  }
+
+  /**
+   * Get summary statistics for a pillar/environment
+   */
+  async getSummary(
+    pillar: Pillar,
+    environment: string,
+    days: number = 7,
+  ): Promise<TestSummary> {
+    const dateFrom = new Date();
+    dateFrom.setDate(dateFrom.getDate() - days);
+
+    const results = await this.query({ pillar, environment, dateFrom });
+
+    const total = results.length;
+    const passed = results.filter((r) => r.status === "passed").length;
+    const failed = results.filter((r) => r.status === "failed").length;
+    const skipped = results.filter((r) => r.status === "skipped").length;
+    const duration = results.reduce((sum, r) => sum + r.duration, 0);
+    const passRate = total > 0 ? (passed / total) * 100 : 0;
+
+    return {
+      pillar,
+      environment,
+      total,
+      passed,
+      failed,
+      skipped,
+      duration,
+      passRate,
+    };
+  }
+
+  /**
+   * Find flaky tests (tests that fail inconsistently)
+   */
+  async getFlakyTests(pillar: Pillar, days: number = 30): Promise<FlakyTest[]> {
+    const dateFrom = new Date();
+    dateFrom.setDate(dateFrom.getDate() - days);
+
+    const results = await this.query({ pillar, dateFrom });
+
+    // Group results by test name
+    const testGroups = results.reduce(
+      (acc, r) => {
+        if (!acc[r.test]) {
+          acc[r.test] = [];
+        }
+        acc[r.test].push(r);
+        return acc;
+      },
+      {} as Record<string, TestResult[]>,
+    );
+
+    // Find tests with both passes and failures
+    const flakyTests: FlakyTest[] = [];
+
+    for (const [testName, testResults] of Object.entries(testGroups)) {
+      const runs = testResults.length;
+      const failures = testResults.filter((r) => r.status === "failed").length;
+
+      // Consider a test flaky if it has both passes and failures
+      if (failures > 0 && failures < runs && runs >= 3) {
+        flakyTests.push({
+          test: testName,
+          runs,
+          failures,
+          flakyRate: (failures / runs) * 100,
+        });
+      }
+    }
+
+    // Sort by flaky rate (most flaky first)
+    return flakyTests.sort((a, b) => b.flakyRate - a.flakyRate);
+  }
+
+  /**
+   * Get trend data for visualization
+   */
+  async getTrends(
+    pillar: Pillar,
+    environment: string,
+    days: number = 30,
+  ): Promise<TrendData[]> {
+    const dateFrom = new Date();
+    dateFrom.setDate(dateFrom.getDate() - days);
+
+    const results = await this.query({ pillar, environment, dateFrom });
+
+    // Group results by date
+    const dateGroups = results.reduce(
+      (acc, r) => {
+        const date = r.timestamp.split("T")[0];
+        if (!acc[date]) {
+          acc[date] = [];
+        }
+        acc[date].push(r);
+        return acc;
+      },
+      {} as Record<string, TestResult[]>,
+    );
+
+    // Calculate metrics for each date
+    const trends: TrendData[] = [];
+
+    for (const [date, dayResults] of Object.entries(dateGroups)) {
+      const total = dayResults.length;
+      const passed = dayResults.filter((r) => r.status === "passed").length;
+      const passRate = total > 0 ? (passed / total) * 100 : 0;
+      const avgDuration =
+        total > 0
+          ? dayResults.reduce((sum, r) => sum + r.duration, 0) / total
+          : 0;
+
+      trends.push({
+        date,
+        passRate,
+        avgDuration,
+        totalTests: total,
+      });
+    }
+
+    // Sort by date (oldest first)
+    return trends.sort((a, b) => a.date.localeCompare(b.date));
+  }
+
+  /**
+   * Read JSONL file and parse results
+   */
+  private async readJSONLFile(filepath: string): Promise<TestResult[]> {
+    const content = await readFile(filepath, "utf-8");
+    const lines = content
+      .trim()
+      .split("\n")
+      .filter((l) => l.length > 0);
+
+    return lines
+      .map((line) => {
+        try {
+          return JSON.parse(line) as TestResult;
+        } catch (error) {
+          console.warn(`Failed to parse line in ${filepath}: ${line}`);
+          return null;
+        }
+      })
+      .filter((r): r is TestResult => r !== null);
+  }
+
+  /**
+   * Get relevant files based on filters
+   */
+  private async getRelevantFiles(filters: QueryFilters): Promise<string[]> {
+    try {
+      const files = await readdir(this.resultsDir);
+
+      return files
+        .filter((f) => f.endsWith(".jsonl"))
+        .filter((f) => {
+          // Filter by pillar if specified
+          if (filters.pillar && !f.startsWith(filters.pillar)) {
+            return false;
+          }
+
+          // Filter by environment if specified
+          if (filters.environment && !f.includes(`-${filters.environment}-`)) {
+            return false;
+          }
+
+          return true;
+        })
+        .map((f) => join(this.resultsDir, f));
+    } catch {
+      return [];
+    }
+  }
+
+  /**
+   * Check if result matches filters
+   */
+  private matchesFilters(result: TestResult, filters: QueryFilters): boolean {
+    if (filters.status && result.status !== filters.status) {
+      return false;
+    }
+
+    if (filters.user && result.user !== filters.user) {
+      return false;
+    }
+
+    if (filters.dateFrom) {
+      const resultDate = new Date(result.timestamp);
+      if (resultDate < filters.dateFrom) {
+        return false;
+      }
+    }
+
+    if (filters.dateTo) {
+      const resultDate = new Date(result.timestamp);
+      if (resultDate > filters.dateTo) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+}
+
+// Export singleton instance
+export const resultReader = new ResultReader();