@monster-orange/den 0.0.0-stage → 0.1.0

package/src/shared/perf/report-normalizer.ts

@@ -0,0 +1,404 @@
+import type {
+  PerfCheckResult,
+  PerfMetricSummary,
+  PerfModuleRunReport,
+  PerfReportValue,
+  PerfRunReport,
+  PerfScenarioSummary,
+  PerfThresholdResult,
+} from "./report-types.js";
+
+interface K6Metric {
+  type?: string;
+  contains?: string;
+  values?: Record<string, number>;
+  thresholds?: Record<string, boolean | { ok?: boolean }>;
+  [key: string]: unknown;
+}
+
+interface K6Check {
+  name?: string;
+  passes?: number;
+  fails?: number;
+}
+
+export interface K6Summary {
+  metrics?: Record<string, K6Metric>;
+  root_group?: {
+    checks?: K6Check[] | Record<string, Omit<K6Check, "name">>;
+  };
+  options?: {
+    scenarios?: Record<string, Record<string, unknown>>;
+  };
+}
+
+interface NormalizeSummaryInput {
+  module: string;
+  summary: K6Summary;
+  artifactPath: string;
+  startedAt?: string;
+  finishedAt?: string;
+}
+
+interface BuildRunReportInput {
+  environment: string;
+  modules: NormalizeSummaryInput[];
+  startedAt?: string;
+  finishedAt?: string;
+}
+
+const CORE_METRIC_DEFS: Array<{
+  key: string;
+  source: string;
+  label: string;
+  unit?: string;
+}> = [
+  { key: "count", source: "http_reqs", label: "HTTP requests" },
+  { key: "count", source: "iterations", label: "Iterations" },
+  { key: "value", source: "vus", label: "Virtual users" },
+];
+
+const LATENCY_METRIC_DEFS: Array<{
+  key: string;
+  source: string;
+  label: string;
+}> = [
+  { key: "avg", source: "http_req_duration", label: "HTTP avg" },
+  { key: "med", source: "http_req_duration", label: "HTTP p50" },
+  { key: "p(95)", source: "http_req_duration", label: "HTTP p95" },
+  { key: "p(99)", source: "http_req_duration", label: "HTTP p99" },
+  { key: "max", source: "http_req_duration", label: "HTTP max" },
+];
+
+const THROUGHPUT_METRIC_DEFS: Array<{
+  key: string;
+  source: string;
+  label: string;
+}> = [
+  { key: "rate", source: "http_reqs", label: "Requests / sec" },
+  { key: "rate", source: "iterations", label: "Iterations / sec" },
+  { key: "rate", source: "http_req_failed", label: "Failure rate" },
+];
+
+export function normalizeK6Summary(
+  input: NormalizeSummaryInput,
+): PerfModuleRunReport {
+  const metrics = input.summary.metrics ?? {};
+  const thresholds = flattenThresholds(metrics);
+  const checks = normalizeChecks(input.summary.root_group?.checks);
+  const scenarios = normalizeScenarios(input.summary.options?.scenarios);
+  const customMetrics = normalizeCustomMetrics(metrics);
+
+  return {
+    module: input.module,
+    status: thresholds.some((threshold) => !threshold.passed)
+      ? "failed"
+      : "passed",
+    startedAt: input.startedAt,
+    finishedAt: input.finishedAt,
+    thresholds,
+    coreMetrics: selectReportValues(metrics, CORE_METRIC_DEFS),
+    latencyMetrics: selectReportValues(metrics, LATENCY_METRIC_DEFS, "ms"),
+    throughputMetrics: selectReportValues(metrics, THROUGHPUT_METRIC_DEFS),
+    scenarios,
+    checks,
+    customMetrics,
+    rawArtifacts: [
+      {
+        kind: "k6-summary",
+        path: input.artifactPath,
+      },
+    ],
+  };
+}
+
+export function buildPerfRunReport(input: BuildRunReportInput): PerfRunReport {
+  const modules = input.modules.map((moduleInput) =>
+    normalizeK6Summary(moduleInput),
+  );
+
+  return {
+    pillar: "performance",
+    environment: input.environment,
+    status: modules.some((module) => module.status === "failed")
+      ? "failed"
+      : "passed",
+    startedAt: input.startedAt,
+    finishedAt: input.finishedAt,
+    modules,
+    rawArtifacts: modules.flatMap((module) => module.rawArtifacts),
+  };
+}
+
+function flattenThresholds(
+  metrics: Record<string, K6Metric>,
+): PerfThresholdResult[] {
+  return Object.entries(metrics).flatMap(([metricKey, metric]) => {
+    const thresholdEntries = Object.entries(metric.thresholds ?? {});
+    return thresholdEntries.map(([expression, passed]) => {
+      const actualValue = deriveThresholdActualValue(metric, expression);
+      return {
+        metric: metricKey,
+        name: `${metricKey}:${expression}`,
+        expression,
+        passed: normalizeThresholdPassed(actualValue, expression, passed),
+        actualValue,
+        unit: metric.contains === "time" ? "ms" : undefined,
+      };
+    });
+  });
+}
+
+function deriveThresholdActualValue(
+  metric: K6Metric,
+  expression: string,
+): number | string | null {
+  const values = getMetricValues(metric);
+  const percentile = expression.match(/^p\(([\d.]+)\)/);
+  if (percentile) {
+    return values[`p(${percentile[1]})`] ?? null;
+  }
+
+  if (expression.startsWith("rate")) {
+    return firstDefinedNumber(values.rate, values.value) ?? null;
+  }
+
+  if (expression.startsWith("avg")) {
+    return values.avg ?? null;
+  }
+
+  if (expression.startsWith("count")) {
+    return values.count ?? null;
+  }
+
+  return values.value ?? values.avg ?? values.rate ?? values.count ?? null;
+}
+
+function selectReportValues(
+  metrics: Record<string, K6Metric>,
+  definitions: Array<{ key: string; source: string; label: string }>,
+  unit?: string,
+): PerfReportValue[] {
+  return definitions.flatMap((definition) => {
+    const value = getMetricDisplayValue(
+      metrics[definition.source],
+      definition.key,
+    );
+    if (typeof value !== "number") {
+      return [];
+    }
+
+    return [
+      {
+        label: definition.label,
+        value,
+        unit,
+      },
+    ];
+  });
+}
+
+function normalizeChecks(
+  checks?: K6Check[] | Record<string, Omit<K6Check, "name">>,
+): PerfCheckResult[] {
+  const normalizedChecks = Array.isArray(checks)
+    ? checks
+    : Object.entries(checks ?? {}).map(([name, check]) => ({
+        ...check,
+        name,
+      }));
+
+  return normalizedChecks
+    .filter(
+      (check): check is Required<Pick<K6Check, "name">> & K6Check =>
+        typeof check.name === "string",
+    )
+    .map((check) => {
+      const passes = check.passes ?? 0;
+      const fails = check.fails ?? 0;
+      const total = passes + fails;
+      return {
+        name: check.name,
+        passes,
+        fails,
+        passRate: total > 0 ? passes / total : null,
+      };
+    });
+}
+
+function normalizeScenarios(
+  scenarios?: Record<string, Record<string, unknown>>,
+): PerfScenarioSummary[] {
+  return Object.entries(scenarios ?? {}).map(([name, scenario]) => ({
+    name,
+    executor:
+      typeof scenario.executor === "string" ? scenario.executor : undefined,
+    startTime:
+      typeof scenario.startTime === "string" ? scenario.startTime : undefined,
+    gracefulStop:
+      typeof scenario.gracefulStop === "string"
+        ? scenario.gracefulStop
+        : undefined,
+    options: scenario,
+  }));
+}
+
+function normalizeCustomMetrics(
+  metrics: Record<string, K6Metric>,
+): PerfMetricSummary[] {
+  const builtInMetrics = new Set([
+    "http_reqs",
+    "http_req_failed",
+    "http_req_duration",
+    "http_req_blocked",
+    "http_req_connecting",
+    "http_req_receiving",
+    "http_req_sending",
+    "http_req_tls_handshaking",
+    "http_req_waiting",
+    "iteration_duration",
+    "iterations",
+    "vus",
+    "vus_max",
+    "checks",
+    "data_received",
+    "data_sent",
+    "group_duration",
+  ]);
+
+  return Object.entries(metrics)
+    .filter(([metricKey]) => {
+      const baseKey = metricKey.replace(/\{.*\}$/, "");
+      return !builtInMetrics.has(baseKey);
+    })
+    .map(([metricKey, metric]) => ({
+      key: metricKey,
+      label: metricKey.replace(/_/g, " "),
+      type: metric.type,
+      contains: metric.contains,
+      values: normalizeMetricValues(getMetricValues(metric)),
+      thresholds: flattenThresholds({ [metricKey]: metric }),
+      isCustom: true,
+    }));
+}
+
+function normalizeMetricValues(
+  values?: Record<string, number>,
+): Record<string, number | null> {
+  return Object.fromEntries(
+    Object.entries(values ?? {}).map(([key, value]) => [
+      key,
+      typeof value === "number" ? value : null,
+    ]),
+  );
+}
+
+function getMetricValues(metric?: K6Metric): Record<string, number> {
+  if (!metric) {
+    return {};
+  }
+
+  if (metric.values && Object.keys(metric.values).length > 0) {
+    return normalizeNumericEntries(metric.values);
+  }
+
+  const topLevelEntries = Object.entries(metric).filter(
+    (entry): entry is [string, number] => {
+      const [key, value] = entry;
+      if (key === "type" || key === "contains" || key === "thresholds") {
+        return false;
+      }
+
+      return typeof value === "number";
+    },
+  );
+
+  return Object.fromEntries(topLevelEntries);
+}
+
+function getMetricDisplayValue(
+  metric: K6Metric | undefined,
+  preferredKey: string,
+): number | undefined {
+  const values = getMetricValues(metric);
+
+  if (metric?.type === "gauge" && preferredKey === "value") {
+    return firstDefinedNumber(values.max, values.value, values.min);
+  }
+
+  if (preferredKey === "rate") {
+    return firstDefinedNumber(values.rate, values.value);
+  }
+
+  return values[preferredKey];
+}
+
+function firstDefinedNumber(
+  ...candidates: Array<number | undefined>
+): number | undefined {
+  return candidates.find((candidate) => typeof candidate === "number");
+}
+
+function normalizeNumericEntries(
+  values: Record<string, number>,
+): Record<string, number> {
+  return Object.fromEntries(
+    Object.entries(values).filter(
+      (entry): entry is [string, number] => typeof entry[1] === "number",
+    ),
+  );
+}
+
+function normalizeThresholdPassed(
+  actualValue: number | string | null,
+  expression: string,
+  value: boolean | { ok?: boolean },
+): boolean {
+  const evaluated = evaluateThresholdExpression(actualValue, expression);
+  if (typeof evaluated === "boolean") {
+    return evaluated;
+  }
+
+  if (typeof value === "boolean") {
+    return value;
+  }
+
+  return value.ok ?? false;
+}
+
+function evaluateThresholdExpression(
+  actualValue: number | string | null,
+  expression: string,
+): boolean | undefined {
+  if (typeof actualValue !== "number") {
+    return undefined;
+  }
+
+  const match = expression.match(
+    /^[a-zA-Z0-9_.()]+\s*(<=|>=|<|>|==)\s*(-?\d+(?:\.\d+)?)$/,
+  );
+  if (!match) {
+    return undefined;
+  }
+
+  const [, operator, targetRaw] = match;
+  const target = Number(targetRaw);
+  if (!Number.isFinite(target)) {
+    return undefined;
+  }
+
+  switch (operator) {
+    case "<":
+      return actualValue < target;
+    case "<=":
+      return actualValue <= target;
+    case ">":
+      return actualValue > target;
+    case ">=":
+      return actualValue >= target;
+    case "==":
+      return actualValue === target;
+    default:
+      return undefined;
+  }
+}

package/src/shared/perf/report-paths.ts

@@ -0,0 +1,19 @@
+import * as path from "path";
+
+export const PERF_REPORT_ROOT = ".perf-report";
+export const PERF_CURRENT_REPORT_DIR = path.join(PERF_REPORT_ROOT, "current");
+export const PERF_CURRENT_REPORT_FILE = path.join(
+  PERF_CURRENT_REPORT_DIR,
+  "report.json",
+);
+
+export function getPerfSummaryArtifactName(moduleName: string): string {
+  return `${moduleName.replace(/\//g, "-")}.summary.json`;
+}
+
+export function getPerfSummaryArtifactPath(moduleName: string): string {
+  return path.join(
+    PERF_CURRENT_REPORT_DIR,
+    getPerfSummaryArtifactName(moduleName),
+  );
+}

package/src/shared/perf/report-store.ts

@@ -0,0 +1,110 @@
+import * as fs from "fs";
+import * as path from "path";
+import type { PerfRunReport } from "./report-types.js";
+import { getDistRoot } from "../paths.js";
+import { buildPerfRunReport, type K6Summary } from "./report-normalizer.js";
+import {
+  PERF_CURRENT_REPORT_DIR,
+  PERF_CURRENT_REPORT_FILE,
+  getPerfSummaryArtifactPath,
+} from "./report-paths.js";
+
+interface WriteCurrentPerfRunReportInput {
+  rootDir?: string;
+  environment: string;
+  modules: string[];
+  startedAt?: string;
+  finishedAt?: string;
+}
+
+export function writeCurrentPerfRunReport(
+  input: WriteCurrentPerfRunReportInput,
+): PerfRunReport {
+  // Default anchored to the den root, NOT process.cwd(): this WRITES
+  // .perf-report/, and perf children spawned by the CLI inherit an arbitrary
+  // user cwd (bin/den never cds).
+  const rootDir = input.rootDir ?? getDistRoot();
+  const report = buildPerfRunReport({
+    environment: input.environment,
+    modules: input.modules.map((moduleName) => ({
+      module: moduleName,
+      summary: readSummaryArtifact(rootDir, moduleName),
+      artifactPath: getPerfSummaryArtifactPath(moduleName),
+      startedAt: input.startedAt,
+      finishedAt: input.finishedAt,
+    })),
+    startedAt: input.startedAt,
+    finishedAt: input.finishedAt,
+  });
+
+  fs.mkdirSync(path.join(rootDir, PERF_CURRENT_REPORT_DIR), {
+    recursive: true,
+  });
+  fs.writeFileSync(
+    path.join(rootDir, PERF_CURRENT_REPORT_FILE),
+    JSON.stringify(report, null, 2),
+    "utf-8",
+  );
+
+  return report;
+}
+
+export function loadCurrentPerfRunReport(
+  rootDir = getDistRoot(),
+): PerfRunReport | null {
+  const reportPath = path.join(rootDir, PERF_CURRENT_REPORT_FILE);
+  if (!fs.existsSync(reportPath)) {
+    return null;
+  }
+
+  // A truncated/corrupt report.json must not crash the request handler — treat
+  // an unparseable report as "no report yet".
+  try {
+    return JSON.parse(fs.readFileSync(reportPath, "utf-8")) as PerfRunReport;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Remove the previous run's normalized report + raw summary artifacts at the
+ * START of a run.
+ *
+ * `.perf-report/current/` is reused across runs, so without this the prior
+ * run's `report.json` (and per-module `*.summary.json`) linger. If the report
+ * is read before the current run regenerates it — or a threshold-breaching run
+ * never regenerates it — `loadCurrentPerfRunReport` silently serves STALE prior
+ * results (the back-to-back staleness bug). Clearing here makes such a read
+ * return `null` ("no report yet", loud) instead of stale data.
+ */
+export function resetCurrentPerfRunReport(rootDir = getDistRoot()): void {
+  try {
+    fs.rmSync(path.join(rootDir, PERF_CURRENT_REPORT_FILE), { force: true });
+  } catch {
+    /* nothing to remove */
+  }
+  try {
+    const dir = path.join(rootDir, PERF_CURRENT_REPORT_DIR);
+    for (const entry of fs.readdirSync(dir)) {
+      if (entry.endsWith(".summary.json")) {
+        fs.rmSync(path.join(dir, entry), { force: true });
+      }
+    }
+  } catch {
+    /* dir may not exist yet */
+  }
+}
+
+function readSummaryArtifact(rootDir: string, moduleName: string): K6Summary {
+  const artifactPath = path.join(
+    rootDir,
+    getPerfSummaryArtifactPath(moduleName),
+  );
+  if (!fs.existsSync(artifactPath)) {
+    throw new Error(`Missing k6 summary artifact for module "${moduleName}"`);
+  }
+
+  // JSON boundary: k6 wrote this artifact, so the shape is trusted here and
+  // every field access in the normalizer is already optional/defensive.
+  return JSON.parse(fs.readFileSync(artifactPath, "utf-8")) as K6Summary;
+}

package/src/shared/perf/report-types.ts

@@ -0,0 +1,70 @@
+export interface PerfArtifactRef {
+  kind: "k6-summary" | "normalized-report";
+  path: string;
+}
+
+export interface PerfReportValue {
+  label: string;
+  value: number | string | null;
+  unit?: string;
+}
+
+export interface PerfThresholdResult {
+  metric: string;
+  name: string;
+  expression: string;
+  passed: boolean;
+  actualValue: number | string | null;
+  unit?: string;
+}
+
+export interface PerfScenarioSummary {
+  name: string;
+  executor?: string;
+  startTime?: string;
+  gracefulStop?: string;
+  env?: Record<string, string>;
+  options?: Record<string, unknown>;
+}
+
+export interface PerfCheckResult {
+  name: string;
+  passes: number;
+  fails: number;
+  passRate: number | null;
+}
+
+export interface PerfMetricSummary {
+  key: string;
+  label: string;
+  type?: string;
+  contains?: string;
+  values: Record<string, number | null>;
+  thresholds: PerfThresholdResult[];
+  isCustom?: boolean;
+}
+
+export interface PerfModuleRunReport {
+  module: string;
+  status: "passed" | "failed";
+  startedAt?: string;
+  finishedAt?: string;
+  thresholds: PerfThresholdResult[];
+  coreMetrics: PerfReportValue[];
+  latencyMetrics: PerfReportValue[];
+  throughputMetrics: PerfReportValue[];
+  scenarios: PerfScenarioSummary[];
+  checks: PerfCheckResult[];
+  customMetrics: PerfMetricSummary[];
+  rawArtifacts: PerfArtifactRef[];
+}
+
+export interface PerfRunReport {
+  pillar: "performance";
+  environment: string;
+  status: "passed" | "failed";
+  startedAt?: string;
+  finishedAt?: string;
+  modules: PerfModuleRunReport[];
+  rawArtifacts: PerfArtifactRef[];
+}

package/src/shared/perf/test-paths.ts

@@ -0,0 +1,25 @@
+import * as path from "path";
+import { getDistRoot } from "../paths.js";
+
+/**
+ * Where a distribution keeps its own k6 suites, relative to the distribution
+ * root: `<dist root>/tests/pillars/perf/<module>/test.js`. The engine ships
+ * none; repo-owned suites are declared in each repo's manifest instead.
+ */
+export const PERF_DIST_TESTS_DIR = path.join("tests", "pillars", "perf");
+
+/** The k6 entry file every distribution perf module provides. */
+export const PERF_TEST_ENTRY = "test.js";
+
+/** Absolute path of the distribution's perf suites directory. */
+export function getPerfTestsRoot(root: string = getDistRoot()): string {
+  return path.resolve(root, PERF_DIST_TESTS_DIR);
+}
+
+/** Absolute path of a distribution perf module's k6 entry file. */
+export function getPerfTestFile(
+  moduleName: string,
+  root: string = getDistRoot(),
+): string {
+  return path.join(getPerfTestsRoot(root), moduleName, PERF_TEST_ENTRY);
+}