@elliemae/encw-leak-runner 1.0.2

package/lib/reporting/tests/junitReporter.test.ts

@@ -0,0 +1,103 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { JunitReporter } from '../junitReporter.js';
+import type { RunSummary } from '../../types/results.js';
+
+function makeSummary(): RunSummary {
+  return {
+    totalDurationMs: 10000,
+    passCount: 1,
+    failCount: 1,
+    results: [
+      {
+        name: 'loan-pipeline',
+        passed: true,
+        thresholdResult: { passed: true, reason: null },
+        report: null,
+        durationMs: 4000,
+        error: null,
+      },
+      {
+        name: 'document-center',
+        passed: false,
+        thresholdResult: {
+          passed: false,
+          reason: 'Retained size delta 18.4 MB exceeds threshold 10.0 MB',
+        },
+        report: null,
+        durationMs: 6000,
+        error: null,
+      },
+    ],
+  };
+}
+
+describe('JunitReporter', () => {
+  let tmpDir: string;
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'junit-test-'));
+  });
+
+  afterEach(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  it('writes test-results.xml to outputDir', async () => {
+    const reporter = new JunitReporter();
+    await reporter.write(makeSummary(), tmpDir);
+    expect(fs.existsSync(path.join(tmpDir, 'test-results.xml'))).toBe(true);
+  });
+
+  it('XML contains one testcase per scenario', async () => {
+    const reporter = new JunitReporter();
+    await reporter.write(makeSummary(), tmpDir);
+    const xml = fs.readFileSync(path.join(tmpDir, 'test-results.xml'), 'utf8');
+    expect(xml).toContain('name="loan-pipeline"');
+    expect(xml).toContain('name="document-center"');
+  });
+
+  it('XML contains a failure element for failing scenarios', async () => {
+    const reporter = new JunitReporter();
+    await reporter.write(makeSummary(), tmpDir);
+    const xml = fs.readFileSync(path.join(tmpDir, 'test-results.xml'), 'utf8');
+    expect(xml).toContain('<failure');
+    expect(xml).toContain('18.4 MB');
+  });
+
+  it('testsuite has correct tests and failures count', async () => {
+    const reporter = new JunitReporter();
+    await reporter.write(makeSummary(), tmpDir);
+    const xml = fs.readFileSync(path.join(tmpDir, 'test-results.xml'), 'utf8');
+    expect(xml).toContain('tests="2"');
+    expect(xml).toContain('failures="1"');
+  });
+
+  it('XML content is properly escaped for special characters', async () => {
+    const reporter = new JunitReporter();
+    const summary: RunSummary = {
+      ...makeSummary(),
+      results: [
+        {
+          name: 'test-<scenario>',
+          passed: false,
+          thresholdResult: {
+            passed: false,
+            reason: 'Error: <leak> & "memory"',
+          },
+          report: null,
+          durationMs: 1000,
+          error: null,
+        },
+      ],
+      failCount: 1,
+      passCount: 0,
+    };
+    await reporter.write(summary, tmpDir);
+    const xml = fs.readFileSync(path.join(tmpDir, 'test-results.xml'), 'utf8');
+    expect(xml).toContain('&lt;scenario&gt;');
+    expect(xml).toContain('&amp;');
+    expect(xml).toContain('&quot;memory&quot;');
+  });
+});

package/lib/runner/aiEnhancementStep.ts

@@ -0,0 +1,39 @@
+import {
+  AiEnhancer,
+  renderComparisonMarkdown,
+  type ComparisonReport,
+  type LeakResult,
+} from '@elliemae/encw-heap-doctor';
+import type { RunnerConfig } from '../types/config.js';
+
+/**
+ * If the config carries an AI config, run the AiEnhancer over the report's leak
+ * results and return freshly-rendered markdown. If the AI call fails, log a
+ * single-line warning to stderr and return the report's original markdown so
+ * the scenario result is unaffected.
+ *
+ * If the config carries no AI config, return the report's original markdown
+ * untouched (no AI module loaded).
+ * @param report
+ * @param config
+ * @param scenarioName
+ */
+export async function enhanceMarkdownIfConfigured(
+  report: ComparisonReport,
+  config: RunnerConfig,
+  scenarioName: string,
+): Promise<string> {
+  if (!config.ai?.apiKey) return report.markdown;
+
+  try {
+    const enhancer = new AiEnhancer(config.ai);
+    await enhancer.enhance(report.afterAnalysis.leakResults as LeakResult[]);
+    return renderComparisonMarkdown(report);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    process.stderr.write(
+      `Genice enhancement failed for ${scenarioName}: ${message}; writing report without AI section.\n`,
+    );
+    return report.markdown;
+  }
+}

package/lib/runner/batchRunner.ts

@@ -0,0 +1,71 @@
+import type { MicroappLeakScenario } from '../types/scenario.js';
+import type { RunnerConfig } from '../types/config.js';
+import type { ScenarioResult, RunSummary } from '../types/results.js';
+import type { ScenarioRegistry } from '../registry/scenarioRegistry.js';
+import { ScenarioRunner } from './scenarioRunner.js';
+
+function buildSummary(results: ScenarioResult[]): RunSummary {
+  return {
+    results,
+    totalDurationMs: results.reduce((acc, r) => acc + r.durationMs, 0),
+    passCount: results.filter((r) => r.passed).length,
+    failCount: results.filter((r) => !r.passed).length,
+  };
+}
+
+export class BatchRunner {
+  private readonly scenarioRunner: ScenarioRunner;
+
+  constructor(
+    private readonly config: RunnerConfig,
+    private readonly registry: ScenarioRegistry,
+  ) {
+    this.scenarioRunner = new ScenarioRunner(config);
+  }
+
+  async runAll(): Promise<RunSummary> {
+    const scenarios = this.registry.list().map((entry) => entry.scenario);
+    return this.executeScenarios(scenarios);
+  }
+
+  async runByTag(tag: string): Promise<RunSummary> {
+    const scenarios = this.registry
+      .filterByTag(tag)
+      .map((entry) => entry.scenario);
+    return this.executeScenarios(scenarios);
+  }
+
+  async runByName(key: string): Promise<RunSummary> {
+    const scenario = this.registry.get(key);
+    if (!scenario) {
+      throw new Error(`No scenario registered with key "${key}"`);
+    }
+    return this.executeScenarios([scenario]);
+  }
+
+  private async executeScenarios(
+    scenarios: readonly MicroappLeakScenario[],
+  ): Promise<RunSummary> {
+    const results = await scenarios.reduce<Promise<ScenarioResult[]>>(
+      async (acc, scenario) => {
+        const prior = await acc;
+        const result = await this.runOne(scenario);
+        return [...prior, result];
+      },
+      Promise.resolve([]),
+    );
+
+    return buildSummary(results);
+  }
+
+  private async runOne(
+    scenario: MicroappLeakScenario,
+  ): Promise<ScenarioResult> {
+    const browser = await this.scenarioRunner.launchBrowser();
+    try {
+      return await this.scenarioRunner.run(scenario, browser);
+    } finally {
+      await browser.close();
+    }
+  }
+}

package/lib/runner/scenarioRunner.ts

@@ -0,0 +1,189 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import {
+  chromium,
+  type Browser,
+  type Page,
+  type Frame,
+} from '@playwright/test';
+import { AuthManager, PageSetup } from '@elliemae/smoked-suite';
+import type { ComparisonReport } from '@elliemae/encw-heap-doctor';
+import { IframeHeapProfiler } from '../browser/iframeHeapProfiler.js';
+import { ThresholdEvaluator } from '../analysis/thresholdEvaluator.js';
+import { DEFAULT_THRESHOLDS } from '../types/config.js';
+import type { RunnerConfig, ResolvedThresholds } from '../types/config.js';
+import type { MicroappLeakScenario } from '../types/scenario.js';
+import type { ScenarioResult } from '../types/results.js';
+import { enhanceMarkdownIfConfigured } from './aiEnhancementStep.js';
+
+async function resolveIframe(page: Page, selector: string): Promise<Frame> {
+  const handle = await page.locator(selector).elementHandle();
+  if (!handle) throw new Error(`Iframe not found for selector: ${selector}`);
+  const frame = await handle.contentFrame();
+  if (!frame)
+    throw new Error(`Could not get content frame for selector: ${selector}`);
+  return frame;
+}
+
+async function forceGarbageCollection(page: Page): Promise<void> {
+  const session = await page.context().newCDPSession(page);
+  await session.send('HeapProfiler.enable');
+  await session.send('HeapProfiler.collectGarbage');
+  await session.send('HeapProfiler.disable');
+  await session.detach();
+  await page.waitForTimeout(2000);
+}
+
+interface SnapshotPaths {
+  before: string;
+  after: string;
+}
+
+function cleanupSnapshots(paths: SnapshotPaths): void {
+  if (paths.before && fs.existsSync(paths.before)) fs.unlinkSync(paths.before);
+  if (paths.after && fs.existsSync(paths.after)) fs.unlinkSync(paths.after);
+}
+
+export class ScenarioRunner {
+  constructor(private readonly config: RunnerConfig) {}
+
+  async launchBrowser(): Promise<Browser> {
+    return chromium.launch({ headless: this.config.runner.headless });
+  }
+
+  async run(
+    scenario: MicroappLeakScenario,
+    browser: Browser,
+  ): Promise<ScenarioResult> {
+    const startTime = Date.now();
+    const context = await browser.newContext({
+      baseURL: this.config.env.baseUrl,
+    });
+    const page = await context.newPage();
+    const paths: SnapshotPaths = { before: '', after: '' };
+    let report: ComparisonReport | null = null;
+    let error: Error | null = null;
+
+    try {
+      report = await this.runScenarioInPage(page, scenario, paths);
+    } catch (err) {
+      error = err instanceof Error ? err : new Error(String(err));
+    } finally {
+      await context.close();
+      cleanupSnapshots(paths);
+    }
+
+    return this.buildResult({ scenario, report, error, startTime });
+  }
+
+  private async runScenarioInPage(
+    page: Page,
+    scenario: MicroappLeakScenario,
+    paths: SnapshotPaths,
+  ): Promise<ComparisonReport | null> {
+    const snapshotsDir = path.join(this.config.runner.outputDir, 'snapshots');
+    const { profiler, frame } = await this.setupAndProfile(
+      page,
+      scenario,
+      snapshotsDir,
+    );
+
+    paths.before = await profiler.captureSnapshot('before');
+    await this.repeatScenarioActions(scenario, page, frame);
+    await forceGarbageCollection(page);
+    paths.after = await profiler.captureSnapshot('after');
+
+    const report = await profiler.compare(
+      'before',
+      'after',
+      this.config.runner.topN,
+    );
+    if (report) await this.writeReport(report, scenario);
+    return report;
+  }
+
+  private async setupAndProfile(
+    page: Page,
+    scenario: MicroappLeakScenario,
+    snapshotsDir: string,
+  ): Promise<{ profiler: IframeHeapProfiler; frame: Frame }> {
+    const auth = new AuthManager();
+    const pageSetup = new PageSetup();
+
+    await pageSetup.apply(page);
+    await auth.login(page, {
+      username: this.config.env.userId,
+      password: this.config.env.password,
+      instanceId: this.config.env.instanceId,
+    });
+    await page.goto(scenario.url());
+
+    const frame = await resolveIframe(page, scenario.microappSelector);
+    fs.mkdirSync(snapshotsDir, { recursive: true });
+    const profiler = new IframeHeapProfiler(page, frame, snapshotsDir);
+    return { profiler, frame };
+  }
+
+  private async repeatScenarioActions(
+    scenario: MicroappLeakScenario,
+    page: Page,
+    frame: Frame,
+  ): Promise<void> {
+    const repeatCount = scenario.repeat?.() ?? 3;
+    for (let i = 0; i < repeatCount; i += 1) {
+      // eslint-disable-next-line no-await-in-loop
+      await scenario.action(page, frame);
+      if (scenario.back) {
+        // eslint-disable-next-line no-await-in-loop
+        await scenario.back(page);
+      }
+    }
+  }
+
+  private async writeReport(
+    report: ComparisonReport,
+    scenario: MicroappLeakScenario,
+  ): Promise<void> {
+    const reportPath = path.join(
+      this.config.runner.outputDir,
+      `${scenario.id}.md`,
+    );
+    fs.mkdirSync(path.dirname(reportPath), { recursive: true });
+    const markdown = await enhanceMarkdownIfConfigured(
+      report,
+      this.config,
+      scenario.name,
+    );
+    fs.writeFileSync(reportPath, markdown);
+  }
+
+  private buildResult(input: {
+    scenario: MicroappLeakScenario;
+    report: ComparisonReport | null;
+    error: Error | null;
+    startTime: number;
+  }): ScenarioResult {
+    const { scenario, report, error, startTime } = input;
+    const thresholds: ResolvedThresholds = {
+      ...DEFAULT_THRESHOLDS,
+      ...scenario.thresholds,
+    };
+    const failureFallback = {
+      passed: false,
+      reason: error?.message ?? 'Snapshot capture failed',
+    };
+    const thresholdResult =
+      report && !error
+        ? ThresholdEvaluator.evaluate(report, thresholds)
+        : failureFallback;
+
+    return {
+      name: scenario.name,
+      passed: thresholdResult.passed,
+      thresholdResult,
+      report,
+      durationMs: Date.now() - startTime,
+      error,
+    };
+  }
+}

package/lib/runner/tests/aiEnhancementStep.test.ts

@@ -0,0 +1,174 @@
+import type { ComparisonReport } from '@elliemae/encw-heap-doctor';
+import { enhanceMarkdownIfConfigured } from '../aiEnhancementStep.js';
+import type { RunnerConfig } from '../../types/config.js';
+
+const mockEnhance = jest.fn<Promise<void>, [unknown]>();
+const mockRender = jest.fn<string, [unknown]>();
+const mockAiEnhancerCtor = jest.fn<{ enhance: typeof mockEnhance }, [unknown]>(
+  () => ({ enhance: mockEnhance }),
+);
+
+jest.mock('@elliemae/encw-heap-doctor', () => ({
+  AiEnhancer: jest.fn((config: unknown) => mockAiEnhancerCtor(config)),
+  renderComparisonMarkdown: (report: unknown): string => mockRender(report),
+}));
+
+function makeReport(): ComparisonReport {
+  return {
+    beforePath: '/tmp/before.heapsnapshot',
+    afterPath: '/tmp/after.heapsnapshot',
+    beforeNodeCount: 100,
+    afterNodeCount: 200,
+    delta: {
+      newNodeCount: 100,
+      removedNodeCount: 0,
+      retainedSizeDelta: 0,
+      newLeakGroups: [],
+      detachedDomDelta: new Map(),
+    },
+    afterAnalysis: {
+      filePath: '/tmp/after.heapsnapshot',
+      nodeCount: 200,
+      edgeCount: 400,
+      leakResults: [],
+      fixPriority: [],
+      markdown: 'after-md',
+    },
+    markdown: 'ORIGINAL-MARKDOWN',
+  };
+}
+
+function configWithAi(
+  aiOverrides: Partial<RunnerConfig['ai']> = {},
+): RunnerConfig {
+  return {
+    env: {
+      baseUrl: 'https://q3.elliemae.io',
+      instanceId: 'BE11226875',
+      userId: 'admin',
+      password: 'pw',
+    },
+    runner: { headless: true, outputDir: '/tmp', topN: 5 },
+    ai: {
+      model: 'Claude3.7',
+      temperature: 0.3,
+      apiKey: 'sk-abc',
+      ...aiOverrides,
+    },
+  };
+}
+
+function configWithoutAi(): RunnerConfig {
+  return {
+    env: {
+      baseUrl: 'https://q3.elliemae.io',
+      instanceId: 'BE11226875',
+      userId: 'admin',
+      password: 'pw',
+    },
+    runner: { headless: true, outputDir: '/tmp', topN: 5 },
+  };
+}
+
+describe('enhanceMarkdownIfConfigured', () => {
+  type StdSpy = jest.SpyInstance<boolean, [string | Uint8Array, ...unknown[]]>;
+  let stderrSpy: StdSpy;
+
+  beforeEach(() => {
+    mockEnhance.mockReset();
+    mockRender.mockReset();
+    mockAiEnhancerCtor.mockClear();
+    stderrSpy = jest
+      .spyOn(process.stderr, 'write')
+      .mockImplementation(() => true) as StdSpy;
+  });
+
+  afterEach(() => {
+    stderrSpy.mockRestore();
+  });
+
+  it('returns report.markdown unchanged when config.ai is undefined', async () => {
+    const report = makeReport();
+    const md = await enhanceMarkdownIfConfigured(
+      report,
+      configWithoutAi(),
+      'scn',
+    );
+    expect(md).toBe('ORIGINAL-MARKDOWN');
+    expect(mockAiEnhancerCtor).not.toHaveBeenCalled();
+    expect(mockEnhance).not.toHaveBeenCalled();
+    expect(mockRender).not.toHaveBeenCalled();
+  });
+
+  it('passes config.ai through to the AiEnhancer constructor when ai is configured', async () => {
+    mockEnhance.mockResolvedValue(undefined);
+    mockRender.mockReturnValue('REGENERATED');
+    await enhanceMarkdownIfConfigured(
+      makeReport(),
+      configWithAi({ apiKey: 'sk-x', model: 'GPT4', temperature: 0.7 }),
+      'scn',
+    );
+    expect(mockAiEnhancerCtor).toHaveBeenCalledWith({
+      apiKey: 'sk-x',
+      model: 'GPT4',
+      temperature: 0.7,
+    });
+  });
+
+  it('calls enhance with afterAnalysis.leakResults when ai is configured', async () => {
+    mockEnhance.mockResolvedValue(undefined);
+    mockRender.mockReturnValue('REGENERATED');
+    const report = makeReport();
+    await enhanceMarkdownIfConfigured(report, configWithAi(), 'scn');
+    expect(mockEnhance).toHaveBeenCalledTimes(1);
+    expect(mockEnhance).toHaveBeenCalledWith(report.afterAnalysis.leakResults);
+  });
+
+  it('returns the re-rendered markdown on AI success', async () => {
+    mockEnhance.mockResolvedValue(undefined);
+    mockRender.mockReturnValue('FRESH-MARKDOWN-WITH-AI');
+    const md = await enhanceMarkdownIfConfigured(
+      makeReport(),
+      configWithAi(),
+      'scn',
+    );
+    expect(md).toBe('FRESH-MARKDOWN-WITH-AI');
+  });
+
+  it('returns the original report.markdown when AI throws', async () => {
+    mockEnhance.mockRejectedValue(new Error('network down'));
+    const md = await enhanceMarkdownIfConfigured(
+      makeReport(),
+      configWithAi(),
+      'export-navigation',
+    );
+    expect(md).toBe('ORIGINAL-MARKDOWN');
+    expect(mockRender).not.toHaveBeenCalled();
+  });
+
+  it('writes a single-line stderr warning naming the scenario when AI throws', async () => {
+    mockEnhance.mockRejectedValue(new Error('boom'));
+    await enhanceMarkdownIfConfigured(
+      makeReport(),
+      configWithAi(),
+      'export-navigation',
+    );
+    expect(stderrSpy).toHaveBeenCalledTimes(1);
+    const message = String(stderrSpy.mock.calls[0]?.[0]);
+    expect(message).toContain('export-navigation');
+    expect(message).toContain('boom');
+    expect(message).toMatch(/\n$/);
+  });
+
+  it('handles non-Error rejections by stringifying them', async () => {
+    mockEnhance.mockRejectedValue('weird-string-rejection');
+    const md = await enhanceMarkdownIfConfigured(
+      makeReport(),
+      configWithAi(),
+      'scn',
+    );
+    expect(md).toBe('ORIGINAL-MARKDOWN');
+    const message = String(stderrSpy.mock.calls[0]?.[0]);
+    expect(message).toContain('weird-string-rejection');
+  });
+});