@quenty/nevermore-cli 4.41.0 → 4.42.0

package/src/utils/testing/parsers/traceback-parser.ts

@@ -0,0 +1,122 @@
+import { OutputHelper } from '@quenty/cli-output-helpers';
+
+export interface AttributedTraceback {
+  /** First line of the error, before the stack. */
+  message: string;
+  /** Roblox instance path of the topmost frame, e.g. "ServerScriptService.Foo.Bar". */
+  source?: string;
+  /** Root of that path, which names the package the frame belongs to. */
+  owner?: string;
+  /** Frames kept for display, topmost first. */
+  frames: string[];
+  /** How many times this same traceback appeared. */
+  count: number;
+}
+
+const STACK_BEGIN = /^\s*Stack Begin\s*$/;
+const STACK_END = /^\s*Stack End\s*$/;
+// Script 'ServerScriptService.EggHunt2026.game.Server.Foo', Line 365 - function bar
+const FRAME = /Script\s+'(?:\[string\s+")?([^'"\]]+)/;
+
+const MAX_FRAMES = 4;
+
+/**
+ * Pull tracebacks out of log output and attribute each to its owning package.
+ *
+ * Attribution reads the script path in the frames rather than the position of
+ * the traceback in the log. Position is unreliable: a deferred callback can
+ * fire during a later package's section, which is why batch mode previously
+ * gave up on tracebacks entirely.
+ */
+export function parseTracebacks(rawOutput: string): AttributedTraceback[] {
+  if (!rawOutput) {
+    return [];
+  }
+
+  const lines = OutputHelper.stripAnsi(rawOutput).split('\n');
+  const bySignature = new Map<string, AttributedTraceback>();
+
+  for (let i = 0; i < lines.length; i++) {
+    if (!STACK_BEGIN.test(lines[i])) {
+      continue;
+    }
+
+    // The error text sits on the line above "Stack Begin".
+    const message = (lines[i - 1] ?? '').trim();
+
+    const frames: string[] = [];
+    let source: string | undefined;
+    for (let j = i + 1; j < lines.length && !STACK_END.test(lines[j]); j++) {
+      const frame = lines[j].trim();
+      if (!frame) {
+        continue;
+      }
+      if (frames.length < MAX_FRAMES) {
+        frames.push(frame);
+      }
+      if (!source) {
+        source = FRAME.exec(frame)?.[1];
+      }
+    }
+
+    // One bug fires per player and per frame, so the same traceback can appear
+    // hundreds of times. Collapse by signature and count instead of printing
+    // every copy.
+    const signature = `${message}::${frames[0] ?? ''}`;
+    const existing = bySignature.get(signature);
+    if (existing) {
+      existing.count++;
+      continue;
+    }
+
+    bySignature.set(signature, {
+      message,
+      source,
+      owner: source ? ownerOf(source) : undefined,
+      frames,
+      count: 1,
+    });
+  }
+
+  return [...bySignature.values()];
+}
+
+/**
+ * Name the package a script path belongs to.
+ *
+ * Batch places hold each package under its own root, so the segment below the
+ * service is the package.
+ */
+export function ownerOf(scriptPath: string): string | undefined {
+  const parts = scriptPath.split('.');
+  return parts.length >= 2 ? parts[1] : undefined;
+}
+
+/** Render tracebacks for a human, newest information first and bounded. */
+export function formatTracebacks(
+  tracebacks: AttributedTraceback[],
+  maxShown = 5
+): string {
+  if (tracebacks.length === 0) {
+    return '';
+  }
+
+  const lines: string[] = [];
+  const shown = tracebacks.slice(0, maxShown);
+
+  for (const traceback of shown) {
+    const where = traceback.source ? ` (${traceback.source})` : '';
+    const times = traceback.count > 1 ? ` ×${traceback.count}` : '';
+    lines.push(`  ${traceback.message}${where}${times}`);
+    for (const frame of traceback.frames) {
+      lines.push(`      ${frame}`);
+    }
+  }
+
+  const hidden = tracebacks.length - shown.length;
+  if (hidden > 0) {
+    lines.push(`  ...and ${hidden} more distinct traceback(s)`);
+  }
+
+  return lines.join('\n');
+}

package/src/utils/testing/reporting/test-github-columns.ts

@@ -19,6 +19,7 @@ export function createTestCommentConfig(): GithubCommentTableConfig {
     sectionId: 'tests',
     extraColumns: createTestColumns(),
     errorHeading: 'Test Results',
+    expectsTestCounts: true,
   };
 }
 

package/src/utils/testing/runner/test-runner.test.ts

@@ -0,0 +1,29 @@
+import { describe, expect, it } from 'vitest';
+
+import { mergeFailureReasons } from './test-runner.js';
+
+describe('mergeFailureReasons', () => {
+  it('does not repeat reasons the context already reported', () => {
+    // Batch mode evaluates the same logs upstream, so both sources describe the
+    // same failures. Concatenating them read as double the failures.
+    const merged = mergeFailureReasons(
+      '3 test suite(s) failed; 19 test(s) failed',
+      ['3 test suite(s) failed', '19 test(s) failed']
+    );
+
+    expect(merged).toEqual(['3 test suite(s) failed', '19 test(s) failed']);
+  });
+
+  it('keeps reasons only one side knows about', () => {
+    const merged = mergeFailureReasons(
+      'the batch runner reported a Luau error',
+      ['19 test(s) failed']
+    );
+
+    expect(merged).toHaveLength(2);
+  });
+
+  it('is empty when nothing failed', () => {
+    expect(mergeFailureReasons(undefined, [])).toEqual([]);
+  });
+});

package/src/utils/testing/runner/test-runner.ts

@@ -26,6 +26,27 @@ export interface SingleTestResult {
    * non-aggregated cloud/local runs — callers should fall back to wall-clock.
    */
   durationMs?: number;
+  /** Why the run failed, when the runner can say more than "it failed". */
+  error?: string;
+}
+
+/**
+ * Combine failure reasons from the job context with those read from the logs.
+ *
+ * In batch mode the context has already evaluated these same logs, so the two
+ * sources overlap. Merging on the individual reason keeps the overlap from
+ * reading as double the failures.
+ */
+export function mergeFailureReasons(
+  contextError: string | undefined,
+  logReasons: string[]
+): string[] {
+  return [
+    ...new Set([
+      ...(contextError ? contextError.split('; ') : []),
+      ...logReasons,
+    ]),
+  ];
 }
 
 export interface SingleTestOptions {
@@ -54,7 +75,10 @@ export async function runSingleTestAsync(
   const {
     packagePath,
     packageName,
-    timeoutMs = 120_000,
+    // 300s is the Open Cloud maximum. The old 120s default failed real suites
+    // with no jest summary at all, so "--timeout 300" had become folklore
+    // everyone had to know before their first useful run.
+    timeoutMs = 300_000,
     scriptText,
     target,
   } = options;
@@ -102,13 +126,24 @@ export async function runSingleTestAsync(
     });
 
     const rawLogs = await context.getLogsAsync(deployment);
-    const parsed = parseTestLogs(rawLogs);
+    // A probe script is arbitrary Luau with no jest in it, so demanding a test
+    // report would fail every --script-text run. Everything else must prove a
+    // runner spoke before it can pass.
+    const parsed = parseTestLogs(rawLogs, {
+      requireTestReport: scriptText === undefined,
+    });
+
+    const reasons = mergeFailureReasons(
+      result.errorMessage,
+      parsed.failureReasons
+    );
 
     return {
       success: result.success && parsed.success,
       logs: parsed.logs,
       testCounts: parseTestCounts(parsed.logs),
       durationMs: result.durationMs,
+      error: reasons.length > 0 ? reasons.join('; ') : undefined,
     };
   } finally {
     await context.releaseAsync(deployment);

package/src/utils/testing/test-log-parser.test.ts

@@ -0,0 +1,70 @@
+import { describe, expect, it } from 'vitest';
+
+import { evaluateTestOutcome, hasTestReport } from './test-log-parser.js';
+
+describe('evaluateTestOutcome', () => {
+  it('passes a clean jest report', () => {
+    const outcome = evaluateTestOutcome('Tests: 275 passed, 275 total', {
+      requireTestReport: true,
+    });
+
+    expect(outcome.success).toBe(true);
+    expect(outcome.failureReasons).toEqual([]);
+  });
+
+  it('refuses to pass output with no jest report in it', () => {
+    // Absence of failure was previously read as success, so an empty log passed.
+    const outcome = evaluateTestOutcome('some unrelated chatter', {
+      requireTestReport: true,
+    });
+
+    expect(outcome.success).toBe(false);
+    expect(outcome.failureReasons.join()).toContain('no jest report');
+  });
+
+  it('exempts probe runs, which have no jest in them at all', () => {
+    const outcome = evaluateTestOutcome('hello from a probe script');
+
+    expect(outcome.success).toBe(true);
+  });
+
+  it('fails on a traceback even when every test passed', () => {
+    // Jest cannot see a deferred-callback crash: it fires outside any test.
+    const logs = [
+      'Tests: 155 passed, 155 total',
+      'attempt to index nil',
+      'Stack Begin',
+      "Script 'ServerScriptService.egghunt2026.Foo', Line 12",
+      'Stack End',
+    ].join('\n');
+
+    const outcome = evaluateTestOutcome(logs, { requireTestReport: true });
+
+    expect(outcome.success).toBe(false);
+    expect(outcome.failureReasons.join()).toContain('traceback');
+  });
+
+  it('reports every reason a run failed, not just the first', () => {
+    const logs = [
+      'Tests: 19 failed, 470 passed, 516 total',
+      'Stack Begin',
+      "Script 'ServerScriptService.egghunt2026.Foo', Line 12",
+      'Stack End',
+    ].join('\n');
+
+    const outcome = evaluateTestOutcome(logs, { requireTestReport: true });
+
+    expect(outcome.failureReasons).toHaveLength(2);
+  });
+});
+
+describe('hasTestReport', () => {
+  it('accepts a runner that ran and found nothing', () => {
+    // Packages without specs are legitimate; passWithNoTests depends on this.
+    expect(hasTestReport('No tests found, exiting with code 0')).toBe(true);
+  });
+
+  it('rejects silence', () => {
+    expect(hasTestReport('')).toBe(false);
+  });
+});

package/src/utils/testing/test-log-parser.ts

@@ -10,27 +10,97 @@ export interface ParsedTestLogs {
   success: boolean;
   logs: string;
   testCounts?: ParsedTestCounts;
+  /** Why the run is not a pass. Empty when it is. */
+  failureReasons: string[];
+}
+
+export interface TestOutcomeOptions {
+  /**
+   * Require proof the runner ran. Off for probe scripts (--script-text), which
+   * legitimately produce no jest report.
+   */
+  requireTestReport?: boolean;
 }
 
 /**
- * Analyze test output for Jest failures and Luau runtime errors.
- * Shared by both Open Cloud log fetching and local run-in-roblox output.
+ * Decide whether test output represents a pass, and say why when it does not.
+ *
+ * The single definition of the rule. Both the single-package path and the batch
+ * path route through here — they previously carried divergent copies that
+ * disagreed about tracebacks, so the same commit could pass one and fail the
+ * other.
  */
-export function parseTestLogs(rawOutput: string): ParsedTestLogs {
+export function evaluateTestOutcome(
+  rawOutput: string,
+  options: TestOutcomeOptions = {}
+): { success: boolean; failureReasons: string[] } {
   const cleanLogs = OutputHelper.stripAnsi(rawOutput);
+  const failureReasons: string[] = [];
 
-  // Check for Jest-style test failures
   const failedSuites = cleanLogs.match(/Test Suites:\s*(\d+)\s+failed/);
   const failedTests = cleanLogs.match(/Tests:\s*(\d+)\s+failed/);
-  const hasJestFailures =
-    (failedSuites && parseInt(failedSuites[1], 10) > 0) ||
-    (failedTests && parseInt(failedTests[1], 10) > 0);
+  if (failedSuites && parseInt(failedSuites[1], 10) > 0) {
+    failureReasons.push(`${failedSuites[1]} test suite(s) failed`);
+  }
+  if (failedTests && parseInt(failedTests[1], 10) > 0) {
+    failureReasons.push(`${failedTests[1]} test(s) failed`);
+  }
+
+  // Tracebacks always fail. They catch use-after-free bugs that jest cannot
+  // see, since a deferred-callback crash fires outside any test.
+  const tracebacks = countTracebacks(cleanLogs);
+  if (tracebacks > 0) {
+    failureReasons.push(
+      `${tracebacks} Luau traceback(s) — see attributed errors below`
+    );
+  }
 
-  // Check for Luau runtime errors (stack traces)
-  const hasRuntimeError = /Stack Begin\s/.test(cleanLogs);
+  if (options.requireTestReport && !hasTestReport(cleanLogs)) {
+    failureReasons.push(
+      'no jest report in output — nothing proves any test ran'
+    );
+  }
+
+  return { success: failureReasons.length === 0, failureReasons };
+}
+
+/**
+ * True when jest reported at all, including reporting that it found nothing.
+ *
+ * "Ran and found no tests" is a legitimate answer for packages that ship
+ * without specs; "we never heard from jest" is not. Keying on the report
+ * rather than on a test count keeps `passWithNoTests` working.
+ */
+export function hasTestReport(rawOutput: string): boolean {
+  const clean = OutputHelper.stripAnsi(rawOutput);
+  return (
+    /Tests:\s+/.test(clean) ||
+    /Test Suites:\s+/.test(clean) ||
+    /No tests found/i.test(clean)
+  );
+}
+
+/** Count Luau tracebacks in output. */
+export function countTracebacks(rawOutput: string): number {
+  if (!rawOutput) {
+    return 0;
+  }
+  return OutputHelper.stripAnsi(rawOutput).match(/Stack Begin\s/g)?.length ?? 0;
+}
+
+/**
+ * Analyze test output for Jest failures and Luau runtime errors.
+ * Shared by both Open Cloud log fetching and local run-in-roblox output.
+ */
+export function parseTestLogs(
+  rawOutput: string,
+  options: TestOutcomeOptions = {}
+): ParsedTestLogs {
+  const outcome = evaluateTestOutcome(rawOutput, options);
 
   return {
-    success: !hasJestFailures && !hasRuntimeError,
+    success: outcome.success,
+    failureReasons: outcome.failureReasons,
     logs: rawOutput,
     testCounts: parseTestCounts(rawOutput),
   };