@quenty/nevermore-cli 4.41.1 → 4.42.0

package/src/utils/testing/parsers/batch-log-parser.ts

@@ -1,5 +1,10 @@
 import { OutputHelper } from '@quenty/cli-output-helpers';
-import { type ParsedTestCounts, parseTestCounts } from '../test-log-parser.js';
+import {
+  type ParsedTestCounts,
+  evaluateTestOutcome,
+  parseTestCounts,
+} from '../test-log-parser.js';
+import { formatTracebacks, parseTracebacks } from './traceback-parser.js';
 
 export interface BatchPackageResult {
   slug: string;
@@ -59,12 +64,17 @@ export function parseBatchTestLogs(
   // ── Pass 1: Extract per-package log sections from markers ──
 
   const logSections = new Map<string, string>();
+  /** Sections closed by an END whose BEGIN never arrived — start of output lost. */
+  const partialSections = new Set<string>();
   const markerDurations = new Map<string, number>();
   let currentSlug: string | null = null;
   let currentLines: string[] = [];
   let summaryLineIndex = -1;
   let beginMarkersSeen = 0;
   let endMarkersSeen = 0;
+  let strayEndMarkers = 0;
+  /** Only one section can have lost its head: the one the log was cut inside. */
+  let headSectionClaimed = false;
 
   // Matches "<slug> PASS|FAIL [<durationMs>]" — slugs have no whitespace.
   const endInnerPattern = /^(\S+)\s+(?:PASS|FAIL)(?:\s+(\d+))?$/;
@@ -86,16 +96,36 @@ export function parseBatchTestLogs(
       const endSlug = match ? match[1] : inner;
       const durationStr = match?.[2];
 
-      if (currentSlug && endSlug === currentSlug) {
-        logSections.set(currentSlug, currentLines.join('\n'));
+      // A section normally closes on the END matching its own BEGIN.
+      const closesOwnSection = endSlug === currentSlug;
+
+      // Open Cloud keeps only the tail of a long run's log, so BEGIN — printed
+      // first — can be lost while END and the summary survive. Recovering that
+      // means letting an END with no open section claim what precedes it, but
+      // only where the head can actually have been dropped: before any BEGIN
+      // has survived, and only once. Past that point the log is well-formed,
+      // and a BEGIN-less END is a message delivered out of order (the API does
+      // not order them), which must not be allowed to claim another package's
+      // output.
+      const closesDroppedHead =
+        currentSlug === null && beginMarkersSeen === 0 && !headSectionClaimed;
+
+      if (closesOwnSection || closesDroppedHead) {
+        logSections.set(endSlug, currentLines.join('\n'));
+        if (closesDroppedHead) {
+          partialSections.add(endSlug);
+          headSectionClaimed = true;
+        }
         if (durationStr !== undefined) {
-          markerDurations.set(currentSlug, parseInt(durationStr, 10));
+          markerDurations.set(endSlug, parseInt(durationStr, 10));
         }
         currentSlug = null;
         currentLines = [];
+      } else {
+        // Reordered marker from another package. Ignore it without resetting
+        // state, so the section it interrupted still closes on its own END.
+        strayEndMarkers++;
       }
-      // If endSlug doesn't match currentSlug, this is a reordered marker
-      // from another package — ignore it without resetting state.
       continue;
     }
 
@@ -104,9 +134,9 @@ export function parseBatchTestLogs(
       break;
     }
 
-    if (currentSlug) {
-      currentLines.push(lines[i]);
-    }
+    // Accumulate unconditionally: output that precedes the first surviving
+    // BEGIN still belongs to whichever package's END closes it.
+    currentLines.push(lines[i]);
   }
 
   // ── Pass 2: Parse the JSON summary for authoritative pcall results ──
@@ -114,12 +144,16 @@ export function parseBatchTestLogs(
   const summaryResults = new Map<string, boolean>();
   const summaryDurations = new Map<string, number>();
   if (summaryLineIndex >= 0 && summaryLineIndex + 1 < lines.length) {
-    const jsonLine = lines
-      .slice(summaryLineIndex + 1)
-      .join('\n')
-      .trim();
-    try {
-      const entries = JSON.parse(jsonLine) as SummaryEntry[];
+    const entries = findSummaryEntries(lines, summaryLineIndex + 1);
+    if (entries === undefined) {
+      OutputHelper.verbose(
+        `[batch-log-parser] Failed to parse JSON summary after line ${summaryLineIndex}: ` +
+          lines
+            .slice(summaryLineIndex + 1, summaryLineIndex + 3)
+            .join('\n')
+            .slice(0, 200)
+      );
+    } else {
       for (const entry of entries) {
         summaryResults.set(entry.slug, entry.success);
         if (typeof entry.durationMs === 'number') {
@@ -134,20 +168,20 @@ export function parseBatchTestLogs(
         );
       }
       console.error(
-        `[batch-log-parser] Parsed ${entries.length} summary entries, ${failures.length} failures`
-      );
-    } catch {
-      OutputHelper.verbose(
-        `[batch-log-parser] Failed to parse JSON summary: ${jsonLine.slice(
-          0,
-          200
-        )}`
+        `[batch-log-parser] Parsed ${entries.length} summary entries, ${failures.length} pcall failures`
       );
     }
   }
 
   // ── Warn when the batch produced no recognizable output ──
 
+  if (strayEndMarkers > 0) {
+    OutputHelper.verbose(
+      `[batch-log-parser] Ignored ${strayEndMarkers} out-of-order END marker(s); ` +
+        'their sections closed on their own boundaries.'
+    );
+  }
+
   const noOutputAtAll = logSections.size === 0 && summaryResults.size === 0;
   if (noOutputAtAll) {
     OutputHelper.warn(
@@ -199,54 +233,68 @@ export function parseBatchTestLogs(
       unattributedClaimed = true;
     }
     const summarySuccess = summaryResults.get(slug);
+    const reasons: string[] = [];
 
-    // The JSON summary (pcall result) is the primary success signal.
-    // Jest failure detection provides a secondary override.
-    //
-    // Gate on attributed output only. Unattributed output is shown but not
-    // judged: whatever broke attribution is reason enough not to trust which
-    // package a failure line belongs to.
+    // The pcall result only proves the script did not throw. It is a floor, not
+    // a verdict — everything below can fail a run it called successful.
     let success = summarySuccess ?? false;
-    let error: string | undefined;
-    if (success && attributedLogs) {
-      const hasJestFailures = _hasJestFailuresInLogs(attributedLogs);
-      if (hasJestFailures) {
+    if (summarySuccess === false) {
+      reasons.push('the batch runner reported a Luau error');
+    } else if (summarySuccess === undefined) {
+      // Absent from the summary is a different fault from failing in it, and
+      // conflating them sends you hunting for a Luau error that never happened.
+      reasons.push('this package is missing from the batch summary');
+    }
+
+    if (attributedLogs !== undefined) {
+      const outcome = evaluateTestOutcome(attributedLogs, {
+        requireTestReport: true,
+      });
+      if (!outcome.success) {
         success = false;
-        error = 'Jest reported failing tests';
+        reasons.push(...outcome.failureReasons);
       }
+    } else {
+      // Judged unreadable rather than judged by content: whatever broke
+      // attribution is reason enough not to trust which package a line is from.
+      success = false;
+      reasons.push(
+        `no output could be attributed to this package ` +
+          `(${rawLogs.length} chars received, ${beginMarkersSeen} BEGIN markers found)`
+      );
     }
 
-    // A pass we cannot read is not a pass. The pcall only proves the script did
-    // not throw, which says nothing about whether any test ran or passed.
-    if (success && !attributedLogs) {
-      success = false;
-      error =
-        `No test output could be attributed to this package ` +
-        `(${rawLogs.length} chars received, ${beginMarkersSeen} BEGIN markers found). ` +
-        'Unattributed output follows.';
+    if (partialSections.has(slug)) {
+      OutputHelper.warn(
+        `[batch-log-parser] ${slug}: section closed by its END marker with no BEGIN — ` +
+          'the start of this output was dropped by the log window, so the section is partial.'
+      );
     }
 
-    if (!success && !noOutputAtAll) {
-      console.error(
-        `[batch-log-parser] ${slug}: summarySuccess=${summarySuccess} hasLogs=${
-          sectionLogs.length > 0
-        } logsLen=${sectionLogs.length}`
+    // Attribute by script path, not by log position: a deferred callback can
+    // fire during another package's section.
+    const tracebacks = parseTracebacks(sectionLogs);
+    const tracebackCount = tracebacks.reduce((sum, t) => sum + t.count, 0);
+    const owned = tracebacks.filter((t) => !t.owner || t.owner === slug);
+    if (tracebacks.length > owned.length) {
+      OutputHelper.verbose(
+        `[batch-log-parser] ${slug}: ignored ${
+          tracebacks.length - owned.length
+        } traceback(s) belonging to other packages`
+      );
+    }
+    if (owned.length > 0) {
+      OutputHelper.warn(
+        `[batch-log-parser] ${slug}: ${owned.length} distinct traceback(s):\n` +
+          formatTracebacks(owned)
       );
     }
 
-    // Deliberately not parsed from unattributed output: attribution is exactly
-    // what failed, so a count read from it could belong to anything. The logs
-    // are still shown, so the jest summary remains readable by eye.
+    const error = reasons.length > 0 ? reasons.join('; ') : undefined;
+
     const testCounts = attributedLogs
       ? parseTestCounts(attributedLogs)
       : undefined;
-    const tracebackCount = countTracebacks(sectionLogs);
-    if (tracebackCount > 0) {
-      OutputHelper.warn(
-        `[batch-log-parser] ${slug}: ${tracebackCount} Luau traceback(s) in output. ` +
-          'Jest does not count these — a deferred-callback crash fires outside any test.'
-      );
-    }
     // Prefer the JSON summary (authoritative, immune to log reordering);
     // fall back to the END-marker value if the summary was truncated.
     const durationMs = summaryDurations.get(slug) ?? markerDurations.get(slug);
@@ -265,31 +313,44 @@ export function parseBatchTestLogs(
 }
 
 /**
- * Count Luau tracebacks in log output.
+ * Find the summary array in the lines following the summary marker.
  *
- * Reported separately from test counts because jest never sees these: a crash in
- * a deferred callback fires outside any test, so a run can print
- * "Tests: 155 passed, 0 failed" while something is genuinely broken.
+ * The marker is printed last, but the API does not deliver messages in order,
+ * so unrelated output can land after it. Treating everything past the marker as
+ * one JSON blob therefore fails intermittently — the same batch parses on one
+ * run and not the next. Scan for the array instead of assuming it is alone.
  */
-export function countTracebacks(rawOutput: string): number {
-  if (!rawOutput) {
-    return 0;
+export function findSummaryEntries(
+  lines: string[],
+  startIndex: number
+): SummaryEntry[] | undefined {
+  const MAX_SPAN = 50;
+
+  for (let i = startIndex; i < lines.length; i++) {
+    if (!lines[i].trim().startsWith('[')) {
+      continue;
+    }
+
+    // Normally one line, but allow the array to span a few in case the runner
+    // or the transport breaks it up.
+    for (let end = i; end < Math.min(lines.length, i + MAX_SPAN); end++) {
+      try {
+        const parsed = JSON.parse(
+          lines
+            .slice(i, end + 1)
+            .join('\n')
+            .trim()
+        );
+        if (Array.isArray(parsed)) {
+          return parsed as SummaryEntry[];
+        }
+      } catch {
+        // Incomplete so far — grow the window and try again.
+      }
+    }
   }
-  const cleanLogs = OutputHelper.stripAnsi(rawOutput);
-  return cleanLogs.match(/Stack Begin\s/g)?.length ?? 0;
-}
 
-/**
- * Check specifically for Jest test suite/test failures in logs.
- * Unlike parseTestLogs, this ignores "Stack Begin" runtime errors since
- * those can come from deferred callbacks of other packages in batch mode.
- */
-function _hasJestFailuresInLogs(rawOutput: string): boolean {
-  const cleanLogs = OutputHelper.stripAnsi(rawOutput);
-  const failedSuites = cleanLogs.match(/Test Suites:\s*(\d+)\s+failed/);
-  const failedTests = cleanLogs.match(/Tests:\s*(\d+)\s+failed/);
-  return (
-    (failedSuites != null && parseInt(failedSuites[1], 10) > 0) ||
-    (failedTests != null && parseInt(failedTests[1], 10) > 0)
-  );
+  return undefined;
 }
+
+export { countTracebacks } from '../test-log-parser.js';

package/src/utils/testing/parsers/traceback-parser.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+  formatTracebacks,
+  ownerOf,
+  parseTracebacks,
+} from './traceback-parser.js';
+
+const TRACEBACK = [
+  'PlayerBadgeHelper: attempt to index nil with UserId',
+  'Stack Begin',
+  'Script \'[string "ServerScriptService.egghunt2026.game.Server.PlayerBadgeHelper"]\', Line 365 - function _awardBadge',
+  'Script \'[string "ServerScriptService.egghunt2026.game.Server.PlayerBadgeHelper"]\', Line 138',
+  'Stack End',
+].join('\n');
+
+describe('parseTracebacks', () => {
+  it('attributes a traceback to the package that owns the failing script', () => {
+    const [traceback] = parseTracebacks(TRACEBACK);
+
+    expect(traceback.owner).toBe('egghunt2026');
+    expect(traceback.source).toContain('PlayerBadgeHelper');
+    expect(traceback.message).toContain('attempt to index nil');
+  });
+
+  it('collapses repeats of the same crash into one entry with a count', () => {
+    // One bug fires per player per frame, so raw output can hold hundreds of
+    // identical copies. Printing them all is what makes logs unreadable.
+    const repeated = [TRACEBACK, TRACEBACK, TRACEBACK].join('\n');
+
+    const tracebacks = parseTracebacks(repeated);
+
+    expect(tracebacks).toHaveLength(1);
+    expect(tracebacks[0].count).toBe(3);
+  });
+
+  it('keeps distinct crashes apart', () => {
+    const other = TRACEBACK.replace('PlayerBadgeHelper', 'TrainHelper');
+
+    expect(parseTracebacks([TRACEBACK, other].join('\n'))).toHaveLength(2);
+  });
+
+  it('finds nothing in clean output', () => {
+    expect(parseTracebacks('Tests: 10 passed, 10 total')).toEqual([]);
+  });
+});
+
+describe('ownerOf', () => {
+  it('names the package below the service', () => {
+    expect(ownerOf('ServerScriptService.brio.Foo.Bar')).toBe('brio');
+  });
+});
+
+describe('formatTracebacks', () => {
+  it('bounds how many it prints and says how many it withheld', () => {
+    const many = Array.from({ length: 8 }, (_, i) =>
+      TRACEBACK.replace('PlayerBadgeHelper:', `Helper${i}:`)
+    ).join('\n');
+
+    const formatted = formatTracebacks(parseTracebacks(many), 5);
+
+    expect(formatted).toContain('and 3 more distinct traceback(s)');
+  });
+});

package/src/utils/testing/parsers/traceback-parser.ts

@@ -0,0 +1,122 @@
+import { OutputHelper } from '@quenty/cli-output-helpers';
+
+export interface AttributedTraceback {
+  /** First line of the error, before the stack. */
+  message: string;
+  /** Roblox instance path of the topmost frame, e.g. "ServerScriptService.Foo.Bar". */
+  source?: string;
+  /** Root of that path, which names the package the frame belongs to. */
+  owner?: string;
+  /** Frames kept for display, topmost first. */
+  frames: string[];
+  /** How many times this same traceback appeared. */
+  count: number;
+}
+
+const STACK_BEGIN = /^\s*Stack Begin\s*$/;
+const STACK_END = /^\s*Stack End\s*$/;
+// Script 'ServerScriptService.EggHunt2026.game.Server.Foo', Line 365 - function bar
+const FRAME = /Script\s+'(?:\[string\s+")?([^'"\]]+)/;
+
+const MAX_FRAMES = 4;
+
+/**
+ * Pull tracebacks out of log output and attribute each to its owning package.
+ *
+ * Attribution reads the script path in the frames rather than the position of
+ * the traceback in the log. Position is unreliable: a deferred callback can
+ * fire during a later package's section, which is why batch mode previously
+ * gave up on tracebacks entirely.
+ */
+export function parseTracebacks(rawOutput: string): AttributedTraceback[] {
+  if (!rawOutput) {
+    return [];
+  }
+
+  const lines = OutputHelper.stripAnsi(rawOutput).split('\n');
+  const bySignature = new Map<string, AttributedTraceback>();
+
+  for (let i = 0; i < lines.length; i++) {
+    if (!STACK_BEGIN.test(lines[i])) {
+      continue;
+    }
+
+    // The error text sits on the line above "Stack Begin".
+    const message = (lines[i - 1] ?? '').trim();
+
+    const frames: string[] = [];
+    let source: string | undefined;
+    for (let j = i + 1; j < lines.length && !STACK_END.test(lines[j]); j++) {
+      const frame = lines[j].trim();
+      if (!frame) {
+        continue;
+      }
+      if (frames.length < MAX_FRAMES) {
+        frames.push(frame);
+      }
+      if (!source) {
+        source = FRAME.exec(frame)?.[1];
+      }
+    }
+
+    // One bug fires per player and per frame, so the same traceback can appear
+    // hundreds of times. Collapse by signature and count instead of printing
+    // every copy.
+    const signature = `${message}::${frames[0] ?? ''}`;
+    const existing = bySignature.get(signature);
+    if (existing) {
+      existing.count++;
+      continue;
+    }
+
+    bySignature.set(signature, {
+      message,
+      source,
+      owner: source ? ownerOf(source) : undefined,
+      frames,
+      count: 1,
+    });
+  }
+
+  return [...bySignature.values()];
+}
+
+/**
+ * Name the package a script path belongs to.
+ *
+ * Batch places hold each package under its own root, so the segment below the
+ * service is the package.
+ */
+export function ownerOf(scriptPath: string): string | undefined {
+  const parts = scriptPath.split('.');
+  return parts.length >= 2 ? parts[1] : undefined;
+}
+
+/** Render tracebacks for a human, newest information first and bounded. */
+export function formatTracebacks(
+  tracebacks: AttributedTraceback[],
+  maxShown = 5
+): string {
+  if (tracebacks.length === 0) {
+    return '';
+  }
+
+  const lines: string[] = [];
+  const shown = tracebacks.slice(0, maxShown);
+
+  for (const traceback of shown) {
+    const where = traceback.source ? ` (${traceback.source})` : '';
+    const times = traceback.count > 1 ? ` ×${traceback.count}` : '';
+    lines.push(`  ${traceback.message}${where}${times}`);
+    for (const frame of traceback.frames) {
+      lines.push(`      ${frame}`);
+    }
+  }
+
+  const hidden = tracebacks.length - shown.length;
+  if (hidden > 0) {
+    lines.push(`  ...and ${hidden} more distinct traceback(s)`);
+  }
+
+  return lines.join('\n');
+}

package/src/utils/testing/runner/test-runner.test.ts

@@ -0,0 +1,29 @@
+import { describe, expect, it } from 'vitest';
+
+import { mergeFailureReasons } from './test-runner.js';
+
+describe('mergeFailureReasons', () => {
+  it('does not repeat reasons the context already reported', () => {
+    // Batch mode evaluates the same logs upstream, so both sources describe the
+    // same failures. Concatenating them read as double the failures.
+    const merged = mergeFailureReasons(
+      '3 test suite(s) failed; 19 test(s) failed',
+      ['3 test suite(s) failed', '19 test(s) failed']
+    );
+
+    expect(merged).toEqual(['3 test suite(s) failed', '19 test(s) failed']);
+  });
+
+  it('keeps reasons only one side knows about', () => {
+    const merged = mergeFailureReasons(
+      'the batch runner reported a Luau error',
+      ['19 test(s) failed']
+    );
+
+    expect(merged).toHaveLength(2);
+  });
+
+  it('is empty when nothing failed', () => {
+    expect(mergeFailureReasons(undefined, [])).toEqual([]);
+  });
+});

package/src/utils/testing/runner/test-runner.ts

@@ -30,6 +30,25 @@ export interface SingleTestResult {
   error?: string;
 }
 
+/**
+ * Combine failure reasons from the job context with those read from the logs.
+ *
+ * In batch mode the context has already evaluated these same logs, so the two
+ * sources overlap. Merging on the individual reason keeps the overlap from
+ * reading as double the failures.
+ */
+export function mergeFailureReasons(
+  contextError: string | undefined,
+  logReasons: string[]
+): string[] {
+  return [
+    ...new Set([
+      ...(contextError ? contextError.split('; ') : []),
+      ...logReasons,
+    ]),
+  ];
+}
+
 export interface SingleTestOptions {
   packagePath: string;
   packageName: string;
@@ -56,7 +75,10 @@ export async function runSingleTestAsync(
   const {
     packagePath,
     packageName,
-    timeoutMs = 120_000,
+    // 300s is the Open Cloud maximum. The old 120s default failed real suites
+    // with no jest summary at all, so "--timeout 300" had become folklore
+    // everyone had to know before their first useful run.
+    timeoutMs = 300_000,
     scriptText,
     target,
   } = options;
@@ -104,14 +126,24 @@ export async function runSingleTestAsync(
     });
 
     const rawLogs = await context.getLogsAsync(deployment);
-    const parsed = parseTestLogs(rawLogs);
+    // A probe script is arbitrary Luau with no jest in it, so demanding a test
+    // report would fail every --script-text run. Everything else must prove a
+    // runner spoke before it can pass.
+    const parsed = parseTestLogs(rawLogs, {
+      requireTestReport: scriptText === undefined,
+    });
+
+    const reasons = mergeFailureReasons(
+      result.errorMessage,
+      parsed.failureReasons
+    );
 
     return {
       success: result.success && parsed.success,
       logs: parsed.logs,
       testCounts: parseTestCounts(parsed.logs),
       durationMs: result.durationMs,
-      error: result.errorMessage,
+      error: reasons.length > 0 ? reasons.join('; ') : undefined,
     };
   } finally {
     await context.releaseAsync(deployment);

package/src/utils/testing/test-log-parser.test.ts

@@ -0,0 +1,70 @@
+import { describe, expect, it } from 'vitest';
+
+import { evaluateTestOutcome, hasTestReport } from './test-log-parser.js';
+
+describe('evaluateTestOutcome', () => {
+  it('passes a clean jest report', () => {
+    const outcome = evaluateTestOutcome('Tests: 275 passed, 275 total', {
+      requireTestReport: true,
+    });
+
+    expect(outcome.success).toBe(true);
+    expect(outcome.failureReasons).toEqual([]);
+  });
+
+  it('refuses to pass output with no jest report in it', () => {
+    // Absence of failure was previously read as success, so an empty log passed.
+    const outcome = evaluateTestOutcome('some unrelated chatter', {
+      requireTestReport: true,
+    });
+
+    expect(outcome.success).toBe(false);
+    expect(outcome.failureReasons.join()).toContain('no jest report');
+  });
+
+  it('exempts probe runs, which have no jest in them at all', () => {
+    const outcome = evaluateTestOutcome('hello from a probe script');
+
+    expect(outcome.success).toBe(true);
+  });
+
+  it('fails on a traceback even when every test passed', () => {
+    // Jest cannot see a deferred-callback crash: it fires outside any test.
+    const logs = [
+      'Tests: 155 passed, 155 total',
+      'attempt to index nil',
+      'Stack Begin',
+      "Script 'ServerScriptService.egghunt2026.Foo', Line 12",
+      'Stack End',
+    ].join('\n');
+
+    const outcome = evaluateTestOutcome(logs, { requireTestReport: true });
+
+    expect(outcome.success).toBe(false);
+    expect(outcome.failureReasons.join()).toContain('traceback');
+  });
+
+  it('reports every reason a run failed, not just the first', () => {
+    const logs = [
+      'Tests: 19 failed, 470 passed, 516 total',
+      'Stack Begin',
+      "Script 'ServerScriptService.egghunt2026.Foo', Line 12",
+      'Stack End',
+    ].join('\n');
+
+    const outcome = evaluateTestOutcome(logs, { requireTestReport: true });
+
+    expect(outcome.failureReasons).toHaveLength(2);
+  });
+});
+
+describe('hasTestReport', () => {
+  it('accepts a runner that ran and found nothing', () => {
+    // Packages without specs are legitimate; passWithNoTests depends on this.
+    expect(hasTestReport('No tests found, exiting with code 0')).toBe(true);
+  });
+
+  it('rejects silence', () => {
+    expect(hasTestReport('')).toBe(false);
+  });
+});