@quenty/nevermore-cli 4.40.0 → 4.41.1

package/src/utils/testing/parsers/batch-log-parser.test.ts

@@ -0,0 +1,85 @@
+import { describe, expect, it } from 'vitest';
+
+import { countTracebacks, parseBatchTestLogs } from './batch-log-parser.js';
+
+const SLUG_MAP = new Map([['egghunt2026', 'egghunt2026']]);
+
+function buildLogs(section: string): string {
+  return [
+    '===BATCH_TEST_BEGIN egghunt2026===',
+    section,
+    '===BATCH_TEST_END egghunt2026 PASS 1000===',
+    '===BATCH_TEST_SUMMARY===',
+    '[{"slug":"egghunt2026","success":true,"durationMs":1000}]',
+  ].join('\n');
+}
+
+describe('parseBatchTestLogs', () => {
+  it('reads test counts out of a package section', () => {
+    const logs = buildLogs('Tests:  275 passed, 275 total');
+
+    const result = parseBatchTestLogs(logs, SLUG_MAP).get('egghunt2026');
+
+    expect(result?.testCounts).toEqual({ passed: 275, failed: 0, total: 275 });
+  });
+
+  it('fails a pass it cannot read, rather than trusting the pcall', () => {
+    // The summary prints last, so it survives a log window that dropped the
+    // head. The pcall said success, but nothing here saw a test run.
+    const logs = [
+      'Tests:  275 passed, 275 total',
+      '===BATCH_TEST_END egghunt2026 PASS 1000===',
+      '===BATCH_TEST_SUMMARY===',
+      '[{"slug":"egghunt2026","success":true,"durationMs":1000}]',
+    ].join('\n');
+
+    const result = parseBatchTestLogs(logs, SLUG_MAP).get('egghunt2026');
+
+    expect(result?.success).toBe(false);
+    expect(result?.error).toContain('No test output could be attributed');
+    expect(result?.testCounts).toBeUndefined();
+  });
+
+  it('still shows output it could not attribute', () => {
+    // 330KB of real test output was being reported as "(no output)" because no
+    // BEGIN marker survived to delimit it.
+    const logs = [
+      'PlayerBadgeHelper: attempt to index nil',
+      '===BATCH_TEST_SUMMARY===',
+      '[{"slug":"egghunt2026","success":true,"durationMs":1000}]',
+    ].join('\n');
+
+    const result = parseBatchTestLogs(logs, SLUG_MAP).get('egghunt2026');
+
+    expect(result?.logs).toContain('PlayerBadgeHelper');
+  });
+
+  it('counts tracebacks that jest reports as a clean pass', () => {
+    const logs = buildLogs(
+      [
+        'Tests:  155 passed, 155 total',
+        'PlayerBadgeHelper: attempt to index nil',
+        'Stack Begin',
+        "Script 'PlayerBadgeHelper', Line 365",
+        'Stack End',
+      ].join('\n')
+    );
+
+    const result = parseBatchTestLogs(logs, SLUG_MAP).get('egghunt2026');
+
+    expect(result?.testCounts?.failed).toBe(0);
+    expect(result?.tracebackCount).toBe(1);
+  });
+});
+
+describe('countTracebacks', () => {
+  it('counts each traceback separately', () => {
+    expect(countTracebacks('Stack Begin\nfoo\nStack End\nStack Begin\n')).toBe(
+      2
+    );
+  });
+
+  it('is zero for empty output', () => {
+    expect(countTracebacks('')).toBe(0);
+  });
+});

package/src/utils/testing/parsers/batch-log-parser.ts

@@ -8,6 +8,14 @@ export interface BatchPackageResult {
   /** Inner pcall execution time reported by the Luau batch runner. */
   durationMs?: number;
   testCounts?: ParsedTestCounts;
+  /** Why this package failed, when the failure is the parser's own verdict. */
+  error?: string;
+  /**
+   * Luau tracebacks found in this package's section. Reported, not gated on:
+   * jest cannot count these, since a deferred-callback crash fires outside any
+   * test — often after the test that scheduled it already passed.
+   */
+  tracebackCount: number;
 }
 
 const BEGIN_MARKER = '===BATCH_TEST_BEGIN ';
@@ -55,6 +63,8 @@ export function parseBatchTestLogs(
   let currentSlug: string | null = null;
   let currentLines: string[] = [];
   let summaryLineIndex = -1;
+  let beginMarkersSeen = 0;
+  let endMarkersSeen = 0;
 
   // Matches "<slug> PASS|FAIL [<durationMs>]" — slugs have no whitespace.
   const endInnerPattern = /^(\S+)\s+(?:PASS|FAIL)(?:\s+(\d+))?$/;
@@ -65,11 +75,13 @@ export function parseBatchTestLogs(
     if (trimmed.startsWith(BEGIN_MARKER) && trimmed.endsWith('===')) {
       currentSlug = trimmed.slice(BEGIN_MARKER.length, -3);
       currentLines = [];
+      beginMarkersSeen++;
       continue;
     }
 
     if (trimmed.startsWith(END_MARKER) && trimmed.endsWith('===')) {
       const inner = trimmed.slice(END_MARKER.length, -3);
+      endMarkersSeen++;
       const match = endInnerPattern.exec(inner);
       const endSlug = match ? match[1] : inner;
       const durationStr = match?.[2];
@@ -144,26 +156,76 @@ export function parseBatchTestLogs(
     );
   }
 
+  // The summary is printed last, so it survives a log fetch that dropped the
+  // output above it. Summary present with no sections is exactly that case, and
+  // it is invisible to the check above — the results below would then be built
+  // from pcall exit status alone, knowing nothing about what actually ran.
+  const lostSectionOutput = logSections.size === 0 && summaryResults.size > 0;
+  if (lostSectionOutput) {
+    OutputHelper.warn(
+      `[batch-log-parser] Received ${rawLogs.length} chars of output but could not ` +
+        `attribute any of it to a package section ` +
+        `(${beginMarkersSeen} BEGIN, ${endMarkersSeen} END markers; summary at line ` +
+        `${summaryLineIndex} of ${lines.length}). ` +
+        'Test counts are not derived from unattributed output.'
+    );
+    // BEGIN is printed first and the summary last, so their positions separate
+    // the two ways this happens: a log window that dropped the head keeps the
+    // summary at the end, while reordered messages put it early.
+    OutputHelper.warn(
+      summaryLineIndex >= lines.length - 2
+        ? '[batch-log-parser] Summary sits at the end of the log — consistent with the head being dropped by a log size/retention limit.'
+        : '[batch-log-parser] Summary sits early in the log — consistent with out-of-order messages ending the scan prematurely.'
+    );
+  }
+
   // ── Pass 3: Combine log sections with summary results ──
 
-  // When the batch produced zero output, surface the raw error in every
-  // package's logs so reporters show the actual failure instead of "(no output)".
-  const fallbackLogs = noOutputAtAll ? rawLogs.trim() : '';
+  // Output that arrived but could not be split into sections is still output.
+  // Reporting "(no output)" over it hides the very thing needed to diagnose the
+  // parse failure, so it is shown verbatim instead. Only safe to hand to a
+  // single package: with several, there is no way to say whose output this is.
+  const fallbackLogs = noOutputAtAll || lostSectionOutput ? rawLogs.trim() : '';
+
+  // The unattributed stream is the same text for every package, so it is
+  // attached once rather than repeated per package across a large log.
+  let unattributedClaimed = false;
 
   for (const [packageName, slug] of slugMap) {
-    const sectionLogs = logSections.get(slug) ?? fallbackLogs;
+    const attributedLogs = logSections.get(slug);
+    let sectionLogs = attributedLogs ?? '';
+    if (!attributedLogs && fallbackLogs && !unattributedClaimed) {
+      sectionLogs = fallbackLogs;
+      unattributedClaimed = true;
+    }
     const summarySuccess = summaryResults.get(slug);
 
     // The JSON summary (pcall result) is the primary success signal.
     // Jest failure detection provides a secondary override.
+    //
+    // Gate on attributed output only. Unattributed output is shown but not
+    // judged: whatever broke attribution is reason enough not to trust which
+    // package a failure line belongs to.
     let success = summarySuccess ?? false;
-    if (success && sectionLogs) {
-      const hasJestFailures = _hasJestFailuresInLogs(sectionLogs);
+    let error: string | undefined;
+    if (success && attributedLogs) {
+      const hasJestFailures = _hasJestFailuresInLogs(attributedLogs);
       if (hasJestFailures) {
         success = false;
+        error = 'Jest reported failing tests';
       }
     }
 
+    // A pass we cannot read is not a pass. The pcall only proves the script did
+    // not throw, which says nothing about whether any test ran or passed.
+    if (success && !attributedLogs) {
+      success = false;
+      error =
+        `No test output could be attributed to this package ` +
+        `(${rawLogs.length} chars received, ${beginMarkersSeen} BEGIN markers found). ` +
+        'Unattributed output follows.';
+    }
+
     if (!success && !noOutputAtAll) {
       console.error(
         `[batch-log-parser] ${slug}: summarySuccess=${summarySuccess} hasLogs=${
@@ -172,7 +234,19 @@ export function parseBatchTestLogs(
       );
     }
 
-    const testCounts = sectionLogs ? parseTestCounts(sectionLogs) : undefined;
+    // Deliberately not parsed from unattributed output: attribution is exactly
+    // what failed, so a count read from it could belong to anything. The logs
+    // are still shown, so the jest summary remains readable by eye.
+    const testCounts = attributedLogs
+      ? parseTestCounts(attributedLogs)
+      : undefined;
+    const tracebackCount = countTracebacks(sectionLogs);
+    if (tracebackCount > 0) {
+      OutputHelper.warn(
+        `[batch-log-parser] ${slug}: ${tracebackCount} Luau traceback(s) in output. ` +
+          'Jest does not count these — a deferred-callback crash fires outside any test.'
+      );
+    }
     // Prefer the JSON summary (authoritative, immune to log reordering);
     // fall back to the END-marker value if the summary was truncated.
     const durationMs = summaryDurations.get(slug) ?? markerDurations.get(slug);
@@ -182,12 +256,29 @@ export function parseBatchTestLogs(
       logs: sectionLogs,
       durationMs,
       testCounts,
+      tracebackCount,
+      error,
     });
   }
 
   return results;
 }
 
+/**
+ * Count Luau tracebacks in log output.
+ *
+ * Reported separately from test counts because jest never sees these: a crash in
+ * a deferred callback fires outside any test, so a run can print
+ * "Tests: 155 passed, 0 failed" while something is genuinely broken.
+ */
+export function countTracebacks(rawOutput: string): number {
+  if (!rawOutput) {
+    return 0;
+  }
+  const cleanLogs = OutputHelper.stripAnsi(rawOutput);
+  return cleanLogs.match(/Stack Begin\s/g)?.length ?? 0;
+}
+
 /**
  * Check specifically for Jest test suite/test failures in logs.
  * Unlike parseTestLogs, this ignores "Stack Begin" runtime errors since

package/src/utils/testing/reporting/test-github-columns.ts

@@ -19,6 +19,7 @@ export function createTestCommentConfig(): GithubCommentTableConfig {
     sectionId: 'tests',
     extraColumns: createTestColumns(),
     errorHeading: 'Test Results',
+    expectsTestCounts: true,
   };
 }
 

package/src/utils/testing/runner/test-runner.ts

@@ -14,6 +14,7 @@ import {
   injectDeployMetadataInPlaceAsync,
   packageUsesManifestAsync,
 } from '../../deploy/deploy-metadata.js';
+import { readPackageVersionAsync } from '../../nevermore-cli-utils.js';
 
 export interface SingleTestResult {
   success: boolean;
@@ -25,6 +26,8 @@ export interface SingleTestResult {
    * non-aggregated cloud/local runs — callers should fall back to wall-clock.
    */
   durationMs?: number;
+  /** Why the run failed, when the runner can say more than "it failed". */
+  error?: string;
 }
 
 export interface SingleTestOptions {
@@ -78,6 +81,7 @@ export async function runSingleTestAsync(
         timestamp: new Date().toISOString(),
         placeId: builtPlace.target.placeId,
         universeId: builtPlace.target.universeId,
+        packageVersion: await readPackageVersionAsync(packagePath),
       })
     );
   }
@@ -107,6 +111,7 @@ export async function runSingleTestAsync(
       logs: parsed.logs,
       testCounts: parseTestCounts(parsed.logs),
       durationMs: result.durationMs,
+      error: result.errorMessage,
     };
   } finally {
     await context.releaseAsync(deployment);