@quenty/nevermore-cli 4.41.0 → 4.41.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@quenty/nevermore-cli",
-  "version": "4.41.0",
+  "version": "4.41.1",
   "description": "CLI interface for Nevermore",
   "type": "module",
   "keywords": [
@@ -25,10 +25,10 @@
     "Quenty"
   ],
   "dependencies": {
-    "@quenty/cli-output-helpers": "1.17.0",
-    "@quenty/nevermore-cli-helpers": "1.16.0",
-    "@quenty/nevermore-template-helpers": "1.18.0",
-    "@quenty/studio-bridge": "0.17.0",
+    "@quenty/cli-output-helpers": "1.17.1",
+    "@quenty/nevermore-cli-helpers": "1.16.1",
+    "@quenty/nevermore-template-helpers": "1.18.1",
+    "@quenty/studio-bridge": "0.17.1",
     "execa": "^9.6.1",
     "find-git-root": "^1.0.4",
     "inquirer": "^13.2.0",
@@ -57,5 +57,5 @@
   "engines": {
     "node": ">=16"
   },
-  "gitHead": "8bd905a5c7e7f1fd4e178b54beeadef94a615d1d"
+  "gitHead": "35993999cfe6fbdfd44721a1835b6ee29602f057"
 }

package/src/commands/batch-command/batch-test-command.ts

@@ -171,6 +171,7 @@ async function _runAsync(args: BatchTestArgs): Promise<void> {
               showLogs: args.logs ?? false,
               verbose: args.verbose,
               actionVerb: 'Testing',
+              expectsTestCounts: true,
             })
           : new SpinnerReporter(state, {
               showLogs: args.logs ?? false,
@@ -251,6 +252,7 @@ async function _runAsync(args: BatchTestArgs): Promise<void> {
           progressSummary: result.testCounts
             ? { kind: 'test-counts' as const, ...result.testCounts }
             : undefined,
+          error: result.error,
         };
       },
     });

package/src/utils/job-context/batch-script-job-context.ts

@@ -113,7 +113,11 @@ export class BatchScriptJobContext implements JobContext {
       return { success: false };
     }
 
-    return { success: result.success, durationMs: result.durationMs };
+    return {
+      success: result.success,
+      durationMs: result.durationMs,
+      errorMessage: result.error,
+    };
   }
 
   async getLogsAsync(deployment: Deployment): Promise<string> {

package/src/utils/open-cloud/open-cloud-client.ts

@@ -336,7 +336,9 @@ export class OpenCloudClient {
   private async _fetchRawLogsAsync(taskPath: string): Promise<string> {
     const apiKey = await this._resolveApiKeyAsync();
     const messages: string[] = [];
+    const structured: { message: string; createTime: string }[] = [];
     let pageToken: string | undefined;
+    let pagesFetched = 0;
 
     do {
       const url = new URL(`https://apis.roblox.com/cloud/v2/${taskPath}/logs`);
@@ -371,18 +373,39 @@ export class OpenCloudClient {
 
       for (const entry of data.luauExecutionSessionTaskLogs ?? []) {
         if (entry.structuredMessages?.length) {
+          // Keep createTime alongside the text: the API does not guarantee
+          // chronological order, and readers downstream assume it.
           for (const msg of entry.structuredMessages) {
-            messages.push(msg.message);
+            structured.push({
+              message: msg.message,
+              createTime: msg.createTime,
+            });
           }
         } else if (entry.messages?.length) {
           messages.push(...entry.messages);
         }
       }
 
+      pagesFetched++;
       pageToken = data.nextPageToken || undefined;
     } while (pageToken);
 
-    return messages.join('\n');
+    // Out-of-order messages silently destroy marker-delimited parsing: a
+    // trailing summary that arrives early ends the scan before any of the
+    // output it summarizes has been read.
+    structured.sort((a, b) => a.createTime.localeCompare(b.createTime));
+    messages.push(...structured.map((m) => m.message));
+
+    const text = messages.join('\n');
+
+    // Log volume is the prime suspect when engine output goes missing between
+    // the task and the parser, and the parser cannot tell a short run from a
+    // truncated fetch. Record what arrived so a real run can settle it.
+    OutputHelper.verbose(
+      `[open-cloud] Task logs: ${pagesFetched} page(s), ${messages.length} messages, ${text.length} chars`
+    );
+
+    return text;
   }
 
   /**

package/src/utils/testing/parsers/batch-log-parser.test.ts

@@ -0,0 +1,85 @@
+import { describe, expect, it } from 'vitest';
+
+import { countTracebacks, parseBatchTestLogs } from './batch-log-parser.js';
+
+const SLUG_MAP = new Map([['egghunt2026', 'egghunt2026']]);
+
+function buildLogs(section: string): string {
+  return [
+    '===BATCH_TEST_BEGIN egghunt2026===',
+    section,
+    '===BATCH_TEST_END egghunt2026 PASS 1000===',
+    '===BATCH_TEST_SUMMARY===',
+    '[{"slug":"egghunt2026","success":true,"durationMs":1000}]',
+  ].join('\n');
+}
+
+describe('parseBatchTestLogs', () => {
+  it('reads test counts out of a package section', () => {
+    const logs = buildLogs('Tests:  275 passed, 275 total');
+
+    const result = parseBatchTestLogs(logs, SLUG_MAP).get('egghunt2026');
+
+    expect(result?.testCounts).toEqual({ passed: 275, failed: 0, total: 275 });
+  });
+
+  it('fails a pass it cannot read, rather than trusting the pcall', () => {
+    // The summary prints last, so it survives a log window that dropped the
+    // head. The pcall said success, but nothing here saw a test run.
+    const logs = [
+      'Tests:  275 passed, 275 total',
+      '===BATCH_TEST_END egghunt2026 PASS 1000===',
+      '===BATCH_TEST_SUMMARY===',
+      '[{"slug":"egghunt2026","success":true,"durationMs":1000}]',
+    ].join('\n');
+
+    const result = parseBatchTestLogs(logs, SLUG_MAP).get('egghunt2026');
+
+    expect(result?.success).toBe(false);
+    expect(result?.error).toContain('No test output could be attributed');
+    expect(result?.testCounts).toBeUndefined();
+  });
+
+  it('still shows output it could not attribute', () => {
+    // 330KB of real test output was being reported as "(no output)" because no
+    // BEGIN marker survived to delimit it.
+    const logs = [
+      'PlayerBadgeHelper: attempt to index nil',
+      '===BATCH_TEST_SUMMARY===',
+      '[{"slug":"egghunt2026","success":true,"durationMs":1000}]',
+    ].join('\n');
+
+    const result = parseBatchTestLogs(logs, SLUG_MAP).get('egghunt2026');
+
+    expect(result?.logs).toContain('PlayerBadgeHelper');
+  });
+
+  it('counts tracebacks that jest reports as a clean pass', () => {
+    const logs = buildLogs(
+      [
+        'Tests:  155 passed, 155 total',
+        'PlayerBadgeHelper: attempt to index nil',
+        'Stack Begin',
+        "Script 'PlayerBadgeHelper', Line 365",
+        'Stack End',
+      ].join('\n')
+    );
+
+    const result = parseBatchTestLogs(logs, SLUG_MAP).get('egghunt2026');
+
+    expect(result?.testCounts?.failed).toBe(0);
+    expect(result?.tracebackCount).toBe(1);
+  });
+});
+
+describe('countTracebacks', () => {
+  it('counts each traceback separately', () => {
+    expect(countTracebacks('Stack Begin\nfoo\nStack End\nStack Begin\n')).toBe(
+      2
+    );
+  });
+
+  it('is zero for empty output', () => {
+    expect(countTracebacks('')).toBe(0);
+  });
+});

package/src/utils/testing/parsers/batch-log-parser.ts

@@ -8,6 +8,14 @@ export interface BatchPackageResult {
   /** Inner pcall execution time reported by the Luau batch runner. */
   durationMs?: number;
   testCounts?: ParsedTestCounts;
+  /** Why this package failed, when the failure is the parser's own verdict. */
+  error?: string;
+  /**
+   * Luau tracebacks found in this package's section. Reported, not gated on:
+   * jest cannot count these, since a deferred-callback crash fires outside any
+   * test — often after the test that scheduled it already passed.
+   */
+  tracebackCount: number;
 }
 
 const BEGIN_MARKER = '===BATCH_TEST_BEGIN ';
@@ -55,6 +63,8 @@ export function parseBatchTestLogs(
   let currentSlug: string | null = null;
   let currentLines: string[] = [];
   let summaryLineIndex = -1;
+  let beginMarkersSeen = 0;
+  let endMarkersSeen = 0;
 
   // Matches "<slug> PASS|FAIL [<durationMs>]" — slugs have no whitespace.
   const endInnerPattern = /^(\S+)\s+(?:PASS|FAIL)(?:\s+(\d+))?$/;
@@ -65,11 +75,13 @@ export function parseBatchTestLogs(
     if (trimmed.startsWith(BEGIN_MARKER) && trimmed.endsWith('===')) {
       currentSlug = trimmed.slice(BEGIN_MARKER.length, -3);
       currentLines = [];
+      beginMarkersSeen++;
       continue;
     }
 
     if (trimmed.startsWith(END_MARKER) && trimmed.endsWith('===')) {
       const inner = trimmed.slice(END_MARKER.length, -3);
+      endMarkersSeen++;
       const match = endInnerPattern.exec(inner);
       const endSlug = match ? match[1] : inner;
       const durationStr = match?.[2];
@@ -144,26 +156,76 @@ export function parseBatchTestLogs(
     );
   }
 
+  // The summary is printed last, so it survives a log fetch that dropped the
+  // output above it. Summary present with no sections is exactly that case, and
+  // it is invisible to the check above — the results below would then be built
+  // from pcall exit status alone, knowing nothing about what actually ran.
+  const lostSectionOutput = logSections.size === 0 && summaryResults.size > 0;
+  if (lostSectionOutput) {
+    OutputHelper.warn(
+      `[batch-log-parser] Received ${rawLogs.length} chars of output but could not ` +
+        `attribute any of it to a package section ` +
+        `(${beginMarkersSeen} BEGIN, ${endMarkersSeen} END markers; summary at line ` +
+        `${summaryLineIndex} of ${lines.length}). ` +
+        'Test counts are not derived from unattributed output.'
+    );
+    // BEGIN is printed first and the summary last, so their positions separate
+    // the two ways this happens: a log window that dropped the head keeps the
+    // summary at the end, while reordered messages put it early.
+    OutputHelper.warn(
+      summaryLineIndex >= lines.length - 2
+        ? '[batch-log-parser] Summary sits at the end of the log — consistent with the head being dropped by a log size/retention limit.'
+        : '[batch-log-parser] Summary sits early in the log — consistent with out-of-order messages ending the scan prematurely.'
+    );
+  }
+
   // ── Pass 3: Combine log sections with summary results ──
 
-  // When the batch produced zero output, surface the raw error in every
-  // package's logs so reporters show the actual failure instead of "(no output)".
-  const fallbackLogs = noOutputAtAll ? rawLogs.trim() : '';
+  // Output that arrived but could not be split into sections is still output.
+  // Reporting "(no output)" over it hides the very thing needed to diagnose the
+  // parse failure, so it is shown verbatim instead. Only safe to hand to a
+  // single package: with several, there is no way to say whose output this is.
+  const fallbackLogs = noOutputAtAll || lostSectionOutput ? rawLogs.trim() : '';
+
+  // The unattributed stream is the same text for every package, so it is
+  // attached once rather than repeated per package across a large log.
+  let unattributedClaimed = false;
 
   for (const [packageName, slug] of slugMap) {
-    const sectionLogs = logSections.get(slug) ?? fallbackLogs;
+    const attributedLogs = logSections.get(slug);
+    let sectionLogs = attributedLogs ?? '';
+    if (!attributedLogs && fallbackLogs && !unattributedClaimed) {
+      sectionLogs = fallbackLogs;
+      unattributedClaimed = true;
+    }
     const summarySuccess = summaryResults.get(slug);
 
     // The JSON summary (pcall result) is the primary success signal.
     // Jest failure detection provides a secondary override.
+    //
+    // Gate on attributed output only. Unattributed output is shown but not
+    // judged: whatever broke attribution is reason enough not to trust which
+    // package a failure line belongs to.
     let success = summarySuccess ?? false;
-    if (success && sectionLogs) {
-      const hasJestFailures = _hasJestFailuresInLogs(sectionLogs);
+    let error: string | undefined;
+    if (success && attributedLogs) {
+      const hasJestFailures = _hasJestFailuresInLogs(attributedLogs);
       if (hasJestFailures) {
         success = false;
+        error = 'Jest reported failing tests';
       }
     }
 
+    // A pass we cannot read is not a pass. The pcall only proves the script did
+    // not throw, which says nothing about whether any test ran or passed.
+    if (success && !attributedLogs) {
+      success = false;
+      error =
+        `No test output could be attributed to this package ` +
+        `(${rawLogs.length} chars received, ${beginMarkersSeen} BEGIN markers found). ` +
+        'Unattributed output follows.';
+    }
+
     if (!success && !noOutputAtAll) {
       console.error(
         `[batch-log-parser] ${slug}: summarySuccess=${summarySuccess} hasLogs=${
@@ -172,7 +234,19 @@ export function parseBatchTestLogs(
       );
     }
 
-    const testCounts = sectionLogs ? parseTestCounts(sectionLogs) : undefined;
+    // Deliberately not parsed from unattributed output: attribution is exactly
+    // what failed, so a count read from it could belong to anything. The logs
+    // are still shown, so the jest summary remains readable by eye.
+    const testCounts = attributedLogs
+      ? parseTestCounts(attributedLogs)
+      : undefined;
+    const tracebackCount = countTracebacks(sectionLogs);
+    if (tracebackCount > 0) {
+      OutputHelper.warn(
+        `[batch-log-parser] ${slug}: ${tracebackCount} Luau traceback(s) in output. ` +
+          'Jest does not count these — a deferred-callback crash fires outside any test.'
+      );
+    }
     // Prefer the JSON summary (authoritative, immune to log reordering);
     // fall back to the END-marker value if the summary was truncated.
     const durationMs = summaryDurations.get(slug) ?? markerDurations.get(slug);
@@ -182,12 +256,29 @@ export function parseBatchTestLogs(
       logs: sectionLogs,
       durationMs,
       testCounts,
+      tracebackCount,
+      error,
     });
   }
 
   return results;
 }
 
+/**
+ * Count Luau tracebacks in log output.
+ *
+ * Reported separately from test counts because jest never sees these: a crash in
+ * a deferred callback fires outside any test, so a run can print
+ * "Tests: 155 passed, 0 failed" while something is genuinely broken.
+ */
+export function countTracebacks(rawOutput: string): number {
+  if (!rawOutput) {
+    return 0;
+  }
+  const cleanLogs = OutputHelper.stripAnsi(rawOutput);
+  return cleanLogs.match(/Stack Begin\s/g)?.length ?? 0;
+}
+
 /**
  * Check specifically for Jest test suite/test failures in logs.
  * Unlike parseTestLogs, this ignores "Stack Begin" runtime errors since

package/src/utils/testing/reporting/test-github-columns.ts

@@ -19,6 +19,7 @@ export function createTestCommentConfig(): GithubCommentTableConfig {
     sectionId: 'tests',
     extraColumns: createTestColumns(),
     errorHeading: 'Test Results',
+    expectsTestCounts: true,
   };
 }
 

package/src/utils/testing/runner/test-runner.ts

@@ -26,6 +26,8 @@ export interface SingleTestResult {
    * non-aggregated cloud/local runs — callers should fall back to wall-clock.
    */
   durationMs?: number;
+  /** Why the run failed, when the runner can say more than "it failed". */
+  error?: string;
 }
 
 export interface SingleTestOptions {
@@ -109,6 +111,7 @@ export async function runSingleTestAsync(
       logs: parsed.logs,
       testCounts: parseTestCounts(parsed.logs),
       durationMs: result.durationMs,
+      error: result.errorMessage,
     };
   } finally {
     await context.releaseAsync(deployment);