@quenty/nevermore-cli 4.41.1 → 4.42.0

package/dist/utils/testing/test-log-parser.test.js

@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest';
+import { evaluateTestOutcome, hasTestReport } from './test-log-parser.js';
+describe('evaluateTestOutcome', () => {
+    it('passes a clean jest report', () => {
+        const outcome = evaluateTestOutcome('Tests: 275 passed, 275 total', {
+            requireTestReport: true,
+        });
+        expect(outcome.success).toBe(true);
+        expect(outcome.failureReasons).toEqual([]);
+    });
+    it('refuses to pass output with no jest report in it', () => {
+        // Absence of failure was previously read as success, so an empty log passed.
+        const outcome = evaluateTestOutcome('some unrelated chatter', {
+            requireTestReport: true,
+        });
+        expect(outcome.success).toBe(false);
+        expect(outcome.failureReasons.join()).toContain('no jest report');
+    });
+    it('exempts probe runs, which have no jest in them at all', () => {
+        const outcome = evaluateTestOutcome('hello from a probe script');
+        expect(outcome.success).toBe(true);
+    });
+    it('fails on a traceback even when every test passed', () => {
+        // Jest cannot see a deferred-callback crash: it fires outside any test.
+        const logs = [
+            'Tests: 155 passed, 155 total',
+            'attempt to index nil',
+            'Stack Begin',
+            "Script 'ServerScriptService.egghunt2026.Foo', Line 12",
+            'Stack End',
+        ].join('\n');
+        const outcome = evaluateTestOutcome(logs, { requireTestReport: true });
+        expect(outcome.success).toBe(false);
+        expect(outcome.failureReasons.join()).toContain('traceback');
+    });
+    it('reports every reason a run failed, not just the first', () => {
+        const logs = [
+            'Tests: 19 failed, 470 passed, 516 total',
+            'Stack Begin',
+            "Script 'ServerScriptService.egghunt2026.Foo', Line 12",
+            'Stack End',
+        ].join('\n');
+        const outcome = evaluateTestOutcome(logs, { requireTestReport: true });
+        expect(outcome.failureReasons).toHaveLength(2);
+    });
+});
+describe('hasTestReport', () => {
+    it('accepts a runner that ran and found nothing', () => {
+        // Packages without specs are legitimate; passWithNoTests depends on this.
+        expect(hasTestReport('No tests found, exiting with code 0')).toBe(true);
+    });
+    it('rejects silence', () => {
+        expect(hasTestReport('')).toBe(false);
+    });
+});
+//# sourceMappingURL=test-log-parser.test.js.map

package/dist/utils/testing/test-log-parser.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-log-parser.test.js","sourceRoot":"","sources":["../../../src/utils/testing/test-log-parser.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9C,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1E,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,OAAO,GAAG,mBAAmB,CAAC,8BAA8B,EAAE;YAClE,iBAAiB,EAAE,IAAI;SACxB,CAAC,CAAC;QAEH,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,6EAA6E;QAC7E,MAAM,OAAO,GAAG,mBAAmB,CAAC,wBAAwB,EAAE;YAC5D,iBAAiB,EAAE,IAAI;SACxB,CAAC,CAAC;QAEH,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,OAAO,GAAG,mBAAmB,CAAC,2BAA2B,CAAC,CAAC;QAEjE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,wEAAwE;QACxE,MAAM,IAAI,GAAG;YACX,8BAA8B;YAC9B,sBAAsB;YACtB,aAAa;YACb,uDAAuD;YACvD,WAAW;SACZ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,EAAE,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,IAAI,GAAG;YACX,yCAAyC;YACzC,aAAa;YACb,uDAAuD;YACvD,WAAW;SACZ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,EAAE,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvE,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,0EAA0E;QAC1E,MAAM,CAAC,aAAa,CAAC,qCAAqC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iBAAiB,EAAE,GAAG,EAAE;QACzB,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@quenty/nevermore-cli",
-  "version": "4.41.1",
+  "version": "4.42.0",
   "description": "CLI interface for Nevermore",
   "type": "module",
   "keywords": [
@@ -57,5 +57,5 @@
   "engines": {
     "node": ">=16"
   },
-  "gitHead": "35993999cfe6fbdfd44721a1835b6ee29602f057"
+  "gitHead": "481392a58dedd5ee476d66c90c14e2f429e23df8"
 }

package/src/commands/batch-command/batch-test-command.ts

@@ -60,64 +60,84 @@ export const batchTestCommand: CommandModule<
   command: 'test',
   describe: 'Run tests for changed packages with test targets',
   builder: (yargs) => {
-    return yargs
-      .option('cloud', {
-        describe: 'Run tests via Open Cloud instead of locally',
-        type: 'boolean',
-        default: false,
-      })
-      .option('api-key', {
-        describe: 'Roblox Open Cloud API key (--cloud only)',
-        type: 'string',
-      })
-      .option('all', {
-        describe: 'Test all packages with test targets, not just changed',
-        type: 'boolean',
-        default: false,
-      })
-      .option('base', {
-        describe: 'Git ref to diff against for change detection',
-        type: 'string',
-        default: 'origin/main',
-      })
-      .option('concurrency', {
-        describe: 'Max parallel tests (0 = unlimited, default: unlimited)',
-        type: 'number',
-      })
-      .option('output', {
-        describe: 'Write JSON results to this file',
-        type: 'string',
-      })
-      .option('limit', {
-        describe: 'Max number of packages to test (for local debugging)',
-        type: 'number',
-      })
-      .option('logs', {
-        describe: 'Show execution logs for all packages (not just failures)',
-        type: 'boolean',
-        default: false,
-      })
-      .option('aggregated', {
-        describe:
-          'Build all packages into a single place and execute one batch script',
-        type: 'boolean',
-        default: true,
-      })
-      .option('batch-place-id', {
-        describe:
-          'Override placeId for the aggregated batch upload (--aggregated only)',
-        type: 'number',
-      })
-      .option('batch-universe-id', {
-        describe:
-          'Override universeId for the aggregated batch upload (--aggregated only)',
-        type: 'number',
-      })
-      .option('timeout', {
-        describe:
-          'Max script execution time in seconds for the whole batch. Sent to the Open Cloud API so Roblox cancels server-side on overrun. Max 300s (API limit). (default: 300)',
-        type: 'number',
-      });
+    return (
+      yargs
+        .option('cloud', {
+          describe: 'Run tests via Open Cloud instead of locally',
+          type: 'boolean',
+          default: false,
+        })
+        .option('api-key', {
+          describe: 'Roblox Open Cloud API key (--cloud only)',
+          type: 'string',
+        })
+        .option('all', {
+          describe: 'Test all packages with test targets, not just changed',
+          type: 'boolean',
+          default: false,
+        })
+        .option('base', {
+          describe: 'Git ref to diff against for change detection',
+          type: 'string',
+          default: 'origin/main',
+        })
+        .option('concurrency', {
+          describe: 'Max parallel tests (0 = unlimited, default: unlimited)',
+          type: 'number',
+        })
+        .option('output', {
+          describe: 'Write JSON results to this file',
+          type: 'string',
+        })
+        .option('limit', {
+          describe: 'Max number of packages to test (for local debugging)',
+          type: 'number',
+        })
+        .option('logs', {
+          describe: 'Show execution logs for all packages (not just failures)',
+          type: 'boolean',
+          default: false,
+        })
+        .option('aggregated', {
+          describe:
+            'Build all packages into a single place and execute one batch script',
+          type: 'boolean',
+          default: true,
+        })
+        .option('batch-place-id', {
+          describe:
+            'Override placeId for the aggregated batch upload (--aggregated only)',
+          type: 'number',
+        })
+        .option('batch-universe-id', {
+          describe:
+            'Override universeId for the aggregated batch upload (--aggregated only)',
+          type: 'number',
+        })
+        .option('timeout', {
+          describe:
+            'Max script execution time in seconds for the whole batch. Sent to the Open Cloud API so Roblox cancels server-side on overrun. Max 300s (API limit). (default: 300)',
+          type: 'number',
+        })
+        // Accepted only to fail with a useful message. A batch runs one shared
+        // script across packages, so there is nothing for arbitrary Luau to mean
+        // here — but yargs strict mode would otherwise reject it with a bare
+        // "Unknown argument" that does not say where to go instead.
+        .option('script-text', {
+          hidden: true,
+          type: 'string',
+        })
+        .check((args) => {
+          if (args.scriptText !== undefined) {
+            throw new Error(
+              '--script-text is not supported by "batch test", which runs one shared script for all packages.\n' +
+                'Run it against a single package instead:\n' +
+                '  cd <package> && nevermore test --cloud --script-text \'print("hi")\''
+            );
+          }
+          return true;
+        })
+    );
   },
   handler: async (args) => {
     try {

package/src/commands/test-command/test-command.ts

@@ -56,9 +56,9 @@ export class TestProjectCommand<T>
       type: 'string',
     });
     args.option('logs', {
-      describe: 'Show execution logs',
+      describe:
+        'Show execution logs. Defaults on with --script-text, whose output is the whole point of the run',
       type: 'boolean',
-      default: false,
     });
     args.option('universe-id', {
       describe:
@@ -84,7 +84,7 @@ export class TestProjectCommand<T>
     });
     args.option('timeout', {
       describe:
-        'Max script execution time in seconds. Sent to the Open Cloud API so Roblox cancels server-side on overrun (default: 120)',
+        'Max script execution time in seconds. Sent to the Open Cloud API so Roblox cancels server-side on overrun. Max 300s (API limit). (default: 300)',
       type: 'number',
     });
 
@@ -94,7 +94,10 @@ export class TestProjectCommand<T>
   public handler = async (args: TestProjectArgs) => {
     const cwd = process.cwd();
     const packageName = (await readPackageNameAsync(cwd)) ?? path.basename(cwd);
-    const showLogs = args.logs ?? false;
+    // A probe run exists to show its output. Hiding it behind an opt-in flag
+    // made --script-text look broken enough that a workaround was written into
+    // the docs instead of a bug report.
+    const showLogs = args.logs ?? args.scriptText !== undefined;
     const useSpinner = process.stdout.isTTY && !args.verbose;
 
     const reporter = new CompositeReporter(

package/src/utils/open-cloud/open-cloud-client.ts

@@ -339,6 +339,7 @@ export class OpenCloudClient {
     const structured: { message: string; createTime: string }[] = [];
     let pageToken: string | undefined;
     let pagesFetched = 0;
+    let entriesFetched = 0;
 
     do {
       const url = new URL(`https://apis.roblox.com/cloud/v2/${taskPath}/logs`);
@@ -386,6 +387,7 @@ export class OpenCloudClient {
         }
       }
 
+      entriesFetched += (data.luauExecutionSessionTaskLogs ?? []).length;
       pagesFetched++;
       pageToken = data.nextPageToken || undefined;
     } while (pageToken);
@@ -402,7 +404,8 @@ export class OpenCloudClient {
     // the task and the parser, and the parser cannot tell a short run from a
     // truncated fetch. Record what arrived so a real run can settle it.
     OutputHelper.verbose(
-      `[open-cloud] Task logs: ${pagesFetched} page(s), ${messages.length} messages, ${text.length} chars`
+      `[open-cloud] Task logs: ${pagesFetched} page(s), ${entriesFetched} entries, ` +
+        `${messages.length} messages, ${text.length} chars`
     );
 
     return text;

package/src/utils/testing/parsers/batch-log-parser.test.ts

@@ -1,6 +1,10 @@
 import { describe, expect, it } from 'vitest';
 
-import { countTracebacks, parseBatchTestLogs } from './batch-log-parser.js';
+import {
+  countTracebacks,
+  findSummaryEntries,
+  parseBatchTestLogs,
+} from './batch-log-parser.js';
 
 const SLUG_MAP = new Map([['egghunt2026', 'egghunt2026']]);
 
@@ -23,10 +27,11 @@ describe('parseBatchTestLogs', () => {
     expect(result?.testCounts).toEqual({ passed: 275, failed: 0, total: 275 });
   });
 
-  it('fails a pass it cannot read, rather than trusting the pcall', () => {
-    // The summary prints last, so it survives a log window that dropped the
-    // head. The pcall said success, but nothing here saw a test run.
+  it('attributes a section closed by END when its BEGIN was dropped', () => {
+    // Open Cloud keeps only the tail of a long run's log, so BEGIN — printed
+    // first — is the first casualty while END and the summary survive.
     const logs = [
+      '  ✓ some test that survived (3 ms)',
       'Tests:  275 passed, 275 total',
       '===BATCH_TEST_END egghunt2026 PASS 1000===',
       '===BATCH_TEST_SUMMARY===',
@@ -35,8 +40,48 @@ describe('parseBatchTestLogs', () => {
 
     const result = parseBatchTestLogs(logs, SLUG_MAP).get('egghunt2026');
 
+    expect(result?.success).toBe(true);
+    expect(result?.testCounts).toEqual({ passed: 275, failed: 0, total: 275 });
+  });
+
+  it('does not let an out-of-order END steal another package’s output', () => {
+    // The API does not deliver messages in order, so a stray END can arrive
+    // mid-section. Only the first section can have lost its head; past that the
+    // log is well-formed and a BEGIN-less END must not claim anything.
+    const twoPackages = new Map([
+      ['alpha', 'alpha'],
+      ['beta', 'beta'],
+    ]);
+    const logs = [
+      '===BATCH_TEST_BEGIN alpha===',
+      'Tests:  10 passed, 10 total',
+      '===BATCH_TEST_END beta PASS 5===',
+      '===BATCH_TEST_END alpha PASS 10===',
+      '===BATCH_TEST_SUMMARY===',
+      '[{"slug":"alpha","success":true},{"slug":"beta","success":true}]',
+    ].join('\n');
+
+    const results = parseBatchTestLogs(logs, twoPackages);
+
+    expect(results.get('alpha')?.testCounts?.total).toBe(10);
+    // beta never had a section of its own; it must not inherit alpha's.
+    expect(results.get('beta')?.testCounts).toBeUndefined();
+    expect(results.get('beta')?.success).toBe(false);
+  });
+
+  it('fails a pass it cannot read, rather than trusting the pcall', () => {
+    // No END marker either, so there is no boundary to attribute output with.
+    // The pcall said success, but nothing here saw a test run.
+    const logs = [
+      'Tests:  275 passed, 275 total',
+      '===BATCH_TEST_SUMMARY===',
+      '[{"slug":"egghunt2026","success":true,"durationMs":1000}]',
+    ].join('\n');
+
+    const result = parseBatchTestLogs(logs, SLUG_MAP).get('egghunt2026');
+
     expect(result?.success).toBe(false);
-    expect(result?.error).toContain('No test output could be attributed');
+    expect(result?.error).toContain('no output could be attributed');
     expect(result?.testCounts).toBeUndefined();
   });
 
@@ -72,6 +117,26 @@ describe('parseBatchTestLogs', () => {
   });
 });
 
+describe('findSummaryEntries', () => {
+  it('finds the summary even when output landed after it', () => {
+    // Messages are not delivered in order, so a stray line can follow the
+    // summary. Treating the whole remainder as one JSON blob made the same
+    // batch parse on one run and fail on the next.
+    const lines = [
+      '[{"slug":"alpha","success":true}]',
+      '  ✓ a stray line that arrived late',
+    ];
+
+    expect(findSummaryEntries(lines, 0)).toEqual([
+      { slug: 'alpha', success: true },
+    ]);
+  });
+
+  it('returns undefined when there is no array to find', () => {
+    expect(findSummaryEntries(['not json at all'], 0)).toBeUndefined();
+  });
+});
+
 describe('countTracebacks', () => {
   it('counts each traceback separately', () => {
     expect(countTracebacks('Stack Begin\nfoo\nStack End\nStack Begin\n')).toBe(