@quenty/nevermore-cli 4.43.4 → 4.44.0

package/src/utils/testing/runner/test-runner.test.ts

@@ -1,6 +1,290 @@
-import { describe, expect, it } from 'vitest';
+import * as fs from 'fs/promises';
+import * as os from 'os';
+import * as path from 'path';
+import {
+  afterAll,
+  afterEach,
+  beforeAll,
+  describe,
+  expect,
+  it,
+  vi,
+} from 'vitest';
+import { type DeployTarget } from '@quenty/nevermore-deploy';
+import { OutputHelper } from '@quenty/cli-output-helpers';
 
-import { mergeFailureReasons } from './test-runner.js';
+import { mergeFailureReasons, runSingleTestAsync } from './test-runner.js';
+import { TEST_RESULTS_FORMAT } from '../structured-test-results.js';
+import {
+  type Deployment,
+  type JobContext,
+  type ScriptRunResult,
+} from '../../job-context/job-context.js';
+
+const PASSING_LOGS = [
+  'Test Suites: 3 passed, 3 total',
+  'Tests:       25 passed, 25 total',
+].join('\n');
+
+function structuredResults(overrides: Record<string, unknown> = {}) {
+  return {
+    format: TEST_RESULTS_FORMAT,
+    success: true,
+    ranJest: true,
+    passed: 25,
+    failed: 0,
+    skipped: 0,
+    total: 25,
+    suitesPassed: 3,
+    suitesFailed: 0,
+    suitesTotal: 3,
+    failures: [],
+    omittedFailures: 0,
+    ...overrides,
+  };
+}
+
+/** Collect what the runner said out loud, so silence can be asserted on. */
+function captureWarnings(): () => string[] {
+  const warnings: string[] = [];
+  vi.spyOn(OutputHelper, 'warn').mockImplementation((message: string) => {
+    warnings.push(message);
+  });
+  vi.spyOn(OutputHelper, 'info').mockImplementation(() => {});
+  return () => warnings;
+}
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+/**
+ * A context that runs nothing: it hands back the run result and logs the test
+ * wants to reason about. Everything else is the minimum the runner touches.
+ */
+function createContext(run: ScriptRunResult, logs: string): JobContext {
+  return {
+    buildPlaceAsync: async (options) => ({
+      rbxlPath: 'unused.rbxl',
+      target: options.target,
+    }),
+    deployBuiltPlaceAsync: async () => ({} as Deployment),
+    runScriptAsync: async () => run,
+    getLogsAsync: async () => ({ text: logs, messages: [] }),
+    releaseAsync: async () => {},
+    releaseBuiltPlaceAsync: async () => {},
+    disposeAsync: async () => {},
+  };
+}
+
+describe('runSingleTestAsync', () => {
+  let packagePath: string;
+
+  beforeAll(async () => {
+    packagePath = await fs.mkdtemp(path.join(os.tmpdir(), 'nevermore-test-'));
+    await fs.writeFile(
+      path.join(packagePath, 'ServerMain.server.lua'),
+      'return nil\n'
+    );
+  });
+
+  afterAll(async () => {
+    await fs.rm(packagePath, { recursive: true, force: true });
+  });
+
+  async function runAsync(run: ScriptRunResult, logs: string) {
+    return runSingleTestAsync(createContext(run, logs), {
+      packagePath,
+      packageName: 'maid',
+      target: {
+        scriptTemplate: 'ServerMain.server.lua',
+      } as unknown as DeployTarget,
+    });
+  }
+
+  it('fails a run whose returned results say it failed', async () => {
+    // The runner used to announce this by throwing, which failed the task. Now
+    // it returns the verdict, and these logs are what a truncated window leaves
+    // behind: a jest report from a suite that passed and no sign of the one
+    // that did not.
+    const result = await runAsync(
+      {
+        success: true,
+        returnValues: [structuredResults({ success: false, failed: 2 })],
+      },
+      PASSING_LOGS
+    );
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('2 test(s) failed');
+  });
+
+  it('reports the counts the run returned, not the ones in the log', async () => {
+    const result = await runAsync(
+      {
+        success: true,
+        returnValues: [structuredResults({ passed: 400, total: 400 })],
+      },
+      PASSING_LOGS
+    );
+
+    expect(result.success).toBe(true);
+    expect(result.testCounts).toEqual({ passed: 400, failed: 0, total: 400 });
+  });
+
+  it('accepts returned results as proof the runner ran', async () => {
+    // Demanding a jest report in the logs is a stand-in for that proof, and it
+    // is the first thing a truncated log window costs.
+    const result = await runAsync(
+      { success: true, returnValues: [structuredResults()] },
+      '(logs truncated)'
+    );
+
+    expect(result.success).toBe(true);
+  });
+
+  it('still fails a run with a traceback in its logs', async () => {
+    // Jest cannot count a deferred-callback crash, so passing results are no
+    // reason to stop reading the logs.
+    const result = await runAsync(
+      { success: true, returnValues: [structuredResults()] },
+      `${PASSING_LOGS}\nStack Begin\nScript 'maid.spec', Line 4\nStack End`
+    );
+
+    expect(result.success).toBe(false);
+  });
+
+  it('falls back to the logs for a script that returned nothing', async () => {
+    // A test script written before results were returned. It must keep working,
+    // and it must not start passing when its logs say otherwise.
+    const failing = await runAsync(
+      { success: true, returnValues: [] },
+      'Tests:       2 failed, 23 passed, 25 total'
+    );
+    expect(failing.success).toBe(false);
+    expect(failing.error).toContain('2 test(s) failed');
+
+    const passing = await runAsync(
+      { success: true, returnValues: [] },
+      PASSING_LOGS
+    );
+    expect(passing.success).toBe(true);
+    expect(passing.testCounts).toEqual({ passed: 25, failed: 0, total: 25 });
+  });
+
+  it('fails a script that returned nothing and logged nothing', async () => {
+    const result = await runAsync(
+      { success: true, returnValues: undefined },
+      ''
+    );
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('nothing proves any test ran');
+  });
+
+  it('records where the counts came from', async () => {
+    const returned = await runAsync(
+      { success: true, returnValues: [structuredResults()] },
+      PASSING_LOGS
+    );
+    expect(returned.countsSource).toBe('returned');
+
+    const scraped = await runAsync(
+      { success: true, returnValues: [] },
+      PASSING_LOGS
+    );
+    expect(scraped.countsSource).toBe('scraped');
+  });
+
+  it('warns out loud when the run returned no results', async () => {
+    // Silence here is what let the structured channel ship inert: a run that
+    // fell back to scraping produced output identical to one that did not.
+    const warnings = captureWarnings();
+
+    await runAsync({ success: true, returnValues: [] }, PASSING_LOGS);
+
+    expect(warnings().some((w) => w.includes('returned no test results'))).toBe(
+      true
+    );
+  });
+
+  it('fails a package with specs whose run never reached jest', async () => {
+    // A Rojo regression that stops shipping jest.config turns a real suite into
+    // a smoke test. Before results were returned, the required jest report caught
+    // that; a smoke-test result retires the report and would otherwise pass with
+    // zero tests.
+    await fs.mkdir(path.join(packagePath, 'src'), { recursive: true });
+    await fs.writeFile(
+      path.join(packagePath, 'src', 'jest.config.lua'),
+      'return {}\n'
+    );
+
+    const result = await runAsync(
+      {
+        success: true,
+        returnValues: [
+          structuredResults({ ranJest: false, passed: 0, total: 0 }),
+        ],
+      },
+      '[NevermoreTestRunner] No jest.config found — smoke test passed (boot success)'
+    );
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('none of its specs ran');
+
+    await fs.rm(path.join(packagePath, 'src'), {
+      recursive: true,
+      force: true,
+    });
+  });
+
+  it('passes a package with no specs that smoke tested', async () => {
+    // No jest.config on disk, so a smoke test is the whole contract.
+    const result = await runAsync(
+      {
+        success: true,
+        returnValues: [
+          structuredResults({ ranJest: false, passed: 0, total: 0 }),
+        ],
+      },
+      '[NevermoreTestRunner] No jest.config found — smoke test passed (boot success)'
+    );
+
+    expect(result.success).toBe(true);
+  });
+
+  it('takes results a context resolved for it, without re-reporting', async () => {
+    // Aggregated batch mode: one execution covers every package, so the batch
+    // log parser splits the per-package results out and has already reported
+    // provenance for the whole batch.
+    const warnings = captureWarnings();
+
+    const result = await runAsync(
+      {
+        success: true,
+        testResults: {
+          success: false,
+          ranJest: true,
+          passed: 273,
+          failed: 2,
+          skipped: 0,
+          total: 275,
+          suitesPassed: 8,
+          suitesFailed: 1,
+          suitesTotal: 9,
+          failures: [],
+          omittedFailures: 0,
+        },
+      },
+      '(this package’s section was truncated away)'
+    );
+
+    expect(result.success).toBe(false);
+    expect(result.countsSource).toBe('returned');
+    expect(result.testCounts).toEqual({ passed: 273, failed: 2, total: 275 });
+    expect(warnings()).toHaveLength(0);
+  });
+});
 
 describe('mergeFailureReasons', () => {
   it('does not repeat reasons the context already reported', () => {
@@ -27,3 +311,49 @@ describe('mergeFailureReasons', () => {
     expect(mergeFailureReasons(undefined, [])).toEqual([]);
   });
 });
+
+describe('runSingleTestAsync log volume reporting', () => {
+  let packagePath: string;
+
+  beforeAll(async () => {
+    packagePath = await fs.mkdtemp(path.join(os.tmpdir(), 'volume-'));
+    await fs.writeFile(
+      path.join(packagePath, 'ServerMain.server.lua'),
+      'return nil\n'
+    );
+  });
+
+  afterAll(async () => {
+    await fs.rm(packagePath, { recursive: true, force: true });
+  });
+
+  function runAsync(logs: string) {
+    return runSingleTestAsync(createContext({ success: true }, logs), {
+      packagePath,
+      packageName: 'gameconfig',
+      target: {
+        scriptTemplate: 'ServerMain.server.lua',
+      } as unknown as DeployTarget,
+    });
+  }
+
+  it('reports how much log text a verdict was read from', async () => {
+    const result = await runAsync('some output with no jest report');
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('verdict read from log text alone');
+    expect(result.error).toContain('31 chars, 1 line(s) received');
+  });
+
+  it('says nothing about log volume when there was no log text', async () => {
+    // A batch package whose section was lost has no text of its own. Reporting
+    // "0 chars received" here contradicts the fetch stats already printed for
+    // the whole batch window, and describes a judgement never made: the verdict
+    // came from the output being absent, which its own reason already says.
+    const result = await runAsync('');
+
+    expect(result.success).toBe(false);
+    expect(result.error).not.toContain('verdict read from log text alone');
+    expect(result.error).not.toContain('0 chars, 0 line(s)');
+  });
+});

package/src/utils/testing/runner/test-runner.ts

@@ -8,6 +8,15 @@ import {
   parseTestLogs,
   parseTestCounts,
 } from '../test-log-parser.js';
+import { describeLogVolume } from '../log-fetch-stats.js';
+import {
+  type StructuredTestResults,
+  describeUnexplainedVerdict,
+  findStructuredTestResults,
+  structuredFailureReasons,
+  toParsedTestCounts,
+} from '../structured-test-results.js';
+import { OutputHelper } from '@quenty/cli-output-helpers';
 import {
   buildDeployMetadataAttributes,
   gatherGitDeployInfo,
@@ -28,6 +37,14 @@ export interface SingleTestResult {
   durationMs?: number;
   /** Why the run failed, when the runner can say more than "it failed". */
   error?: string;
+  /**
+   * Where `testCounts` came from. `returned` means the run handed them over as a
+   * value; `scraped` means they were read out of log text, which is the channel
+   * truncation destroys. Absent when there are no counts at all.
+   */
+  countsSource?: 'returned' | 'scraped';
+  /** True when the verdict rests on returned results with no log to read. */
+  logsLost?: boolean;
 }
 
 /**
@@ -60,6 +77,12 @@ export interface SingleTestOptions {
   timeoutMs?: number;
   /** Luau code to execute directly, bypassing the configured scriptTemplate. */
   scriptText?: string;
+  /**
+   * Skip this run's counts-provenance line, because the caller reports it for
+   * every package at once. Set by an aggregated batch, where one fetch serves
+   * the whole run and the per-package line says the same thing each time.
+   */
+  suppressCountsProvenance?: boolean;
 }
 
 /**
@@ -125,31 +148,230 @@ export async function runSingleTestAsync(
       timeoutMs,
     });
 
-    const rawLogs = await context.getLogsAsync(deployment);
+    const logs = await context.getLogsAsync(deployment);
+    const rawLogs = logs.text;
+    const logVolume = describeLogVolume(rawLogs, logs.stats);
+
+    // The runner used to announce a failing suite by throwing, which failed the
+    // task. It returns its verdict now, so the verdict has to be read: without
+    // this, a failing suite whose report fell outside the truncated log window
+    // would come back a pass.
+    //
+    // A context that already resolved the results (aggregated batch, where one
+    // execution's return value belongs to no single package) hands them over
+    // directly; otherwise they are decoded from what the script returned.
+    const structured =
+      result.testResults ?? findStructuredTestResults(result.returnValues);
+
     // A probe script is arbitrary Luau with no jest in it, so demanding a test
     // report would fail every --script-text run. Everything else must prove a
-    // runner spoke before it can pass.
+    // runner spoke before it can pass — and returned results are that proof,
+    // where a scraped report is only what survived truncation.
     const parsed = parseTestLogs(rawLogs, {
-      requireTestReport: scriptText === undefined,
+      requireTestReport: scriptText === undefined && structured === undefined,
     });
 
-    const reasons = mergeFailureReasons(
-      result.errorMessage,
-      parsed.failureReasons
-    );
+    // A run that reports it never reached jest, for a package that has a
+    // jest.config on disk, did not test anything — the built place lost the
+    // config. Before results were returned, the required jest report caught this
+    // as "nothing proves any test ran"; a smoke-test result would otherwise
+    // retire that check and pass with zero tests.
+    const smokeTestedWithSpecs =
+      structured !== undefined &&
+      !structured.ranJest &&
+      scriptText === undefined &&
+      (await packageHasJestConfigAsync(packagePath));
+
+    const reasons = mergeFailureReasons(result.errorMessage, [
+      ...(smokeTestedWithSpecs
+        ? [
+            'this package has a jest.config but the run reported no jest.config ' +
+              'in the built place, so none of its specs ran',
+          ]
+        : []),
+      ...(structured ? structuredFailureReasons(structured) : []),
+      ...parsed.failureReasons,
+      // Said only when there was log text and it did not hold up: the reasons
+      // above are then statements about what arrived, and how much arrived is
+      // the first thing to check. With no text at all the verdict came from the
+      // absence of it, which the reason for that absence already explains —
+      // saying "0 chars received" beside it only contradicts the fetch that
+      // reported the whole window.
+      ...(structured === undefined &&
+      parsed.failureReasons.length > 0 &&
+      rawLogs.length > 0
+        ? [`verdict read from log text alone (${logVolume})`]
+        : []),
+    ]);
+
+    // Returned counts outrank scraped ones: same numbers when the log
+    // survived, real numbers when it did not.
+    const scrapedCounts = parseTestCounts(parsed.logs);
+    const testCounts = structured
+      ? toParsedTestCounts(structured)
+      : scrapedCounts;
+
+    reportCountsProvenance({
+      packageName,
+      structured,
+      scrapedCounts,
+      logVolume,
+      // A probe is not a test run, so it has no results to be missing. Neither
+      // is one package of an aggregated batch, whose context reports the whole
+      // batch's provenance once, as a group — repeating it per package printed
+      // the same fetch stats 78 times over.
+      silent:
+        scriptText !== undefined ||
+        result.testResults !== undefined ||
+        (options.suppressCountsProvenance ?? false),
+      hadReturnChannel: result.returnValues !== undefined,
+    });
 
     return {
-      success: result.success && parsed.success,
+      success:
+        result.success &&
+        parsed.success &&
+        (structured?.success ?? true) &&
+        !smokeTestedWithSpecs,
       logs: parsed.logs,
-      testCounts: parseTestCounts(parsed.logs),
+      testCounts,
+      countsSource: structured
+        ? 'returned'
+        : scrapedCounts
+        ? 'scraped'
+        : undefined,
       durationMs: result.durationMs,
       error: reasons.length > 0 ? reasons.join('; ') : undefined,
+      logsLost: result.logsLost,
     };
   } finally {
     await context.releaseAsync(deployment);
   }
 }
 
+/**
+ * Say where a run's counts came from, and complain when they had to be scraped.
+ *
+ * The structured channel exists because Open Cloud truncates a long run's logs.
+ * A channel that is plumbed but not flowing produces output identical to one
+ * that works, so the fallback is a warning rather than silence — the first
+ * version of this shipped inert and looked green.
+ */
+function reportCountsProvenance(options: {
+  packageName: string;
+  structured?: StructuredTestResults;
+  scrapedCounts?: ParsedTestCounts;
+  silent: boolean;
+  hadReturnChannel: boolean;
+  /** How much log text there was to scrape, for the fallback warning. */
+  logVolume: string;
+}): void {
+  const {
+    packageName,
+    structured,
+    scrapedCounts,
+    silent,
+    hadReturnChannel,
+    logVolume,
+  } = options;
+
+  if (silent) {
+    return;
+  }
+
+  if (structured) {
+    OutputHelper.info(
+      `${packageName}: counts returned by the run — ` +
+        `${structured.passed} passed, ${structured.failed} failed, ` +
+        `${structured.total} total.`
+    );
+
+    const unexplained = describeUnexplainedVerdict(structured);
+    if (unexplained) {
+      OutputHelper.warn(`${packageName}: ${unexplained}`);
+    }
+
+    // A suite that runs and counts nothing is not the same as one that passed.
+    if (structured.ranJest && structured.total === 0) {
+      OutputHelper.warn(
+        `${packageName}: jest ran but found no tests to run. If this package has ` +
+          `specs, they are not reaching the test place.`
+      );
+    }
+
+    // Two channels reporting the same run must agree. When they do not, one is
+    // lying and neither total stands on its own.
+    if (scrapedCounts && scrapedCounts.total !== structured.total) {
+      OutputHelper.warn(
+        `${packageName}: returned counts disagree with the log — the run returned ` +
+          `${structured.passed}/${structured.failed}/${structured.total} ` +
+          `(passed/failed/total), its jest report says ` +
+          `${scrapedCounts.passed}/${scrapedCounts.failed}/${scrapedCounts.total}. ` +
+          `Reporting the returned counts; one of the two channels is wrong.`
+      );
+    }
+    return;
+  }
+
+  OutputHelper.warn(
+    `${packageName}: the run returned no test results, so its counts were ` +
+      (scrapedCounts ? 'scraped from log text' : 'unavailable') +
+      ` — the channel Open Cloud truncates on long runs (${logVolume}). ` +
+      (hadReturnChannel
+        ? 'The run delivered a return channel but nothing recognizable in it: the ' +
+          'test script should end with "return results" (see docs/testing/testing.md).'
+        : 'No return channel was delivered at all, so the transport lost it.')
+  );
+}
+
+/**
+ * Whether this package ships a jest.config, i.e. whether it has specs to run.
+ *
+ * Deliberately a bounded walk, not a recursive glob: every package under `src/`
+ * has a symlinked, self-referential `node_modules`, so recursive search here
+ * hangs. Two levels covers the layouts in use (`src/jest.config.lua` for a
+ * package, `src/modules/jest.config.lua` for a game) without needing a list of
+ * them.
+ */
+export async function packageHasJestConfigAsync(
+  packagePath: string
+): Promise<boolean> {
+  async function scanAsync(dir: string, depth: number): Promise<boolean> {
+    let entries;
+    try {
+      entries = await fs.readdir(dir, { withFileTypes: true });
+    } catch {
+      return false;
+    }
+
+    for (const entry of entries) {
+      if (entry.isFile() && entry.name.startsWith('jest.config')) {
+        return true;
+      }
+    }
+
+    if (depth === 0) {
+      return false;
+    }
+
+    for (const entry of entries) {
+      if (!entry.isDirectory()) {
+        continue;
+      }
+      // `test` holds the test place, never the specs the place runs.
+      if (entry.name === 'node_modules' || entry.name === 'test') {
+        continue;
+      }
+      if (await scanAsync(path.join(dir, entry.name), depth - 1)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  return scanAsync(packagePath, 2);
+}
+
 /**
  * Read a test script from the deploy target's configured script path.
  */