@quenty/nevermore-cli 4.43.3 → 4.44.0

package/src/utils/testing/structured-test-results.test.ts

@@ -0,0 +1,267 @@
+/**
+ * Unit tests for decoding the results a test run returns. The decoder stands
+ * between a transport and a pass/fail verdict, so the cases that matter are the
+ * ones where a value is not what it claims to be.
+ */
+
+import { describe, expect, it } from 'vitest';
+
+import {
+  TEST_RESULTS_FORMAT,
+  decodeStructuredTestResults,
+  describeUnexplainedVerdict,
+  findStructuredTestResults,
+  structuredFailureReasons,
+  toParsedTestCounts,
+} from './structured-test-results.js';
+
+function results(overrides: Record<string, unknown> = {}) {
+  return {
+    format: TEST_RESULTS_FORMAT,
+    success: true,
+    ranJest: true,
+    passed: 25,
+    failed: 0,
+    skipped: 1,
+    total: 26,
+    suitesPassed: 3,
+    suitesFailed: 0,
+    suitesTotal: 3,
+    failures: [],
+    omittedFailures: 0,
+    ...overrides,
+  };
+}
+
+describe('decodeStructuredTestResults', () => {
+  it('decodes a results table', () => {
+    const decoded = decodeStructuredTestResults(results());
+
+    expect(decoded).toEqual({
+      success: true,
+      ranJest: true,
+      passed: 25,
+      failed: 0,
+      skipped: 1,
+      total: 26,
+      suitesPassed: 3,
+      suitesFailed: 0,
+      suitesTotal: 3,
+      failures: [],
+      omittedFailures: 0,
+      error: undefined,
+    });
+  });
+
+  it('ignores a value that is not tagged as results', () => {
+    // A probe run (--script-text) returns whatever it likes, and none of it is
+    // a verdict on anything.
+    expect(decodeStructuredTestResults({ success: true })).toBeUndefined();
+    expect(
+      decodeStructuredTestResults({ format: 'something-else', success: true })
+    ).toBeUndefined();
+    expect(
+      decodeStructuredTestResults('nevermore-test-results@1')
+    ).toBeUndefined();
+    expect(decodeStructuredTestResults(undefined)).toBeUndefined();
+    expect(decodeStructuredTestResults([results()])).toBeUndefined();
+  });
+
+  it('reads only an explicit true as a pass', () => {
+    // Anything else means the verdict did not survive the trip, and a lost
+    // verdict must not read as a passing one.
+    expect(
+      decodeStructuredTestResults(results({ success: 'yes' }))?.success
+    ).toBe(false);
+    expect(
+      decodeStructuredTestResults(results({ success: undefined }))?.success
+    ).toBe(false);
+  });
+
+  it('treats a malformed count as zero rather than as a number', () => {
+    const decoded = decodeStructuredTestResults(
+      results({ passed: 'lots', total: null, failed: Number.NaN })
+    );
+
+    expect(decoded?.passed).toBe(0);
+    expect(decoded?.total).toBe(0);
+    expect(decoded?.failed).toBe(0);
+  });
+
+  it('keeps only well-formed failures', () => {
+    const decoded = decodeStructuredTestResults(
+      results({
+        failures: [
+          { name: 'Maid does a thing', message: 'expected true' },
+          { name: 'Maid does another thing' },
+          { message: 'no name' },
+          'not a failure',
+        ],
+      })
+    );
+
+    expect(decoded?.failures).toEqual([
+      { name: 'Maid does a thing', message: 'expected true' },
+      { name: 'Maid does another thing', message: undefined },
+    ]);
+  });
+
+  it('accepts an empty failure list however the transport spelled it', () => {
+    // An empty Lua table marshals as an array on the bridge and can arrive as an
+    // object from the cloud.
+    expect(
+      decodeStructuredTestResults(results({ failures: {} }))?.failures
+    ).toEqual([]);
+  });
+});
+
+describe('findStructuredTestResults', () => {
+  it('finds results among other returned values', () => {
+    const found = findStructuredTestResults([
+      42,
+      'log',
+      results({ passed: 9 }),
+    ]);
+
+    expect(found?.passed).toBe(9);
+  });
+
+  it('is undefined when the script returned nothing', () => {
+    // A test script written before this convention returns nil, which is the
+    // case that has to keep falling back to the logs.
+    expect(findStructuredTestResults([])).toBeUndefined();
+    expect(findStructuredTestResults(undefined)).toBeUndefined();
+  });
+});
+
+describe('structuredFailureReasons', () => {
+  it('says nothing about a passing run', () => {
+    expect(
+      structuredFailureReasons(decodeStructuredTestResults(results())!)
+    ).toEqual([]);
+  });
+
+  it('words counts the way the log parser does, so the overlap merges', () => {
+    const reasons = structuredFailureReasons(
+      decodeStructuredTestResults(
+        results({ success: false, failed: 2, suitesFailed: 1 })
+      )!
+    );
+
+    expect(reasons).toContain('2 test(s) failed');
+    expect(reasons).toContain('1 test suite(s) failed');
+  });
+
+  it('explains a failure no count can account for', () => {
+    const reasons = structuredFailureReasons(
+      decodeStructuredTestResults(results({ success: false }))!
+    );
+
+    expect(reasons).toEqual(['the test runner reported the run as failed']);
+  });
+
+  it('carries the message the runner supplied', () => {
+    const reasons = structuredFailureReasons(
+      decodeStructuredTestResults(
+        results({
+          success: false,
+          error: '[NevermoreTestRunner] Jest run failed',
+        })
+      )!
+    );
+
+    expect(reasons).toContain('[NevermoreTestRunner] Jest run failed');
+  });
+});
+
+describe('structuredFailureReasons wording', () => {
+  it('never builds a reason out of zeros', () => {
+    // "0 test(s) and 0 test suite(s) failed" was a real failure reason once. A
+    // reason saying nothing failed is unreadable as either verdict, and it is
+    // what made the bug behind it hard to see.
+    for (const overrides of [
+      { success: false },
+      { success: false, failed: 0, suitesFailed: 0 },
+      {
+        success: false,
+        error: '[NevermoreTestRunner] the run was interrupted',
+      },
+    ]) {
+      const reasons = structuredFailureReasons(
+        decodeStructuredTestResults(results(overrides))!
+      );
+
+      expect(reasons.length).toBeGreaterThan(0);
+      for (const reason of reasons) {
+        expect(reason).not.toMatch(/\b0 test\(s\)/);
+        expect(reason).not.toMatch(/\b0 test suite\(s\)/);
+      }
+    }
+  });
+
+  it('does not restate the counts the reasons already carry', () => {
+    const reasons = structuredFailureReasons(
+      decodeStructuredTestResults(
+        results({
+          success: false,
+          failed: 2,
+          suitesFailed: 1,
+          error:
+            '[NevermoreTestRunner] 2 test(s) failed, 1 test suite(s) failed',
+        })
+      )!
+    );
+
+    expect(reasons).toEqual(['1 test suite(s) failed', '2 test(s) failed']);
+  });
+});
+
+describe('describeUnexplainedVerdict', () => {
+  it('is silent about a passing run', () => {
+    // The direction that broke: a fully passing result must read as a pass and
+    // raise nothing at all.
+    expect(
+      describeUnexplainedVerdict(decodeStructuredTestResults(results())!)
+    ).toBeUndefined();
+  });
+
+  it('is silent about a failure its counts explain', () => {
+    expect(
+      describeUnexplainedVerdict(
+        decodeStructuredTestResults(results({ success: false, failed: 2 }))!
+      )
+    ).toBeUndefined();
+    expect(
+      describeUnexplainedVerdict(
+        decodeStructuredTestResults(
+          results({ success: false, suitesFailed: 1 })
+        )!
+      )
+    ).toBeUndefined();
+  });
+
+  it('describes a failure no count supports', () => {
+    // The exact signature of a runner reading the wrong field: plausible counts,
+    // and every package failed anyway.
+    const description = describeUnexplainedVerdict(
+      decodeStructuredTestResults(
+        results({ success: false, passed: 311, failed: 0, total: 311 })
+      )!
+    );
+
+    expect(description).toContain('nothing failed');
+    expect(description).toContain('311 passed');
+  });
+});
+
+describe('toParsedTestCounts', () => {
+  it('narrows to the counts the reporters render', () => {
+    expect(toParsedTestCounts(decodeStructuredTestResults(results())!)).toEqual(
+      {
+        passed: 25,
+        failed: 0,
+        total: 26,
+      }
+    );
+  });
+});

package/src/utils/testing/structured-test-results.ts

@@ -0,0 +1,199 @@
+import { type ParsedTestCounts } from './test-log-parser.js';
+
+/**
+ * Tag NevermoreTestRunnerUtils stamps on the results table a test script
+ * returns. A test place also runs probe scripts (`--script-text`) that return
+ * whatever they like, so results are recognized by this rather than by shape.
+ */
+export const TEST_RESULTS_FORMAT = 'nevermore-test-results@1';
+
+/** One failed test, or one suite that failed before its tests could run. */
+export interface StructuredTestFailure {
+  name: string;
+  message?: string;
+}
+
+/**
+ * What a test run says about itself, returned as a value instead of printed.
+ *
+ * Open Cloud truncates a long run's logs, so counts scraped out of that text go
+ * missing on exactly the runs where they matter most. Mirrors the
+ * `TestRunResults` table in `NevermoreTestRunnerUtils`.
+ */
+export interface StructuredTestResults {
+  success: boolean;
+  /** False for a smoke test, whose counts are all zero because nothing counted. */
+  ranJest: boolean;
+  passed: number;
+  failed: number;
+  /** Pending plus todo. */
+  skipped: number;
+  total: number;
+  suitesPassed: number;
+  suitesFailed: number;
+  suitesTotal: number;
+  /** Capped by the runner; `omittedFailures` says how many did not fit. */
+  failures: StructuredTestFailure[];
+  omittedFailures: number;
+  /** Why the run failed when no individual test can say so. */
+  error?: string;
+}
+
+function readNumber(source: Record<string, unknown>, key: string): number {
+  const value = source[key];
+  return typeof value === 'number' && Number.isFinite(value) ? value : 0;
+}
+
+function readFailures(value: unknown): StructuredTestFailure[] {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+
+  const failures: StructuredTestFailure[] = [];
+  for (const entry of value) {
+    if (typeof entry !== 'object' || entry === null) {
+      continue;
+    }
+    const record = entry as Record<string, unknown>;
+    if (typeof record.name !== 'string') {
+      continue;
+    }
+    failures.push({
+      name: record.name,
+      message: typeof record.message === 'string' ? record.message : undefined,
+    });
+  }
+  return failures;
+}
+
+/**
+ * Decode a single returned value into results, or undefined if it is not one.
+ *
+ * Fields are read defensively rather than trusted: the value crossed a
+ * transport, and the two transports spell exotic Luau types differently, so a
+ * malformed field must not become a verdict.
+ */
+export function decodeStructuredTestResults(
+  value: unknown
+): StructuredTestResults | undefined {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return undefined;
+  }
+
+  const record = value as Record<string, unknown>;
+  if (record.format !== TEST_RESULTS_FORMAT) {
+    return undefined;
+  }
+
+  return {
+    // Only an explicit `true` is a pass. A results table that lost its verdict
+    // in transit must not read as one.
+    success: record.success === true,
+    ranJest: record.ranJest === true,
+    passed: readNumber(record, 'passed'),
+    failed: readNumber(record, 'failed'),
+    skipped: readNumber(record, 'skipped'),
+    total: readNumber(record, 'total'),
+    suitesPassed: readNumber(record, 'suitesPassed'),
+    suitesFailed: readNumber(record, 'suitesFailed'),
+    suitesTotal: readNumber(record, 'suitesTotal'),
+    failures: readFailures(record.failures),
+    omittedFailures: readNumber(record, 'omittedFailures'),
+    error: typeof record.error === 'string' ? record.error : undefined,
+  };
+}
+
+/**
+ * Find the results a run returned, if it returned any.
+ *
+ * `undefined` covers both "the transport delivered no return channel" and "the
+ * script returned nothing recognizable" — a caller falls back to the logs
+ * either way, which is what a test script written before this convention gets.
+ */
+export function findStructuredTestResults(
+  returnValues: unknown[] | undefined
+): StructuredTestResults | undefined {
+  if (!returnValues) {
+    return undefined;
+  }
+
+  for (const value of returnValues) {
+    const results = decodeStructuredTestResults(value);
+    if (results) {
+      return results;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Describe a verdict the run's own counts do not support, for a warning.
+ *
+ * A run reporting failure while nothing in it failed is self-contradicting, and
+ * it is precisely what a runner reading the wrong field produces: the counts
+ * stay plausible, so nothing looks wrong except that every package fails at
+ * once. That shipped once — a runner consulted jest-lua's inverted
+ * `AggregatedResult.success` and failed every passing suite.
+ *
+ * Reported, never corrected. A runner that says it failed may know something its
+ * counts cannot express (an interrupted run, a snapshot check, a reporter
+ * error), so the verdict stands and the contradiction is made loud instead.
+ */
+export function describeUnexplainedVerdict(
+  results: StructuredTestResults
+): string | undefined {
+  if (results.success || results.failed > 0 || results.suitesFailed > 0) {
+    return undefined;
+  }
+
+  return (
+    `the run reports failure but its own counts show nothing failed ` +
+    `(${results.passed} passed, 0 failed, ${results.total} total, ` +
+    `0 of ${results.suitesTotal} suite(s) failed)` +
+    (results.error
+      ? `. It says: ${results.error}`
+      : ' and gives no reason at all')
+  );
+}
+
+/** Counts in the shape the reporters already render. */
+export function toParsedTestCounts(
+  results: StructuredTestResults
+): ParsedTestCounts {
+  return {
+    passed: results.passed,
+    failed: results.failed,
+    total: results.total,
+  };
+}
+
+/**
+ * Why the runner says the run failed. Empty when it says it passed.
+ *
+ * Phrased to match the log parser's wording so that merging the two sources
+ * collapses the overlap instead of reporting every failure twice.
+ */
+export function structuredFailureReasons(
+  results: StructuredTestResults
+): string[] {
+  if (results.success) {
+    return [];
+  }
+
+  const reasons: string[] = [];
+  if (results.suitesFailed > 0) {
+    reasons.push(`${results.suitesFailed} test suite(s) failed`);
+  }
+  if (results.failed > 0) {
+    reasons.push(`${results.failed} test(s) failed`);
+  }
+
+  // The runner's own message restates those two lines when a count explains the
+  // failure, so it is only worth repeating when nothing else can say why —
+  // an interrupted run, a failed snapshot check, a result shape it could not
+  // read. That case is also the one where a reason is most needed.
+  if (reasons.length === 0) {
+    reasons.push(results.error ?? 'the test runner reported the run as failed');
+  }
+  return reasons;
+}

package/templates/batch-test-runner.lua

@@ -0,0 +1,242 @@
+-- batch-test-runner.lua
+-- Runs all test scripts sequentially in a single execution task.
+-- PACKAGE_SLUGS_JSON placeholder is replaced with a JSON array of slug strings at runtime.
+
+local CollectionService = game:GetService("CollectionService")
+local HttpService = game:GetService("HttpService")
+local ReplicatedStorage = game:GetService("ReplicatedStorage")
+local RunService = game:GetService("RunService")
+local ServerScriptService = game:GetService("ServerScriptService")
+
+-- stylua: ignore
+local packageSlugs: { string } = HttpService:JSONDecode([==[{{ PACKAGE_SLUGS_JSON }}]==])
+
+-- Discover test scripts via tags (set by combine-test-places.lua)
+local scriptSources = {}
+for _, slug in packageSlugs do
+	local tagged = CollectionService:GetTagged(`_BatchTest_{slug}`)
+	local inst = tagged[1]
+	if inst then
+		scriptSources[slug] = inst.Source
+	else
+		warn(`[BatchTest] Failed to find script source for {slug}`)
+	end
+end
+
+-- Snapshot all package roots (keep references even when unparented)
+local allPackages = {}
+for _, child in ServerScriptService:GetChildren() do
+	allPackages[child.Name] = child
+end
+
+-- Tag on a NevermoreTestRunnerUtils results table. A test script returns one;
+-- one written before the convention returns nil, and a probe returns whatever
+-- it likes, so only a tagged table is read as results.
+local RESULTS_FORMAT = "nevermore-test-results@1"
+
+-- Module NevermoreTestRunnerUtils leaves a run's results in, regardless of what
+-- the script did with them. Most test scripts discard the return value
+-- (`if runTestsIfNeededAsync(root) then return end`), and without this their
+-- counts exist only in Jest's printed report — the first thing a long batch's
+-- log window drops. Read as a fallback, never in preference to what the script
+-- actually returned.
+--
+-- Named rather than required by path: this script is raw source with no loader,
+-- so it finds the module under the package it just ran and requires the
+-- instance. Requiring the same instance twice returns the same table, which is
+-- what makes this a mailbox and not a copy.
+local STATE_MODULE_NAME = "NevermoreTestRunnerState.global"
+
+--[[
+	The state module inside one package, or nil for a package that has no test
+	runner in it at all.
+]]
+local function findStateModule(package: Instance): ModuleScript?
+	local found = package:FindFirstChild(STATE_MODULE_NAME, true)
+
+	return if found and found:IsA("ModuleScript") then found else nil
+end
+
+type TestCounts = {
+	passed: number,
+	failed: number,
+	skipped: number,
+	total: number,
+	suitesPassed: number,
+	suitesFailed: number,
+	suitesTotal: number,
+}
+
+type Result = {
+	success: boolean,
+	slug: string,
+	durationMs: number,
+	error: string?,
+	-- Absent when the package's test script returned nothing structured, which
+	-- is the only case where a reader still has to fall back to its logs.
+	counts: TestCounts?,
+	-- False for a smoke test, whose counts are zero because nothing counted them.
+	-- Carried so a reader never has to guess whether zero means "no tests".
+	ranJest: boolean?,
+}
+
+-- Counts only. This summary is printed as one log line and read back by the
+-- CLI, so it stays short enough to survive whatever the engine does to a long
+-- line; the failure text a run also returns is per-package and already in that
+-- package's own log section.
+local function toCounts(returned: any): TestCounts?
+	if type(returned) ~= "table" or returned.format ~= RESULTS_FORMAT then
+		return nil
+	end
+
+	local counts: { [string]: number } = {}
+	for _, field in { "passed", "failed", "skipped", "total", "suitesPassed", "suitesFailed", "suitesTotal" } do
+		if type(returned[field]) ~= "number" then
+			return nil
+		end
+		counts[field] = returned[field]
+	end
+
+	return counts :: any
+end
+
+local function isStructuredFailure(returned: any): boolean
+	return type(returned) == "table" and returned.format == RESULTS_FORMAT and returned.success ~= true
+end
+
+local results: { Result } = {}
+
+for _, slug in packageSlugs do
+	local scriptSource = scriptSources[slug]
+	if not scriptSource then
+		print("===BATCH_TEST_BEGIN " .. slug .. "===")
+		warn(`[BatchTest] {slug}: No test script found for {slug}. Is it tagged with _BatchTest_{slug}?`)
+		print("===BATCH_TEST_END " .. slug .. " FAIL 0===")
+		table.insert(results, { slug = slug, success = false, durationMs = 0, error = "No test script found" })
+		continue
+	end
+
+	-- ISOLATE: unparent all, show only this package
+	for _, package in allPackages do
+		package.Parent = nil
+	end
+	allPackages[slug].Parent = ServerScriptService
+
+	-- Snapshot services for cleanup
+	local preWorkspace = {}
+	for _, child in workspace:GetChildren() do
+		preWorkspace[child] = true
+	end
+	local preRunService = {}
+	for _, child in ReplicatedStorage:GetChildren() do
+		preRunService[child] = true
+	end
+
+	-- Yield before the BEGIN marker so deferred callbacks triggered by
+	-- reparenting flush before the marker enters the log stream. This
+	-- prevents Open Cloud log reordering from placing leaked output
+	-- inside this package's section.
+	RunService.Heartbeat:Wait()
+
+	print("===BATCH_TEST_BEGIN " .. slug .. "===")
+
+	-- Cleared before the run, so a package that leaves nothing behind cannot be
+	-- credited with the previous package's counts — worse than having none.
+	local stateModule = findStateModule(allPackages[slug])
+	local state: any = if stateModule then (require :: any)(stateModule) else nil
+	if state then
+		state.results = nil
+	end
+
+	-- Measure pcall execution time only. The surrounding cleanup/yield cost
+	-- is amortized across all packages and would otherwise dominate fast tests.
+	local startClock = os.clock()
+
+	local testOk, returned = pcall(function()
+		local fn, compileErr = loadstring(scriptSource, slug)
+		if not fn then
+			error("Compile error: " .. tostring(compileErr))
+		end
+		return fn()
+	end)
+
+	local durationMs = math.floor((os.clock() - startClock) * 1000 + 0.5)
+
+	-- Yield before the END marker for the same reason: leaked deferred
+	-- callbacks may fire during fn() and their output must land before
+	-- the marker in the log stream.
+	RunService.Heartbeat:Wait()
+
+	-- CLEANUP: restore service state
+	allPackages[slug].Parent = nil
+	for _, child in workspace:GetChildren() do
+		if not preWorkspace[child] and not child:IsA("Terrain") and not child:IsA("Camera") then
+			pcall(child.Destroy, child)
+		end
+	end
+	for _, child in ReplicatedStorage:GetChildren() do
+		if not preRunService[child] then
+			pcall(child.Destroy, child)
+		end
+	end
+	-- Destroy injected LoaderLinks (non-archivable, created by LoaderLinkCreator)
+	for _, desc in allPackages[slug]:GetDescendants() do
+		if not desc.Archivable and desc:IsA("ModuleScript") and desc.Name == "loader" then
+			pcall(desc.Destroy, desc)
+		end
+	end
+
+	-- Flush lingering deferred callbacks. Heavy packages (Binder, ServiceBag)
+	-- schedule deferred callbacks via CollectionService that survive cleanup.
+	-- Yielding across multiple heartbeat frames is more reliable than a fixed
+	-- time wait, since each frame drains its deferred queue.
+	for _ = 1, 3 do
+		RunService.Heartbeat:Wait()
+	end
+
+	-- What the script returned, or what the runner left behind when the script
+	-- returned nothing. Both are the same table whenever both exist, so
+	-- preferring the returned one costs nothing and keeps a script that does
+	-- return its results independent of the state module.
+	local published = if state then state.results else nil
+	local reported = if testOk and toCounts(returned) then returned else published
+
+	-- A test script that does not throw has still failed if the results say so,
+	-- which is the whole reason they are read at all: a failing suite used to
+	-- announce itself by erroring, and an error carries no counts.
+	local counts = if testOk then toCounts(reported) else nil
+	local success = testOk and not isStructuredFailure(reported)
+	local failureReason: string? = nil
+	if not testOk then
+		failureReason = tostring(returned)
+	elseif not success then
+		failureReason = tostring((reported :: any).error or "the test runner reported the run as failed")
+	end
+
+	if failureReason then
+		warn("[BatchTest] " .. slug .. ": " .. failureReason)
+	end
+	print("===BATCH_TEST_END " .. slug .. (if success then " PASS " else " FAIL ") .. tostring(durationMs) .. "===")
+	table.insert(results, {
+		slug = slug,
+		success = success,
+		durationMs = durationMs,
+		error = failureReason,
+		counts = counts,
+		ranJest = if counts then (reported :: any).ranJest == true else nil,
+	})
+end
+
+-- Restore all packages
+for _, package in allPackages do
+	package.Parent = ServerScriptService
+end
+
+-- Printed as well as returned. The print is what a person reads in the log and
+-- what an older CLI parses; the return value is the copy that cannot be
+-- truncated, since an oversize return fails the task outright rather than
+-- arriving short. Counts only, so it stays far inside that limit.
+print("===BATCH_TEST_SUMMARY===")
+print(HttpService:JSONEncode(results))
+
+return results