@quenty/nevermore-cli 4.47.0 → 4.47.2

package/templates/batch-test-runner.lua

@@ -1,6 +1,10 @@
--- batch-test-runner.lua
--- Runs all test scripts sequentially in a single execution task.
--- PACKAGE_SLUGS_JSON placeholder is replaced with a JSON array of slug strings at runtime.
+--[[
+	Runs every package's test script sequentially in one execution task, isolating
+	each package under ServerScriptService and cleaning up after it.
+
+	PACKAGE_SLUGS_JSON is replaced with a JSON array of slugs at runtime. Test
+	scripts are found by the `_BatchTest_<slug>` tag set by combine-test-places.lua.
+]]
 
 local CollectionService = game:GetService("CollectionService")
 local HttpService = game:GetService("HttpService")
@@ -8,56 +12,28 @@ local ReplicatedStorage = game:GetService("ReplicatedStorage")
 local RunService = game:GetService("RunService")
 local ServerScriptService = game:GetService("ServerScriptService")
 
--- stylua: ignore
-local packageSlugs: { string } = HttpService:JSONDecode([==[{{ PACKAGE_SLUGS_JSON }}]==])
-
--- Discover test scripts via tags (set by combine-test-places.lua)
-local scriptSources = {}
-for _, slug in packageSlugs do
-	local tagged = CollectionService:GetTagged(`_BatchTest_{slug}`)
-	local inst = tagged[1]
-	if inst then
-		scriptSources[slug] = inst.Source
-	else
-		warn(`[BatchTest] Failed to find script source for {slug}`)
-	end
-end
-
--- Snapshot all package roots (keep references even when unparented)
-local allPackages = {}
-for _, child in ServerScriptService:GetChildren() do
-	allPackages[child.Name] = child
-end
+local BatchTestRunnerUtils = {}
 
--- Tag on a NevermoreTestRunnerUtils results table. A test script returns one;
--- one written before the convention returns nil, and a probe returns whatever
--- it likes, so only a tagged table is read as results.
+-- stylua: ignore
+local PACKAGE_SLUGS: { PackageSlug } = HttpService:JSONDecode([==[{{ PACKAGE_SLUGS_JSON }}]==])
 local RESULTS_FORMAT = "nevermore-test-results@1"
-
--- Module NevermoreTestRunnerUtils leaves a run's results in, regardless of what
--- the script did with them. Most test scripts discard the return value
--- (`if runTestsIfNeededAsync(root) then return end`), and without this their
--- counts exist only in Jest's printed report — the first thing a long batch's
--- log window drops. Read as a fallback, never in preference to what the script
--- actually returned.
---
--- Named rather than required by path: this script is raw source with no loader,
--- so it finds the module under the package it just ran and requires the
--- instance. Requiring the same instance twice returns the same table, which is
--- what makes this a mailbox and not a copy.
 local STATE_MODULE_NAME = "NevermoreTestRunnerState.global"
+local COUNT_FIELDS = { "passed", "failed", "skipped", "total", "suitesPassed", "suitesFailed", "suitesTotal" }
 
---[[
-	The state module inside one package, or nil for a package that has no test
-	runner in it at all.
-]]
-local function findStateModule(package: Instance): ModuleScript?
-	local found = package:FindFirstChild(STATE_MODULE_NAME, true)
+type PackageSlug = string
+type PackageLookupMap = { [PackageSlug]: Instance }
+type ScriptSourceLookupMap = { [PackageSlug]: string }
 
-	return if found and found:IsA("ModuleScript") then found else nil
-end
+type TestResult = {
+	success: boolean,
+	slug: PackageSlug,
+	durationMs: number,
+	error: string?,
+	counts: TestCountsResult?,
+	ranJest: boolean?,
+}
 
-type TestCounts = {
+type TestCountsResult = {
 	passed: number,
 	failed: number,
 	skipped: number,
@@ -67,92 +43,142 @@ type TestCounts = {
 	suitesTotal: number,
 }
 
-type Result = {
-	success: boolean,
-	slug: string,
-	durationMs: number,
-	error: string?,
-	-- Absent when the package's test script returned nothing structured, which
-	-- is the only case where a reader still has to fall back to its logs.
-	counts: TestCounts?,
-	-- False for a smoke test, whose counts are zero because nothing counted them.
-	-- Carried so a reader never has to guess whether zero means "no tests".
-	ranJest: boolean?,
-}
+function BatchTestRunnerUtils.getScriptSourceLookupMap(slugs: { PackageSlug }): ScriptSourceLookupMap
+	local scriptSourceLookupMap: ScriptSourceLookupMap = {}
+	for _, slug in slugs do
+		local inst = CollectionService:GetTagged(`_BatchTest_{slug}`)[1]
+		if inst then
+			scriptSourceLookupMap[slug] = (inst :: Script).Source
+		else
+			warn(`[BatchTest] Failed to find script source for {slug}`)
+		end
+	end
+	return table.freeze(scriptSourceLookupMap)
+end
 
--- Counts only. This summary is printed as one log line and read back by the
--- CLI, so it stays short enough to survive whatever the engine does to a long
--- line; the failure text a run also returns is per-package and already in that
--- package's own log section.
-local function toCounts(returned: any): TestCounts?
-	if type(returned) ~= "table" or returned.format ~= RESULTS_FORMAT then
-		return nil
+function BatchTestRunnerUtils.detachPackages(): (PackageLookupMap, () -> ())
+	local packageLookupMap: PackageLookupMap = {}
+	for _, child in ServerScriptService:GetChildren() do
+		packageLookupMap[child.Name] = child
+		child.Parent = nil
 	end
+	table.freeze(packageLookupMap)
 
-	local counts: { [string]: number } = {}
-	for _, field in { "passed", "failed", "skipped", "total", "suitesPassed", "suitesFailed", "suitesTotal" } do
-		if type(returned[field]) ~= "number" then
-			return nil
+	local function restore()
+		for _, package in packageLookupMap do
+			package.Parent = ServerScriptService
 		end
-		counts[field] = returned[field]
 	end
 
-	return counts :: any
+	return packageLookupMap, restore
 end
 
-local function isStructuredFailure(returned: any): boolean
-	return type(returned) == "table" and returned.format == RESULTS_FORMAT and returned.success ~= true
+function BatchTestRunnerUtils.toSet<T>(list: { T }): { [T]: true }
+	local set: { [T]: true } = {}
+	for _, item in list do
+		set[item] = true
+	end
+	return table.freeze(set)
 end
 
-local results: { Result } = {}
+function BatchTestRunnerUtils.isolatePackage(package: Instance): () -> ()
+	package.Parent = ServerScriptService
 
-for _, slug in packageSlugs do
-	local scriptSource = scriptSources[slug]
-	if not scriptSource then
-		print("===BATCH_TEST_BEGIN " .. slug .. "===")
-		warn(`[BatchTest] {slug}: No test script found for {slug}. Is it tagged with _BatchTest_{slug}?`)
-		print("===BATCH_TEST_END " .. slug .. " FAIL 0===")
-		table.insert(results, { slug = slug, success = false, durationMs = 0, error = "No test script found" })
-		continue
-	end
+	local preWorkspace = BatchTestRunnerUtils.toSet(workspace:GetChildren())
+	local preReplicatedStorage = BatchTestRunnerUtils.toSet(ReplicatedStorage:GetChildren())
 
-	-- ISOLATE: unparent all, show only this package
-	for _, package in allPackages do
+	return function()
 		package.Parent = nil
+
+		for _, child in workspace:GetChildren() do
+			if not preWorkspace[child] and not child:IsA("Terrain") and not child:IsA("Camera") then
+				pcall(child.Destroy, child)
+			end
+		end
+		for _, child in ReplicatedStorage:GetChildren() do
+			if not preReplicatedStorage[child] then
+				pcall(child.Destroy, child)
+			end
+		end
+		for _, desc in package:GetDescendants() do
+			if not desc.Archivable and desc:IsA("ModuleScript") and desc.Name == "loader" then
+				pcall(desc.Destroy, desc)
+			end
+		end
 	end
-	allPackages[slug].Parent = ServerScriptService
+end
 
-	-- Snapshot services for cleanup
-	local preWorkspace = {}
-	for _, child in workspace:GetChildren() do
-		preWorkspace[child] = true
+function BatchTestRunnerUtils.resetTestResultsMailbox(package: Instance): any
+	local found = package:FindFirstChild(STATE_MODULE_NAME, true)
+	if not (found and found:IsA("ModuleScript")) then
+		return nil
 	end
-	local preRunService = {}
-	for _, child in ReplicatedStorage:GetChildren() do
-		preRunService[child] = true
+
+	local state = (require :: any)(found)
+	state.results = nil
+	return state
+end
+
+function BatchTestRunnerUtils.isFailedTestResult(returned: any): boolean
+	return type(returned) == "table" and returned.format == RESULTS_FORMAT and returned.success ~= true
+end
+
+function BatchTestRunnerUtils.readTestResultCounts(returned: any): TestCountsResult?
+	if type(returned) ~= "table" or returned.format ~= RESULTS_FORMAT then
+		return nil
 	end
 
-	-- Yield before the BEGIN marker so deferred callbacks triggered by
-	-- reparenting flush before the marker enters the log stream. This
-	-- prevents Open Cloud log reordering from placing leaked output
-	-- inside this package's section.
-	RunService.Heartbeat:Wait()
+	local counts: { [string]: number } = {}
+	for _, field in COUNT_FIELDS do
+		if type(returned[field]) ~= "number" then
+			return nil
+		end
+		counts[field] = returned[field]
+	end
 
-	print("===BATCH_TEST_BEGIN " .. slug .. "===")
+	return table.freeze(counts) :: any
+end
 
-	-- Cleared before the run, so a package that leaves nothing behind cannot be
-	-- credited with the previous package's counts — worse than having none.
-	local stateModule = findStateModule(allPackages[slug])
-	local state: any = if stateModule then (require :: any)(stateModule) else nil
-	if state then
-		state.results = nil
+function BatchTestRunnerUtils.createTestResult(
+	slug: PackageSlug,
+	testOk: boolean,
+	returned: any,
+	state: any,
+	durationMs: number
+): TestResult
+	local mailboxResults = if state then state.results else nil
+	local reported = if testOk and BatchTestRunnerUtils.readTestResultCounts(returned) then returned else mailboxResults
+
+	local counts = if testOk then BatchTestRunnerUtils.readTestResultCounts(reported) else nil
+	local success = testOk and not BatchTestRunnerUtils.isFailedTestResult(reported)
+	local failureReason: string? = nil
+	if not testOk then
+		failureReason = tostring(returned)
+	elseif not success then
+		failureReason = tostring((reported :: any).error or "the test runner reported the run as failed")
 	end
 
-	-- Measure pcall execution time only. The surrounding cleanup/yield cost
-	-- is amortized across all packages and would otherwise dominate fast tests.
+	return table.freeze({
+		slug = slug,
+		success = success,
+		durationMs = durationMs,
+		error = failureReason,
+		counts = counts,
+		ranJest = if counts then (reported :: any).ranJest == true else nil,
+	})
+end
+
+function BatchTestRunnerUtils.reportMissingScriptResult(slug: PackageSlug): TestResult
+	print("===BATCH_TEST_BEGIN " .. slug .. "===")
+	warn(`[BatchTest] {slug}: No test script found for {slug}. Is it tagged with _BatchTest_{slug}?`)
+	print("===BATCH_TEST_END " .. slug .. " FAIL 0===")
+	return table.freeze({ slug = slug, success = false, durationMs = 0, error = "No test script found" })
+end
+
+function BatchTestRunnerUtils.runScriptTimed(slug: PackageSlug, scriptSource: string): (boolean, any, number)
 	local startClock = os.clock()
 
-	local testOk, returned = pcall(function()
+	local ok, returned = pcall(function()
 		local fn, compileErr = loadstring(scriptSource, slug)
 		if not fn then
 			error("Compile error: " .. tostring(compileErr))
@@ -161,82 +187,60 @@ for _, slug in packageSlugs do
 	end)
 
 	local durationMs = math.floor((os.clock() - startClock) * 1000 + 0.5)
+	return ok, returned, durationMs
+end
+
+function BatchTestRunnerUtils.runPackage(
+	packageLookupMap: PackageLookupMap,
+	slug: PackageSlug,
+	scriptSource: string?
+): TestResult
+	if not scriptSource then
+		return BatchTestRunnerUtils.reportMissingScriptResult(slug)
+	end
+
+	local package = packageLookupMap[slug]
+	local restore = BatchTestRunnerUtils.isolatePackage(package)
 
-	-- Yield before the END marker for the same reason: leaked deferred
-	-- callbacks may fire during fn() and their output must land before
-	-- the marker in the log stream.
 	RunService.Heartbeat:Wait()
+	print("===BATCH_TEST_BEGIN " .. slug .. "===")
 
-	-- CLEANUP: restore service state
-	allPackages[slug].Parent = nil
-	for _, child in workspace:GetChildren() do
-		if not preWorkspace[child] and not child:IsA("Terrain") and not child:IsA("Camera") then
-			pcall(child.Destroy, child)
-		end
-	end
-	for _, child in ReplicatedStorage:GetChildren() do
-		if not preRunService[child] then
-			pcall(child.Destroy, child)
-		end
-	end
-	-- Destroy injected LoaderLinks (non-archivable, created by LoaderLinkCreator)
-	for _, desc in allPackages[slug]:GetDescendants() do
-		if not desc.Archivable and desc:IsA("ModuleScript") and desc.Name == "loader" then
-			pcall(desc.Destroy, desc)
-		end
-	end
+	local state = BatchTestRunnerUtils.resetTestResultsMailbox(package)
+	local testOk, returned, durationMs = BatchTestRunnerUtils.runScriptTimed(slug, scriptSource)
+
+	RunService.Heartbeat:Wait()
+	restore()
 
-	-- Flush lingering deferred callbacks. Heavy packages (Binder, ServiceBag)
-	-- schedule deferred callbacks via CollectionService that survive cleanup.
-	-- Yielding across multiple heartbeat frames is more reliable than a fixed
-	-- time wait, since each frame drains its deferred queue.
 	for _ = 1, 3 do
 		RunService.Heartbeat:Wait()
 	end
 
-	-- What the script returned, or what the runner left behind when the script
-	-- returned nothing. Both are the same table whenever both exist, so
-	-- preferring the returned one costs nothing and keeps a script that does
-	-- return its results independent of the state module.
-	local published = if state then state.results else nil
-	local reported = if testOk and toCounts(returned) then returned else published
-
-	-- A test script that does not throw has still failed if the results say so,
-	-- which is the whole reason they are read at all: a failing suite used to
-	-- announce itself by erroring, and an error carries no counts.
-	local counts = if testOk then toCounts(reported) else nil
-	local success = testOk and not isStructuredFailure(reported)
-	local failureReason: string? = nil
-	if not testOk then
-		failureReason = tostring(returned)
-	elseif not success then
-		failureReason = tostring((reported :: any).error or "the test runner reported the run as failed")
+	local result = BatchTestRunnerUtils.createTestResult(slug, testOk, returned, state, durationMs)
+	if result.error then
+		warn("[BatchTest] " .. slug .. ": " .. result.error)
 	end
+	print(
+		"===BATCH_TEST_END " .. slug .. (if result.success then " PASS " else " FAIL ") .. tostring(durationMs) .. "==="
+	)
 
-	if failureReason then
-		warn("[BatchTest] " .. slug .. ": " .. failureReason)
-	end
-	print("===BATCH_TEST_END " .. slug .. (if success then " PASS " else " FAIL ") .. tostring(durationMs) .. "===")
-	table.insert(results, {
-		slug = slug,
-		success = success,
-		durationMs = durationMs,
-		error = failureReason,
-		counts = counts,
-		ranJest = if counts then (reported :: any).ranJest == true else nil,
-	})
+	return result
 end
 
--- Restore all packages
-for _, package in allPackages do
-	package.Parent = ServerScriptService
+function BatchTestRunnerUtils.printResultsSummary(results: { TestResult })
+	print("===BATCH_TEST_SUMMARY===")
+	print(HttpService:JSONEncode(results))
+end
+
+local scriptSourceLookupMap = BatchTestRunnerUtils.getScriptSourceLookupMap(PACKAGE_SLUGS)
+local packageLookupMap, restorePackages = BatchTestRunnerUtils.detachPackages()
+
+local results: { TestResult } = {}
+for _, slug in PACKAGE_SLUGS do
+	table.insert(results, BatchTestRunnerUtils.runPackage(packageLookupMap, slug, scriptSourceLookupMap[slug]))
 end
+table.freeze(results)
 
--- Printed as well as returned. The print is what a person reads in the log and
--- what an older CLI parses; the return value is the copy that cannot be
--- truncated, since an oversize return fails the task outright rather than
--- arriving short. Counts only, so it stays far inside that limit.
-print("===BATCH_TEST_SUMMARY===")
-print(HttpService:JSONEncode(results))
+restorePackages()
+BatchTestRunnerUtils.printResultsSummary(results)
 
 return results

package/templates/game-template/src/scripts/Server/ServerMain.server.lua

@@ -10,8 +10,9 @@ local require = require(loader).bootstrapGame(root)
 
 local NevermoreTestRunnerUtils = require("NevermoreTestRunnerUtils")
 
-if NevermoreTestRunnerUtils.runTestsIfNeededAsync(root.game) then
-	return
+local results = NevermoreTestRunnerUtils.runTestsIfNeededAsync(root.game)
+if results then
+	return results
 end
 
 local serviceBag = require("ServiceBag").new()

package/templates/nevermore-library-package-template/test/scripts/Server/ServerMain.server.lua

@@ -8,6 +8,7 @@ local loader = root:FindFirstChild("LoaderUtils", true).Parent
 local require = require(loader).bootstrapGame(root)
 
 local NevermoreTestRunnerUtils = require("NevermoreTestRunnerUtils")
-if NevermoreTestRunnerUtils.runTestsIfNeededAsync(root) then
-	return
+local results = NevermoreTestRunnerUtils.runTestsIfNeededAsync(root)
+if results then
+	return results
 end

package/templates/nevermore-service-package-template/test/scripts/Server/ServerMain.server.lua

@@ -8,8 +8,9 @@ local loader = root:FindFirstChild("LoaderUtils", true).Parent
 local require = require(loader).bootstrapGame(root)
 
 local NevermoreTestRunnerUtils = require("NevermoreTestRunnerUtils")
-if NevermoreTestRunnerUtils.runTestsIfNeededAsync(root) then
-	return
+local results = NevermoreTestRunnerUtils.runTestsIfNeededAsync(root)
+if results then
+	return results
 end
 
 local serviceBag = require("ServiceBag").new()