devrock-gametests 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,304 @@
1
+ import { existsSync } from 'node:fs';
2
+ import { readFile, rm, writeFile } from 'node:fs/promises';
3
+ import path from 'node:path';
4
+ import { setTimeout as sleep } from 'node:timers/promises';
5
+ import { buildAddOn, buildRunFolder, ensureTemplateWorld, logFile, runDir, runRoot, testsRegistered, testsRequired, testsRun, testsSkipped, timestamp, } from './deploy.js';
6
+ import { runOutcome } from './outcome.js';
7
+ import { assignBatches, batchesOf, batchSeconds, expectedSeconds, mergeRuns, transcriptLengthUntil, } from './parallel.js';
8
+ import { ensureImage, removeLeftoverContainers } from './podman.js';
9
+ import { listGameTestsInWorker } from './registrations.js';
10
+ import { isComplete, NAMED_TAG, parseReport, stoppedCleanly, tieLabel } from './report.js';
11
+ import { BedrockServer } from './server.js';
12
+ import { TEST_TAG } from './shared.js';
13
+ import { prepareWorld, waitForTickingArea } from './world.js';
14
+ const log = (message) => console.log(`[devrock-gametests] ${message}`);
15
+ /** Where `run` lays out test plots. y=-60 is the first air block above a flat world's grass. */
16
+ const RUN_ORIGIN = '8 -60 8';
17
+ /** If the engine hasn't announced the run by now, it isn't going to. */
18
+ const ANNOUNCE_TIMEOUT_MS = 30_000;
19
+ /** Tests may wait out timers in game time, so each container gets 20 minutes for its tests. */
20
+ const RUN_TIMEOUT_MS = 20 * 60_000;
21
+ /** How long a run must stay complete before its server stops, so a test's next attempt can load. One second is 20 ticks. */
22
+ const SETTLE_MS = 1000;
23
+ const relative = (file) => path.relative(process.cwd(), file);
24
+ /** The seconds each batch took in earlier full runs, which `run` deals batches by. */
25
+ const batchSecondsFile = (project) => path.join(project.workDir, 'batch-seconds.json');
26
+ const seconds = (ms) => `${(ms / 1000).toFixed(1)}s`;
27
+ /** Waits for the test area's chunks to load, and returns where in the transcript the probe ran. */
28
+ async function probeTickingArea(server) {
29
+ const start = server.transcript.length;
30
+ await waitForTickingArea(server);
31
+ return { start, end: server.transcript.length };
32
+ }
33
+ /** Runs one container's tests with `runset <runTag>`. Never throws: whatever goes wrong ends up in the result. */
34
+ async function runContainer(project, image, templateLevelDat, index, { share, batches }, runTag, stamp, verbose) {
35
+ const file = logFile(project, 'run', `${stamp}-${index}`);
36
+ const started = Date.now();
37
+ let server;
38
+ let runStart = 0;
39
+ let runEnd = 0;
40
+ let probe = { start: 0, end: 0 };
41
+ let sentAt = 0;
42
+ let error;
43
+ try {
44
+ const serverDir = await buildRunFolder(project, templateLevelDat, runDir(project, index), share);
45
+ server = new BedrockServer({ image, serverDir, logFile: file, echo: verbose, label: `${index} ` });
46
+ await server.waitForReady();
47
+ await prepareWorld(server, templateLevelDat);
48
+ // The first batch must not start before the test area has loaded: with several servers starting at once, its
49
+ // chunks load late, and a mob can't find a path through chunks that aren't there yet.
50
+ probe = await probeTickingArea(server);
51
+ runStart = probe.end;
52
+ server.send(`execute in overworld positioned ${RUN_ORIGIN} run gametest runset ${runTag}`);
53
+ sentAt = Date.now();
54
+ let completeSince;
55
+ while (server.running && Date.now() - sentAt < RUN_TIMEOUT_MS) {
56
+ const report = parseReport(server.transcript.slice(runStart), server.transcript, project.ourContent, server.sent);
57
+ const given = testsRun(server.transcript);
58
+ const done = isComplete(report, given, testsRequired(server.transcript));
59
+ completeSince = done ? (completeSince ?? Date.now()) : undefined;
60
+ if (completeSince !== undefined && Date.now() - completeSince >= SETTLE_MS)
61
+ break;
62
+ if (report.expected === null && Date.now() - sentAt > ANNOUNCE_TIMEOUT_MS)
63
+ break;
64
+ await sleep(250);
65
+ }
66
+ }
67
+ catch (caught) {
68
+ error = caught.message;
69
+ }
70
+ finally {
71
+ // Verdicts count only up to here, and never past the deadline, so a test that ends after it can't pass, even
72
+ // in the moment before the loop noticed.
73
+ runEnd =
74
+ server === undefined
75
+ ? 0
76
+ : Math.min(server.transcript.length, sentAt === 0 ? Number.POSITIVE_INFINITY : transcriptLengthUntil(server.timeline, sentAt + RUN_TIMEOUT_MS));
77
+ await server?.stop();
78
+ }
79
+ const timeline = server?.timeline ?? [];
80
+ const lastVerdict = timeline.findLast(({ line }) => /^onTest(?:Passed|Failed): /.test(line))?.at;
81
+ return {
82
+ index,
83
+ // What the game says it runs here, which the listing outside the game can only estimate.
84
+ tests: server === undefined ? batches.flatMap((batch) => batch.tests) : testsRun(server.transcript),
85
+ skipped: server === undefined ? [] : testsSkipped(server.transcript),
86
+ registered: server === undefined ? [] : testsRegistered(server.transcript),
87
+ required: server === undefined ? new Map() : testsRequired(server.transcript),
88
+ batches,
89
+ error,
90
+ stoppedCleanly: server !== undefined && stoppedCleanly(server.transcript),
91
+ // The content log is read to the end, so a warning printed after the last verdict or while the server stopped
92
+ // fails the run too.
93
+ report: server === undefined
94
+ ? undefined
95
+ : parseReport(server.transcript.slice(runStart, runEnd), server.transcript, project.ourContent, server.sent, probe),
96
+ logFile: file,
97
+ timeline,
98
+ ms: Date.now() - started,
99
+ testMs: lastVerdict === undefined ? 0 : lastVerdict - sentAt,
100
+ };
101
+ }
102
+ async function readBatchSeconds(project) {
103
+ const file = batchSecondsFile(project);
104
+ if (!existsSync(file))
105
+ return new Map();
106
+ try {
107
+ return new Map(Object.entries(JSON.parse(await readFile(file, 'utf8'))));
108
+ }
109
+ catch {
110
+ return new Map();
111
+ }
112
+ }
113
+ /** Keeps what this run measured, and what earlier runs measured for batches this one didn't run. */
114
+ async function writeBatchSeconds(project, earlier, results) {
115
+ const measured = new Map(earlier);
116
+ for (const { timeline } of results) {
117
+ for (const [name, batchTime] of batchSeconds(timeline))
118
+ measured.set(name, Math.round(batchTime * 10) / 10);
119
+ }
120
+ await writeFile(batchSecondsFile(project), `${JSON.stringify(Object.fromEntries(measured), null, 2)}\n`);
121
+ }
122
+ /**
123
+ * Runs every test tagged TEST_TAG, split by whole batch across up to `containers` containers that run at the same
124
+ * time. Returns 0 if everything passed, 1 if a test, the content log or a container failed, and 2 if a container
125
+ * gave no result.
126
+ */
127
+ export async function runAll(project, verbose, containers) {
128
+ // After the build, so the listing sees any file the build makes that the entry imports.
129
+ return run(project, verbose, async () => {
130
+ const earlier = await readBatchSeconds(project);
131
+ let registrations = [];
132
+ let listingFailed = false;
133
+ try {
134
+ registrations = await listGameTestsInWorker([project.entry], {
135
+ define: { ...project.define },
136
+ root: project.root,
137
+ });
138
+ }
139
+ catch (error) {
140
+ listingFailed = true;
141
+ // Only the plan needs the listing: the test packs run what the game registers either way.
142
+ log('Listing the tests outside the game failed, so one server runs them all. Each test still runs and ' +
143
+ `reports as usual. The listing's error: ${error.message}`);
144
+ }
145
+ const batches = batchesOf(registrations, earlier);
146
+ // With no tests, one container still runs, so the content log is checked.
147
+ const groups = batches.length > 0 ? assignBatches(batches, containers) : [[]];
148
+ const owners = Object.fromEntries(groups.flatMap((group, i) => group.map((batch) => [batch.name, i + 1])));
149
+ const listed = batches.flatMap((batch) => batch.tests);
150
+ return {
151
+ kind: 'full',
152
+ containers: groups.map((group, i) => ({ share: { kind: 'split', container: i + 1, owners }, batches: group })),
153
+ description: `${listed.length} GameTests tagged '${TEST_TAG}' in ${batches.length} batches`,
154
+ listed,
155
+ listingFailed,
156
+ registered: registrations.length,
157
+ earlier,
158
+ };
159
+ });
160
+ }
161
+ /**
162
+ * Runs the named tests, in any letter case, on one server with the same `runset` a full run uses, so every attempt,
163
+ * rotation and batch runs as it would there. Returns like `runAll`, and fails a name the game has no test for.
164
+ */
165
+ export async function runNamed(project, ids) {
166
+ return run(project, false, async () => ({
167
+ kind: 'named',
168
+ containers: [{ share: { kind: 'named', ids }, batches: [] }],
169
+ description: `${ids.length} named GameTests`,
170
+ ids,
171
+ }));
172
+ }
173
+ /** Readies the image, the template world and the build, then plans the run with `planRun` and runs it. */
174
+ async function run(project, verbose, planRun) {
175
+ const started = Date.now();
176
+ removeLeftoverContainers();
177
+ const image = await ensureImage(project, log);
178
+ const templateLevelDat = await ensureTemplateWorld(project, image, log);
179
+ buildAddOn(project);
180
+ await rm(runRoot(project), { recursive: true, force: true });
181
+ const plan = await planRun();
182
+ const count = plan.containers.length;
183
+ log(`Running ${plan.description} on ${count} Bedrock Dedicated Server ${project.serverVersion} ` +
184
+ `${count === 1 ? 'container' : 'containers'}`);
185
+ const runTag = plan.kind === 'full' ? TEST_TAG : NAMED_TAG;
186
+ const stamp = timestamp();
187
+ const results = await Promise.all(plan.containers.map((container, i) => runContainer(project, image, templateLevelDat, i + 1, container, runTag, stamp, verbose)));
188
+ if (plan.kind === 'full')
189
+ await writeBatchSeconds(project, plan.earlier, results);
190
+ const merged = mergeRuns(results);
191
+ const ran = results.flatMap((result) => result.tests);
192
+ const registered = [...new Set(results.flatMap((result) => result.registered ?? []))];
193
+ const allReported = results.every((result) => result.report !== undefined);
194
+ const { exitCode, missing, noneRan } = runOutcome(plan, ran, registered, allReported, merged.exitCode);
195
+ for (const id of missing) {
196
+ merged.verdicts.push({ id, passed: false, error: 'No test is registered with this id.', container: 1 });
197
+ }
198
+ const name = (container) => (count > 1 ? ` in container ${container}` : '');
199
+ console.log(`\n${merged.contentErrors.length > 0 ? 'FAIL' : 'PASS'} content log: ${merged.contentErrors.length} errors and warnings`);
200
+ for (const { entry, tie, container } of merged.contentErrors) {
201
+ console.log(` ${entry} (${tieLabel(tie)})${name(container)}`);
202
+ }
203
+ if (merged.verdicts.length === 0 && noneRan === undefined) {
204
+ log(`No tests are registered with the tag '${TEST_TAG}', so only the content log was checked.`);
205
+ }
206
+ else if (merged.verdicts.length > 0) {
207
+ console.log();
208
+ for (const { id, passed, error, container } of merged.verdicts) {
209
+ console.log(`${passed ? 'PASS' : 'FAIL'} ${id}${passed ? '' : name(container)}`);
210
+ if (error)
211
+ console.log(` ${error}`);
212
+ }
213
+ }
214
+ for (const { container, message } of merged.problems)
215
+ console.log(`\nFAIL container ${container}: ${message}`);
216
+ if (plan.kind === 'full') {
217
+ const unlisted = ran.filter((id) => !plan.listed.includes(id));
218
+ if (unlisted.length > 0) {
219
+ console.log(`\nThe listing outside the game missed ${unlisted.length} tests the game registered: ${unlisted.join(', ')}. ` +
220
+ 'Each ran with its batch, or in container 1 if the listing found no test in that batch. Register tests ' +
221
+ 'at the top level of a module, so a run can time their batches.');
222
+ }
223
+ const phantom = [...new Set(plan.listed)].filter((id) => !ran.includes(id));
224
+ if (phantom.length > 0 && ran.length > 0) {
225
+ console.log(`\nThe listing found ${phantom.length} tests the game never registered: ${phantom.join(', ')}.`);
226
+ }
227
+ if (noneRan === 'listed') {
228
+ console.log(`\nFAIL The listing found ${plan.listed.length} tests, but the game registered none of them with the tag ` +
229
+ `'${TEST_TAG}'. Check that the test pack's script loaded.`);
230
+ }
231
+ if (noneRan === 'untagged') {
232
+ console.log(`\nFAIL ${Math.max(plan.registered, registered.length)} tests are registered, but none has the tag ` +
233
+ `'${TEST_TAG}', so nothing ran. ` +
234
+ 'Give each test .tag(TEST_TAG) from devrock-gametests/game.');
235
+ }
236
+ if (noneRan === 'listing-failed') {
237
+ console.log(`\nFAIL The listing failed, and the game registered no test with the tag '${TEST_TAG}', so nothing ran. ` +
238
+ 'Check that the entry registers its tests while the scripts load.');
239
+ }
240
+ }
241
+ console.log('\nContainers:');
242
+ for (const result of results) {
243
+ const parts = [
244
+ ...(plan.kind === 'full'
245
+ ? [
246
+ `${result.tests.length} tests in ${result.batches.length} batches`,
247
+ `expected ${seconds(expectedSeconds(result.batches) * 1000)}`,
248
+ ]
249
+ : [`${result.tests.length} tests`]),
250
+ `tests took ${seconds(result.testMs)}`,
251
+ `container ${seconds(result.ms)}`,
252
+ ];
253
+ console.log(` ${result.index}: ${parts.join(', ')}. Log: ${relative(result.logFile)}`);
254
+ }
255
+ const failed = merged.verdicts.filter((verdict) => !verdict.passed).length;
256
+ console.log(`\n${merged.verdicts.length - failed} passed, ${failed} failed, content log ` +
257
+ `${merged.contentErrors.length > 0 ? 'failed' : 'clean'} (BDS ${project.serverVersion}, ` +
258
+ `${count} ${count === 1 ? 'container' : 'containers'}, ${seconds(Date.now() - started)})`);
259
+ return exitCode;
260
+ }
261
+ const WAIT_RE = /^wait (\d+)$/;
262
+ /**
263
+ * Sends console commands to a fresh server and prints what it answered. A `wait <ms>` command pauses here instead
264
+ * of being sent. Waits `settleMs` after the last one before it stops the server.
265
+ */
266
+ export async function runConsole(project, commands, settleMs) {
267
+ removeLeftoverContainers();
268
+ const image = await ensureImage(project, log);
269
+ const templateLevelDat = await ensureTemplateWorld(project, image, log);
270
+ buildAddOn(project);
271
+ await rm(runRoot(project), { recursive: true, force: true });
272
+ const serverDir = await buildRunFolder(project, templateLevelDat, runDir(project, 1));
273
+ const file = logFile(project, 'console');
274
+ log(`Starting Bedrock Dedicated Server ${project.serverVersion}`);
275
+ const server = new BedrockServer({ image, serverDir, logFile: file });
276
+ let start = 0;
277
+ let end = 0;
278
+ try {
279
+ await server.waitForReady();
280
+ await prepareWorld(server, templateLevelDat);
281
+ await waitForTickingArea(server);
282
+ start = server.transcript.length;
283
+ for (const command of commands) {
284
+ const wait = WAIT_RE.exec(command);
285
+ if (wait)
286
+ await sleep(Number(wait[1]));
287
+ else
288
+ server.send(command);
289
+ }
290
+ await sleep(settleMs);
291
+ end = server.transcript.length;
292
+ }
293
+ finally {
294
+ await server.stop();
295
+ }
296
+ process.stdout.write(server.transcript.slice(start, end));
297
+ log(`Full log: ${relative(file)}`);
298
+ // A server that didn't stop cleanly may have been lost before it answered, so the answers above may be cut short.
299
+ if (!stoppedCleanly(server.transcript)) {
300
+ log("The server didn't stop cleanly, so its answers may be incomplete.");
301
+ return 2;
302
+ }
303
+ return 0;
304
+ }
@@ -0,0 +1,53 @@
1
+ import type { Log } from './podman.js';
2
+ import { type Project } from './project.js';
3
+ export declare const LEVEL_NAME = "gametest";
4
+ export declare function serverProperties(): string;
5
+ /** Where the template world's level.dat lives. It exists only once it is complete. */
6
+ export declare const templateLevelDat: (project: Project) => string;
7
+ /** Holds one run folder per container. */
8
+ export declare const runRoot: (project: Project) => string;
9
+ /** The run folder of container `index`, which counts from 1. */
10
+ export declare const runDir: (project: Project, index: number) => string;
11
+ export declare const timestamp: () => string;
12
+ export declare const logFile: (project: Project, kind: string, stamp?: string) => string;
13
+ /**
14
+ * Makes the version's template world unless it exists, and returns its level.dat. Experiments only switch on in a
15
+ * world that already exists, so a first boot generates the world, and level.dat gets Beta APIs after it stops.
16
+ */
17
+ export declare function ensureTemplateWorld(project: Project, image: string, log: Log): Promise<string>;
18
+ /** Runs the project's build command, which builds the add-on into its packs. */
19
+ export declare function buildAddOn(project: Project): void;
20
+ /**
21
+ * Which tests a container's `runset` runs, which its test pack decides as each test registers.
22
+ *
23
+ * - `split`: one container of a full run. A test that tags itself TEST_TAG keeps it if this container runs its batch.
24
+ * - `named`: a named run. The named tests, in any letter case, also get NAMED_TAG, whatever tags they have.
25
+ */
26
+ export type Share = {
27
+ readonly kind: 'split';
28
+ /** This container, counting from 1. */
29
+ readonly container: number;
30
+ /** The container that runs each batch the listing found, by batch name. Any other batch runs in container 1. */
31
+ readonly owners: Readonly<Record<string, number>>;
32
+ } | {
33
+ readonly kind: 'named';
34
+ readonly ids: readonly string[];
35
+ };
36
+ /**
37
+ * Rebuilds the run folder `dir` from nothing: server.properties, config, and a world with the template's level.dat,
38
+ * the built add-on packs and the test pack, whose test runtime follows `share`. Returns `dir`.
39
+ */
40
+ export declare function buildRunFolder(project: Project, levelDat: string, dir: string, share?: Share): Promise<string>;
41
+ /** The tag a test outside a container's share runs under instead of TEST_TAG, so that container's runset skips it. */
42
+ export declare const ELSEWHERE_TAG = "devrock_gametests_elsewhere";
43
+ /** The tests a container's test pack kept the run tag on, so its `runset` runs them, from its transcript. */
44
+ export declare const testsRun: (transcript: string) => string[];
45
+ /** The tests a full run's test pack registered but left to another container, from its transcript. */
46
+ export declare const testsSkipped: (transcript: string) => string[];
47
+ /** Every test a container's test pack registered, tagged or not, from its transcript. */
48
+ export declare const testsRegistered: (transcript: string) => string[];
49
+ /**
50
+ * How many passes each test that called `requiredSuccessfulAttempts` or `rotateTest` needs, from a container's
51
+ * transcript. A rotated test needs its passes in each of its 4 rotations.
52
+ */
53
+ export declare function testsRequired(transcript: string): Map<string, number>;