@quenty/nevermore-cli 4.50.1 → 4.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/commands/batch-command/batch-bundle-command.d.ts +12 -0
- package/dist/commands/batch-command/batch-bundle-command.d.ts.map +1 -0
- package/dist/commands/batch-command/batch-bundle-command.js +96 -0
- package/dist/commands/batch-command/batch-bundle-command.js.map +1 -0
- package/dist/commands/batch-command/batch-test-command.d.ts +1 -0
- package/dist/commands/batch-command/batch-test-command.d.ts.map +1 -1
- package/dist/commands/batch-command/batch-test-command.js +103 -15
- package/dist/commands/batch-command/batch-test-command.js.map +1 -1
- package/dist/commands/batch-command/index.d.ts.map +1 -1
- package/dist/commands/batch-command/index.js +2 -0
- package/dist/commands/batch-command/index.js.map +1 -1
- package/dist/commands/tools-command/ci-post-test-results.d.ts +1 -0
- package/dist/commands/tools-command/ci-post-test-results.d.ts.map +1 -1
- package/dist/commands/tools-command/ci-post-test-results.js +6 -1
- package/dist/commands/tools-command/ci-post-test-results.js.map +1 -1
- package/dist/utils/job-context/batch-script-job-context.d.ts +12 -0
- package/dist/utils/job-context/batch-script-job-context.d.ts.map +1 -1
- package/dist/utils/job-context/batch-script-job-context.js +19 -0
- package/dist/utils/job-context/batch-script-job-context.js.map +1 -1
- package/dist/utils/sourcemap/sourcemap-loader.d.ts +3 -3
- package/dist/utils/sourcemap/sourcemap-loader.d.ts.map +1 -1
- package/dist/utils/sourcemap/sourcemap-loader.js +6 -7
- package/dist/utils/sourcemap/sourcemap-loader.js.map +1 -1
- package/dist/utils/testing/batch-run.e2e.test.js +36 -0
- package/dist/utils/testing/batch-run.e2e.test.js.map +1 -1
- package/dist/utils/testing/runner/prebuilt-batch-bundle.d.ts +26 -0
- package/dist/utils/testing/runner/prebuilt-batch-bundle.d.ts.map +1 -0
- package/dist/utils/testing/runner/prebuilt-batch-bundle.js +200 -0
- package/dist/utils/testing/runner/prebuilt-batch-bundle.js.map +1 -0
- package/dist/utils/testing/runner/prebuilt-batch-bundle.test.d.ts +2 -0
- package/dist/utils/testing/runner/prebuilt-batch-bundle.test.d.ts.map +1 -0
- package/dist/utils/testing/runner/prebuilt-batch-bundle.test.js +181 -0
- package/dist/utils/testing/runner/prebuilt-batch-bundle.test.js.map +1 -0
- package/dist/utils/testing/runner/test-runner.d.ts +8 -1
- package/dist/utils/testing/runner/test-runner.d.ts.map +1 -1
- package/dist/utils/testing/runner/test-runner.js +19 -6
- package/dist/utils/testing/runner/test-runner.js.map +1 -1
- package/package.json +7 -6
- package/src/commands/batch-command/batch-bundle-command.ts +140 -0
- package/src/commands/batch-command/batch-test-command.ts +156 -16
- package/src/commands/batch-command/index.ts +2 -0
- package/src/commands/tools-command/ci-post-test-results.ts +11 -1
- package/src/utils/job-context/batch-script-job-context.ts +34 -3
- package/src/utils/sourcemap/sourcemap-loader.ts +7 -9
- package/src/utils/testing/batch-run.e2e.test.ts +42 -0
- package/src/utils/testing/runner/prebuilt-batch-bundle.test.ts +252 -0
- package/src/utils/testing/runner/prebuilt-batch-bundle.ts +267 -0
- package/src/utils/testing/runner/test-runner.ts +43 -13
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -42,6 +42,8 @@ class BatchDeployment implements Deployment {
|
|
|
42
42
|
) {}
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
+
type BatchBuild = Pick<CombinedProjectResult, 'chunks' | 'primaryTarget'>;
|
|
46
|
+
|
|
45
47
|
/** One combined place's upload and execution, each started at most once. */
|
|
46
48
|
interface ChunkRun {
|
|
47
49
|
chunk: CombinedPlaceChunk;
|
|
@@ -98,7 +100,7 @@ export class BatchScriptJobContext implements JobContext {
|
|
|
98
100
|
private _reporter?: Reporter;
|
|
99
101
|
|
|
100
102
|
// Lazy-promise state
|
|
101
|
-
private _combinedBuildPromise?: Promise<
|
|
103
|
+
private _combinedBuildPromise?: Promise<BatchBuild>;
|
|
102
104
|
private _chunkRuns?: ChunkRun[];
|
|
103
105
|
private _executionPromise?: Promise<BatchExecutionOutcome>;
|
|
104
106
|
|
|
@@ -119,6 +121,12 @@ export class BatchScriptJobContext implements JobContext {
|
|
|
119
121
|
*/
|
|
120
122
|
maxChunkBytes?: number;
|
|
121
123
|
reporter?: Reporter;
|
|
124
|
+
/**
|
|
125
|
+
* Places already built by `batch bundle`, run instead of building. They
|
|
126
|
+
* upload to `batchPlaceId` in `batchUniverseId`, which are then required.
|
|
127
|
+
* The context owns `buildContext` and cleans it up like its own build.
|
|
128
|
+
*/
|
|
129
|
+
prebuilt?: { chunks: CombinedPlaceChunk[]; buildContext: BuildContext };
|
|
122
130
|
}
|
|
123
131
|
) {
|
|
124
132
|
this._inner = inner;
|
|
@@ -130,6 +138,25 @@ export class BatchScriptJobContext implements JobContext {
|
|
|
130
138
|
this._maxChunkBytes = options?.maxChunkBytes;
|
|
131
139
|
this._reporter = options?.reporter;
|
|
132
140
|
|
|
141
|
+
if (options?.prebuilt) {
|
|
142
|
+
if (!this._batchPlaceId || !this._batchUniverseId) {
|
|
143
|
+
throw new Error(
|
|
144
|
+
'Prebuilt places need batchPlaceId and batchUniverseId to upload to'
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
const { chunks, buildContext } = options.prebuilt;
|
|
148
|
+
this._combinedBuildContext = buildContext;
|
|
149
|
+
this._chunkRuns = chunks.map((chunk) => ({ chunk }));
|
|
150
|
+
this._combinedBuildPromise = Promise.resolve({
|
|
151
|
+
chunks,
|
|
152
|
+
primaryTarget: {
|
|
153
|
+
placeId: this._batchPlaceId,
|
|
154
|
+
universeId: this._batchUniverseId,
|
|
155
|
+
project: '',
|
|
156
|
+
},
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
|
|
133
160
|
const withBasePlace = batchTargets.filter((t) => t.target.basePlace);
|
|
134
161
|
if (withBasePlace.length > 0) {
|
|
135
162
|
OutputHelper.warn(
|
|
@@ -203,6 +230,10 @@ export class BatchScriptJobContext implements JobContext {
|
|
|
203
230
|
return this._getBatchExecutionAsync();
|
|
204
231
|
}
|
|
205
232
|
|
|
233
|
+
async getBuiltChunksAsync(): Promise<CombinedPlaceChunk[]> {
|
|
234
|
+
return (await this._getChunkRunsAsync()).map((run) => run.chunk);
|
|
235
|
+
}
|
|
236
|
+
|
|
206
237
|
async getLogsAsync(deployment: Deployment): Promise<JobLogs> {
|
|
207
238
|
const batchDeployment = deployment as BatchDeployment;
|
|
208
239
|
const { results, stats } = await this._getBatchExecutionAsync();
|
|
@@ -249,14 +280,14 @@ export class BatchScriptJobContext implements JobContext {
|
|
|
249
280
|
|
|
250
281
|
// ---- Lazy-promise internals ----
|
|
251
282
|
|
|
252
|
-
private _getCombinedBuildAsync(): Promise<
|
|
283
|
+
private _getCombinedBuildAsync(): Promise<BatchBuild> {
|
|
253
284
|
if (!this._combinedBuildPromise) {
|
|
254
285
|
this._combinedBuildPromise = this._doCombinedBuildAsync();
|
|
255
286
|
}
|
|
256
287
|
return this._combinedBuildPromise;
|
|
257
288
|
}
|
|
258
289
|
|
|
259
|
-
private async _doCombinedBuildAsync(): Promise<
|
|
290
|
+
private async _doCombinedBuildAsync(): Promise<BatchBuild> {
|
|
260
291
|
OutputHelper.verbose('Building combined batch place...');
|
|
261
292
|
|
|
262
293
|
// Set all packages to "waiting" — they're queued for building
|
|
@@ -4,17 +4,16 @@ import type { SourcemapNode } from './sourcemap-types.js';
|
|
|
4
4
|
import { SourcemapResolver } from './sourcemap-resolver.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* Try to load a `SourcemapResolver` from `sourcemap.json`
|
|
8
|
-
*
|
|
7
|
+
* Try to load a `SourcemapResolver` for `repoRoot`, from its `sourcemap.json`
|
|
8
|
+
* unless another file is given.
|
|
9
9
|
*
|
|
10
10
|
* Returns `undefined` if the file doesn't exist or can't be parsed — callers
|
|
11
11
|
* should fall back to heuristic resolution.
|
|
12
12
|
*/
|
|
13
13
|
export function tryLoadSourcemapResolver(
|
|
14
|
-
repoRoot: string
|
|
14
|
+
repoRoot: string,
|
|
15
|
+
sourcemapPath = path.join(repoRoot, 'sourcemap.json')
|
|
15
16
|
): SourcemapResolver | undefined {
|
|
16
|
-
const sourcemapPath = path.join(repoRoot, 'sourcemap.json');
|
|
17
|
-
|
|
18
17
|
let content: string;
|
|
19
18
|
try {
|
|
20
19
|
content = fs.readFileSync(sourcemapPath, 'utf-8');
|
|
@@ -22,12 +21,11 @@ export function tryLoadSourcemapResolver(
|
|
|
22
21
|
return undefined;
|
|
23
22
|
}
|
|
24
23
|
|
|
25
|
-
|
|
24
|
+
// A fork's build supplies this file, so a malformed tree is not fatal.
|
|
26
25
|
try {
|
|
27
|
-
root = JSON.parse(content) as SourcemapNode;
|
|
26
|
+
const root = JSON.parse(content) as SourcemapNode;
|
|
27
|
+
return SourcemapResolver.fromSourcemap(root, repoRoot);
|
|
28
28
|
} catch {
|
|
29
29
|
return undefined;
|
|
30
30
|
}
|
|
31
|
-
|
|
32
|
-
return SourcemapResolver.fromSourcemap(root, repoRoot);
|
|
33
31
|
}
|
|
@@ -142,6 +142,7 @@ class FakeTransport implements JobContext {
|
|
|
142
142
|
public scriptContent?: string;
|
|
143
143
|
public scriptContents: string[] = [];
|
|
144
144
|
public deployedPaths: string[] = [];
|
|
145
|
+
public deployedTargets: unknown[] = [];
|
|
145
146
|
public deployCount = 0;
|
|
146
147
|
public runCount = 0;
|
|
147
148
|
public messagesFor?: (script: string) => TaskLogMessage[];
|
|
@@ -160,6 +161,7 @@ class FakeTransport implements JobContext {
|
|
|
160
161
|
): Promise<Deployment> {
|
|
161
162
|
this.deployCount++;
|
|
162
163
|
this.deployedPaths.push(options.builtPlace.rbxlPath);
|
|
164
|
+
this.deployedTargets.push(options.builtPlace.target);
|
|
163
165
|
return {};
|
|
164
166
|
}
|
|
165
167
|
|
|
@@ -506,4 +508,44 @@ describe('an aggregated batch run, end to end', () => {
|
|
|
506
508
|
])
|
|
507
509
|
);
|
|
508
510
|
});
|
|
511
|
+
|
|
512
|
+
it('runs prebuilt places on the target it was given, without building', async () => {
|
|
513
|
+
const generate = vi.mocked(
|
|
514
|
+
(await import('./runner/combined-project-generator.js'))
|
|
515
|
+
.generateCombinedProjectAsync
|
|
516
|
+
);
|
|
517
|
+
generate.mockClear();
|
|
518
|
+
const { targets } = createBatch();
|
|
519
|
+
const transport = new FakeTransport(TASK_MESSAGES);
|
|
520
|
+
const cleanupAsync = vi.fn(async () => {});
|
|
521
|
+
const batch = new BatchScriptJobContext(transport, targets, {
|
|
522
|
+
repoRoot: '/repo',
|
|
523
|
+
batchPlaceId: 900,
|
|
524
|
+
batchUniverseId: 901,
|
|
525
|
+
prebuilt: {
|
|
526
|
+
chunks: [
|
|
527
|
+
{
|
|
528
|
+
rbxlPath: '/bundle/place-1.rbxl',
|
|
529
|
+
slugMap: SLUG_MAP,
|
|
530
|
+
sizeBytes: 1024,
|
|
531
|
+
},
|
|
532
|
+
],
|
|
533
|
+
buildContext: { cleanupAsync } as any,
|
|
534
|
+
},
|
|
535
|
+
});
|
|
536
|
+
|
|
537
|
+
const alpha = await runPackageAsync(batch, '@quenty/alpha');
|
|
538
|
+
const beta = await runPackageAsync(batch, '@quenty/beta');
|
|
539
|
+
|
|
540
|
+
expect(generate).not.toHaveBeenCalled();
|
|
541
|
+
expect(transport.deployedPaths).toEqual(['/bundle/place-1.rbxl']);
|
|
542
|
+
expect(transport.deployedTargets).toEqual([
|
|
543
|
+
{ placeId: 900, universeId: 901, project: '' },
|
|
544
|
+
]);
|
|
545
|
+
expect(alpha.success).toBe(true);
|
|
546
|
+
expect(beta.success).toBe(false);
|
|
547
|
+
|
|
548
|
+
await batch.disposeAsync();
|
|
549
|
+
expect(cleanupAsync).toHaveBeenCalledOnce();
|
|
550
|
+
});
|
|
509
551
|
});
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
import * as fs from 'fs/promises';
|
|
2
|
+
import * as os from 'os';
|
|
3
|
+
import * as path from 'path';
|
|
4
|
+
import { Header } from 'tar';
|
|
5
|
+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
6
|
+
import {
|
|
7
|
+
type PrebuiltBatchBundle,
|
|
8
|
+
readPrebuiltBatchBundleAsync,
|
|
9
|
+
writePrebuiltBatchBundleAsync,
|
|
10
|
+
} from './prebuilt-batch-bundle.js';
|
|
11
|
+
|
|
12
|
+
interface RawEntry {
|
|
13
|
+
path: string;
|
|
14
|
+
body?: string;
|
|
15
|
+
type?: 'File' | 'SymbolicLink' | 'Directory';
|
|
16
|
+
linkpath?: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function rawTar(entries: RawEntry[]): Buffer {
|
|
20
|
+
const blocks: Buffer[] = [];
|
|
21
|
+
for (const entry of entries) {
|
|
22
|
+
const body = Buffer.from(entry.body ?? '');
|
|
23
|
+
const header = new Header({
|
|
24
|
+
path: entry.path,
|
|
25
|
+
type: entry.type ?? 'File',
|
|
26
|
+
linkpath: entry.linkpath,
|
|
27
|
+
size: entry.type && entry.type !== 'File' ? 0 : body.length,
|
|
28
|
+
mode: 0o644,
|
|
29
|
+
mtime: new Date(0),
|
|
30
|
+
});
|
|
31
|
+
header.encode();
|
|
32
|
+
blocks.push(header.block!);
|
|
33
|
+
if (!entry.type || entry.type === 'File') {
|
|
34
|
+
const padded = Buffer.alloc(Math.ceil(body.length / 512) * 512);
|
|
35
|
+
body.copy(padded);
|
|
36
|
+
blocks.push(padded);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
blocks.push(Buffer.alloc(1024));
|
|
40
|
+
return Buffer.concat(blocks);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function validManifest(): any {
|
|
44
|
+
return {
|
|
45
|
+
formatVersion: 1,
|
|
46
|
+
chunks: [
|
|
47
|
+
{
|
|
48
|
+
file: 'place-1.rbxl',
|
|
49
|
+
packages: [
|
|
50
|
+
{ name: '@quenty/rx', slug: 'rx', hasJestConfig: true },
|
|
51
|
+
{ name: '@quenty/maid', slug: 'maid', hasJestConfig: false },
|
|
52
|
+
],
|
|
53
|
+
},
|
|
54
|
+
],
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
describe('prebuilt batch bundle', () => {
|
|
59
|
+
let workDir: string;
|
|
60
|
+
const opened: PrebuiltBatchBundle[] = [];
|
|
61
|
+
|
|
62
|
+
beforeEach(async () => {
|
|
63
|
+
workDir = await fs.mkdtemp(path.join(os.tmpdir(), 'prebuilt-bundle-'));
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
afterEach(async () => {
|
|
67
|
+
for (const bundle of opened.splice(0)) {
|
|
68
|
+
await bundle.buildContext.cleanupAsync();
|
|
69
|
+
}
|
|
70
|
+
await fs.rm(workDir, { recursive: true, force: true });
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
async function readAsync(file: string) {
|
|
74
|
+
const bundle = await readPrebuiltBatchBundleAsync(file);
|
|
75
|
+
opened.push(bundle);
|
|
76
|
+
return bundle;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async function writeRawAsync(entries: RawEntry[]): Promise<string> {
|
|
80
|
+
const file = path.join(workDir, 'raw.nevermore.tests.tar');
|
|
81
|
+
await fs.writeFile(file, rawTar(entries));
|
|
82
|
+
return file;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function writeManifestAsync(manifest: any): Promise<string> {
|
|
86
|
+
return writeRawAsync([
|
|
87
|
+
{ path: 'manifest.json', body: JSON.stringify(manifest) },
|
|
88
|
+
{ path: 'place-1.rbxl', body: 'place bytes' },
|
|
89
|
+
]);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
it('round-trips places, slugs and jest.config flags through one file', async () => {
|
|
93
|
+
const source = path.join(workDir, 'source.rbxl');
|
|
94
|
+
await fs.writeFile(source, 'place bytes');
|
|
95
|
+
const file = path.join(workDir, 'out', 'pr-1.nevermore.tests.tar');
|
|
96
|
+
await writePrebuiltBatchBundleAsync(
|
|
97
|
+
file,
|
|
98
|
+
[
|
|
99
|
+
{
|
|
100
|
+
rbxlPath: source,
|
|
101
|
+
slugMap: new Map([
|
|
102
|
+
['@quenty/rx', 'rx'],
|
|
103
|
+
['@quenty/maid', 'maid'],
|
|
104
|
+
]),
|
|
105
|
+
sizeBytes: 11,
|
|
106
|
+
},
|
|
107
|
+
],
|
|
108
|
+
new Map([['@quenty/rx', true]])
|
|
109
|
+
);
|
|
110
|
+
|
|
111
|
+
const bundle = await readAsync(file);
|
|
112
|
+
|
|
113
|
+
expect(bundle.packages).toEqual([
|
|
114
|
+
{ name: '@quenty/rx', hasJestConfig: true },
|
|
115
|
+
{ name: '@quenty/maid', hasJestConfig: false },
|
|
116
|
+
]);
|
|
117
|
+
expect(bundle.chunks).toHaveLength(1);
|
|
118
|
+
expect(bundle.chunks[0]!.sizeBytes).toBe(11);
|
|
119
|
+
expect([...bundle.chunks[0]!.slugMap]).toEqual([
|
|
120
|
+
['@quenty/rx', 'rx'],
|
|
121
|
+
['@quenty/maid', 'maid'],
|
|
122
|
+
]);
|
|
123
|
+
expect(await fs.readFile(bundle.chunks[0]!.rbxlPath, 'utf-8')).toBe(
|
|
124
|
+
'place bytes'
|
|
125
|
+
);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it('reads an empty bundle as no packages', async () => {
|
|
129
|
+
const file = path.join(workDir, 'empty.nevermore.tests.tar');
|
|
130
|
+
await writePrebuiltBatchBundleAsync(file, [], new Map());
|
|
131
|
+
|
|
132
|
+
const bundle = await readAsync(file);
|
|
133
|
+
|
|
134
|
+
expect(bundle.packages).toEqual([]);
|
|
135
|
+
expect(bundle.chunks).toEqual([]);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
it('removes the unpacked places on cleanup', async () => {
|
|
139
|
+
const bundle = await readPrebuiltBatchBundleAsync(
|
|
140
|
+
await writeManifestAsync(validManifest())
|
|
141
|
+
);
|
|
142
|
+
const placePath = bundle.chunks[0]!.rbxlPath;
|
|
143
|
+
|
|
144
|
+
await bundle.buildContext.cleanupAsync();
|
|
145
|
+
|
|
146
|
+
await expect(fs.stat(placePath)).rejects.toThrow();
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it('never writes outside its own directory for an entry that escapes', async () => {
|
|
150
|
+
const file = await writeRawAsync([
|
|
151
|
+
{ path: 'manifest.json', body: JSON.stringify(validManifest()) },
|
|
152
|
+
{ path: 'place-1.rbxl', body: 'place bytes' },
|
|
153
|
+
{ path: '../escaped.rbxl', body: 'evil' },
|
|
154
|
+
]);
|
|
155
|
+
|
|
156
|
+
await expect(readAsync(file)).rejects.toThrow('unexpected entry');
|
|
157
|
+
await expect(
|
|
158
|
+
fs.stat(path.join(os.tmpdir(), 'escaped.rbxl'))
|
|
159
|
+
).rejects.toThrow();
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
it('rejects a symlink entry', async () => {
|
|
163
|
+
const file = await writeRawAsync([
|
|
164
|
+
{ path: 'manifest.json', body: JSON.stringify(validManifest()) },
|
|
165
|
+
{ path: 'place-1.rbxl', type: 'SymbolicLink', linkpath: '/etc/passwd' },
|
|
166
|
+
]);
|
|
167
|
+
|
|
168
|
+
await expect(readAsync(file)).rejects.toThrow('not a regular file');
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
it('rejects a place the manifest does not list', async () => {
|
|
172
|
+
const file = await writeRawAsync([
|
|
173
|
+
{ path: 'manifest.json', body: JSON.stringify(validManifest()) },
|
|
174
|
+
{ path: 'place-1.rbxl', body: 'place bytes' },
|
|
175
|
+
{ path: 'place-2.rbxl', body: 'extra' },
|
|
176
|
+
]);
|
|
177
|
+
|
|
178
|
+
await expect(readAsync(file)).rejects.toThrow('every place it holds');
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
it('rejects a manifest that lists a place the bundle does not hold', async () => {
|
|
182
|
+
const manifest = validManifest();
|
|
183
|
+
manifest.chunks[0].file = 'place-2.rbxl';
|
|
184
|
+
|
|
185
|
+
await expect(readAsync(await writeManifestAsync(manifest))).rejects.toThrow(
|
|
186
|
+
'invalid place file'
|
|
187
|
+
);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
it('rejects a file that is not a tar', async () => {
|
|
191
|
+
const file = path.join(workDir, 'junk.nevermore.tests.tar');
|
|
192
|
+
await fs.writeFile(file, 'not a tar at all');
|
|
193
|
+
|
|
194
|
+
await expect(readAsync(file)).rejects.toThrow();
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
it('rejects a slug that could break out of the batch script', async () => {
|
|
198
|
+
const manifest = validManifest();
|
|
199
|
+
manifest.chunks[0].packages[0].slug = 'rx"]] print("hi") --';
|
|
200
|
+
|
|
201
|
+
await expect(readAsync(await writeManifestAsync(manifest))).rejects.toThrow(
|
|
202
|
+
'invalid slug'
|
|
203
|
+
);
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
it('accepts a multi-place target name with a free-form place name', async () => {
|
|
207
|
+
const manifest = validManifest();
|
|
208
|
+
manifest.chunks[0].packages[0].name = '@quenty/rx - Hub (Test) + Lobby é';
|
|
209
|
+
|
|
210
|
+
const bundle = await readAsync(await writeManifestAsync(manifest));
|
|
211
|
+
|
|
212
|
+
expect(bundle.packages[0]!.name).toBe('@quenty/rx - Hub (Test) + Lobby é');
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
it.each([
|
|
216
|
+
'@quenty/rx | injected',
|
|
217
|
+
'@quenty/rx<img>',
|
|
218
|
+
'@quenty/rx\nline',
|
|
219
|
+
'@quenty/rx [Approved](https://evil.example)',
|
|
220
|
+
'@quenty/rx ',
|
|
221
|
+
'@quenty/rx cc @someuser',
|
|
222
|
+
'@quenty/rx https://evil.example',
|
|
223
|
+
])(
|
|
224
|
+
'rejects a package name that could break the PR comment: %j',
|
|
225
|
+
async (name) => {
|
|
226
|
+
const manifest = validManifest();
|
|
227
|
+
manifest.chunks[0].packages[0].name = name;
|
|
228
|
+
|
|
229
|
+
await expect(
|
|
230
|
+
readAsync(await writeManifestAsync(manifest))
|
|
231
|
+
).rejects.toThrow('invalid package name');
|
|
232
|
+
}
|
|
233
|
+
);
|
|
234
|
+
|
|
235
|
+
it('rejects a package listed twice', async () => {
|
|
236
|
+
const manifest = validManifest();
|
|
237
|
+
manifest.chunks[0].packages[1].name = '@quenty/rx';
|
|
238
|
+
|
|
239
|
+
await expect(readAsync(await writeManifestAsync(manifest))).rejects.toThrow(
|
|
240
|
+
'more than once'
|
|
241
|
+
);
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
it('rejects an unknown format version', async () => {
|
|
245
|
+
const manifest = validManifest();
|
|
246
|
+
manifest.formatVersion = 2;
|
|
247
|
+
|
|
248
|
+
await expect(readAsync(await writeManifestAsync(manifest))).rejects.toThrow(
|
|
249
|
+
'Unsupported bundle format'
|
|
250
|
+
);
|
|
251
|
+
});
|
|
252
|
+
});
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import * as fs from 'fs/promises';
|
|
2
|
+
import { createReadStream, createWriteStream } from 'fs';
|
|
3
|
+
import * as path from 'path';
|
|
4
|
+
import * as tar from 'tar';
|
|
5
|
+
import { BuildContext } from '@quenty/nevermore-template-helpers';
|
|
6
|
+
import { type CombinedPlaceChunk } from './combined-project-generator.js';
|
|
7
|
+
import { PLACE_UPLOAD_MAX_BYTES } from '../../open-cloud/open-cloud-client.js';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Places built by `nevermore batch bundle`, unpacked from a
|
|
11
|
+
* `*.nevermore.tests.tar` and ready for `batch test --bundle` to run.
|
|
12
|
+
*/
|
|
13
|
+
export interface PrebuiltBatchBundle {
|
|
14
|
+
chunks: CombinedPlaceChunk[];
|
|
15
|
+
packages: PrebuiltBatchPackage[];
|
|
16
|
+
/** Holds the unpacked places; whoever runs them owns its cleanup. */
|
|
17
|
+
buildContext: BuildContext;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface PrebuiltBatchPackage {
|
|
21
|
+
name: string;
|
|
22
|
+
hasJestConfig: boolean;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
interface ManifestFile {
|
|
26
|
+
formatVersion: number;
|
|
27
|
+
chunks: Array<{
|
|
28
|
+
file: string;
|
|
29
|
+
packages: Array<{ name: string; slug: string; hasJestConfig: boolean }>;
|
|
30
|
+
}>;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Suggested file suffix, following `deploy.nevermore.json`. */
|
|
34
|
+
export const BUNDLE_FILE_SUFFIX = '.nevermore.tests.tar';
|
|
35
|
+
|
|
36
|
+
const MANIFEST_FILE_NAME = 'manifest.json';
|
|
37
|
+
const FORMAT_VERSION = 1;
|
|
38
|
+
const MAX_CHUNKS = 32;
|
|
39
|
+
const MAX_PACKAGES = 1000;
|
|
40
|
+
const MAX_MANIFEST_BYTES = 1024 * 1024;
|
|
41
|
+
const CHUNK_FILE_PATTERN = /^place-\d{1,3}\.rbxl$/;
|
|
42
|
+
// Names reach the bot's PR comment, so a fork's bundle gets no Markdown links,
|
|
43
|
+
// images, URLs, tables or @mentions: only a leading scope `@`, and plain text
|
|
44
|
+
// for a multi-place target's place name ("pkg - Hub (Test)").
|
|
45
|
+
const PACKAGE_NAME_PATTERN = /^@?[\p{L}\p{N} /()+.,'_~-]{1,200}$/u;
|
|
46
|
+
const SLUG_PATTERN = /^[A-Za-z0-9_.-]{1,100}$/;
|
|
47
|
+
|
|
48
|
+
export async function writePrebuiltBatchBundleAsync(
|
|
49
|
+
file: string,
|
|
50
|
+
chunks: CombinedPlaceChunk[],
|
|
51
|
+
hasJestConfig: Map<string, boolean>
|
|
52
|
+
): Promise<void> {
|
|
53
|
+
const stage = await BuildContext.createAsync({ prefix: 'test-bundle-' });
|
|
54
|
+
const stageDir = stage.buildDir;
|
|
55
|
+
try {
|
|
56
|
+
const manifest: ManifestFile = {
|
|
57
|
+
formatVersion: FORMAT_VERSION,
|
|
58
|
+
chunks: [],
|
|
59
|
+
};
|
|
60
|
+
for (const [index, chunk] of chunks.entries()) {
|
|
61
|
+
const placeFile = `place-${index + 1}.rbxl`;
|
|
62
|
+
await fs.copyFile(chunk.rbxlPath, path.join(stageDir, placeFile));
|
|
63
|
+
manifest.chunks.push({
|
|
64
|
+
file: placeFile,
|
|
65
|
+
packages: [...chunk.slugMap].map(([name, slug]) => ({
|
|
66
|
+
name,
|
|
67
|
+
slug,
|
|
68
|
+
hasJestConfig: hasJestConfig.get(name) ?? false,
|
|
69
|
+
})),
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
await fs.writeFile(
|
|
73
|
+
path.join(stageDir, MANIFEST_FILE_NAME),
|
|
74
|
+
JSON.stringify(manifest, null, 2)
|
|
75
|
+
);
|
|
76
|
+
|
|
77
|
+
await fs.mkdir(path.dirname(path.resolve(file)), { recursive: true });
|
|
78
|
+
await tar.c({ file, cwd: stageDir, portable: true }, [
|
|
79
|
+
MANIFEST_FILE_NAME,
|
|
80
|
+
...manifest.chunks.map((c) => c.file),
|
|
81
|
+
]);
|
|
82
|
+
} finally {
|
|
83
|
+
await stage.cleanupAsync();
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Read a bundle file. It is untrusted: entries are read from the archive, never
|
|
89
|
+
* extracted by it, and only `manifest.json` and `place-N.rbxl` regular files
|
|
90
|
+
* are accepted.
|
|
91
|
+
*/
|
|
92
|
+
export async function readPrebuiltBatchBundleAsync(
|
|
93
|
+
file: string
|
|
94
|
+
): Promise<PrebuiltBatchBundle> {
|
|
95
|
+
let stat;
|
|
96
|
+
try {
|
|
97
|
+
stat = await fs.lstat(file);
|
|
98
|
+
} catch {
|
|
99
|
+
throw new Error(`Bundle file not found: ${file}`);
|
|
100
|
+
}
|
|
101
|
+
if (!stat.isFile()) {
|
|
102
|
+
throw new Error(`Bundle ${file} is not a regular file`);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const buildContext = await BuildContext.createAsync({
|
|
106
|
+
prefix: 'test-bundle-',
|
|
107
|
+
});
|
|
108
|
+
const dir = buildContext.buildDir;
|
|
109
|
+
try {
|
|
110
|
+
const { manifestText, placeSizes } = await _unpackAsync(file, dir);
|
|
111
|
+
return { ..._parseManifest(manifestText, dir, placeSizes), buildContext };
|
|
112
|
+
} catch (err) {
|
|
113
|
+
await buildContext.cleanupAsync();
|
|
114
|
+
throw err;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async function _unpackAsync(
|
|
119
|
+
file: string,
|
|
120
|
+
dir: string
|
|
121
|
+
): Promise<{ manifestText: string; placeSizes: Map<string, number> }> {
|
|
122
|
+
let manifestText: string | undefined;
|
|
123
|
+
const placeSizes = new Map<string, number>();
|
|
124
|
+
const writes: Promise<void>[] = [];
|
|
125
|
+
let failure: Error | undefined;
|
|
126
|
+
|
|
127
|
+
const parser = new tar.Parser({
|
|
128
|
+
strict: true,
|
|
129
|
+
onReadEntry: (entry) => {
|
|
130
|
+
const name = entry.path;
|
|
131
|
+
const reject = (message: string) => {
|
|
132
|
+
failure ??= new Error(message);
|
|
133
|
+
entry.resume();
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
if (entry.type !== 'File' && entry.type !== 'OldFile') {
|
|
137
|
+
return reject(`Bundle entry ${name} is not a regular file`);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
if (name === MANIFEST_FILE_NAME) {
|
|
141
|
+
if (manifestText !== undefined || entry.size > MAX_MANIFEST_BYTES) {
|
|
142
|
+
return reject('Bundle has more than one manifest, or one too large');
|
|
143
|
+
}
|
|
144
|
+
const parts: Buffer[] = [];
|
|
145
|
+
entry.on('data', (part: Buffer) => parts.push(part));
|
|
146
|
+
entry.on('end', () => {
|
|
147
|
+
manifestText = Buffer.concat(parts).toString('utf-8');
|
|
148
|
+
});
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
if (!CHUNK_FILE_PATTERN.test(name) || placeSizes.has(name)) {
|
|
153
|
+
return reject(`Bundle has an unexpected entry: ${name}`);
|
|
154
|
+
}
|
|
155
|
+
if (placeSizes.size >= MAX_CHUNKS) {
|
|
156
|
+
return reject(`Bundle must hold at most ${MAX_CHUNKS} places`);
|
|
157
|
+
}
|
|
158
|
+
if (entry.size > PLACE_UPLOAD_MAX_BYTES) {
|
|
159
|
+
return reject(`${name} is over the place upload limit`);
|
|
160
|
+
}
|
|
161
|
+
placeSizes.set(name, entry.size);
|
|
162
|
+
writes.push(
|
|
163
|
+
new Promise<void>((resolve, rejectWrite) => {
|
|
164
|
+
const out = createWriteStream(path.join(dir, name));
|
|
165
|
+
out.on('finish', () => resolve());
|
|
166
|
+
out.on('error', rejectWrite);
|
|
167
|
+
entry.on('error', rejectWrite);
|
|
168
|
+
entry.pipe(out);
|
|
169
|
+
})
|
|
170
|
+
);
|
|
171
|
+
},
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
await new Promise<void>((resolve, rejectRead) => {
|
|
175
|
+
const input = createReadStream(file);
|
|
176
|
+
input.on('error', rejectRead);
|
|
177
|
+
parser.on('error', (err: Error) =>
|
|
178
|
+
rejectRead(new Error(`Bundle is not a valid tar file: ${err.message}`))
|
|
179
|
+
);
|
|
180
|
+
parser.on('end', () => resolve());
|
|
181
|
+
input.pipe(parser);
|
|
182
|
+
});
|
|
183
|
+
await Promise.all(writes);
|
|
184
|
+
|
|
185
|
+
if (failure) {
|
|
186
|
+
throw failure;
|
|
187
|
+
}
|
|
188
|
+
if (manifestText === undefined) {
|
|
189
|
+
throw new Error(`Bundle is missing ${MANIFEST_FILE_NAME}`);
|
|
190
|
+
}
|
|
191
|
+
return { manifestText, placeSizes };
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function _parseManifest(
|
|
195
|
+
manifestText: string,
|
|
196
|
+
dir: string,
|
|
197
|
+
placeSizes: Map<string, number>
|
|
198
|
+
): { chunks: CombinedPlaceChunk[]; packages: PrebuiltBatchPackage[] } {
|
|
199
|
+
let parsed: unknown;
|
|
200
|
+
try {
|
|
201
|
+
parsed = JSON.parse(manifestText);
|
|
202
|
+
} catch {
|
|
203
|
+
throw new Error('Bundle manifest is not valid JSON');
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
const manifest = parsed as Partial<ManifestFile>;
|
|
207
|
+
if (manifest?.formatVersion !== FORMAT_VERSION) {
|
|
208
|
+
throw new Error(
|
|
209
|
+
`Unsupported bundle format ${String(manifest?.formatVersion)} ` +
|
|
210
|
+
`(expected ${FORMAT_VERSION})`
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
if (!Array.isArray(manifest.chunks) || manifest.chunks.length > MAX_CHUNKS) {
|
|
214
|
+
throw new Error(`Bundle must list at most ${MAX_CHUNKS} places`);
|
|
215
|
+
}
|
|
216
|
+
if (manifest.chunks.length !== placeSizes.size) {
|
|
217
|
+
throw new Error('Bundle manifest does not list every place it holds');
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
const seenNames = new Set<string>();
|
|
221
|
+
const seenSlugs = new Set<string>();
|
|
222
|
+
const seenFiles = new Set<string>();
|
|
223
|
+
const chunks: CombinedPlaceChunk[] = [];
|
|
224
|
+
const packages: PrebuiltBatchPackage[] = [];
|
|
225
|
+
|
|
226
|
+
for (const entry of manifest.chunks) {
|
|
227
|
+
const file = entry?.file;
|
|
228
|
+
const sizeBytes =
|
|
229
|
+
typeof file === 'string' ? placeSizes.get(file) : undefined;
|
|
230
|
+
if (sizeBytes === undefined || seenFiles.has(file!)) {
|
|
231
|
+
throw new Error(`Bundle lists an invalid place file: ${String(file)}`);
|
|
232
|
+
}
|
|
233
|
+
seenFiles.add(file!);
|
|
234
|
+
|
|
235
|
+
if (!Array.isArray(entry.packages) || entry.packages.length === 0) {
|
|
236
|
+
throw new Error(`${file} lists no packages`);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const slugMap = new Map<string, string>();
|
|
240
|
+
for (const pkg of entry.packages) {
|
|
241
|
+
const { name, slug, hasJestConfig } = pkg ?? {};
|
|
242
|
+
if (typeof name !== 'string' || !PACKAGE_NAME_PATTERN.test(name)) {
|
|
243
|
+
throw new Error(
|
|
244
|
+
`Bundle lists an invalid package name: ${String(name)}`
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
if (typeof slug !== 'string' || !SLUG_PATTERN.test(slug)) {
|
|
248
|
+
throw new Error(`Bundle lists an invalid slug for ${name}`);
|
|
249
|
+
}
|
|
250
|
+
if (seenNames.has(name) || seenSlugs.has(slug)) {
|
|
251
|
+
throw new Error(`Bundle lists ${name} (${slug}) more than once`);
|
|
252
|
+
}
|
|
253
|
+
seenNames.add(name);
|
|
254
|
+
seenSlugs.add(slug);
|
|
255
|
+
slugMap.set(name, slug);
|
|
256
|
+
packages.push({ name, hasJestConfig: hasJestConfig === true });
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
chunks.push({ rbxlPath: path.join(dir, file!), slugMap, sizeBytes });
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
if (packages.length > MAX_PACKAGES) {
|
|
263
|
+
throw new Error(`Bundle lists more than ${MAX_PACKAGES} packages`);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
return { chunks, packages };
|
|
267
|
+
}
|