@quenty/nevermore-cli 4.48.0 → 4.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/commands/batch-command/batch-test-command.d.ts +1 -0
- package/dist/commands/batch-command/batch-test-command.d.ts.map +1 -1
- package/dist/commands/batch-command/batch-test-command.js +11 -0
- package/dist/commands/batch-command/batch-test-command.js.map +1 -1
- package/dist/utils/build/place-recompression.d.ts +25 -0
- package/dist/utils/build/place-recompression.d.ts.map +1 -0
- package/dist/utils/build/place-recompression.js +166 -0
- package/dist/utils/build/place-recompression.js.map +1 -0
- package/dist/utils/build/place-recompression.test.d.ts +2 -0
- package/dist/utils/build/place-recompression.test.d.ts.map +1 -0
- package/dist/utils/build/place-recompression.test.js +68 -0
- package/dist/utils/build/place-recompression.test.js.map +1 -0
- package/dist/utils/job-context/batch-script-job-context.d.ts +20 -8
- package/dist/utils/job-context/batch-script-job-context.d.ts.map +1 -1
- package/dist/utils/job-context/batch-script-job-context.js +94 -22
- package/dist/utils/job-context/batch-script-job-context.js.map +1 -1
- package/dist/utils/open-cloud/open-cloud-client.d.ts +12 -0
- package/dist/utils/open-cloud/open-cloud-client.d.ts.map +1 -1
- package/dist/utils/open-cloud/open-cloud-client.js +37 -1
- package/dist/utils/open-cloud/open-cloud-client.js.map +1 -1
- package/dist/utils/testing/batch-run.e2e.test.js +78 -13
- package/dist/utils/testing/batch-run.e2e.test.js.map +1 -1
- package/dist/utils/testing/runner/combined-project-generator.d.ts +33 -5
- package/dist/utils/testing/runner/combined-project-generator.d.ts.map +1 -1
- package/dist/utils/testing/runner/combined-project-generator.js +116 -19
- package/dist/utils/testing/runner/combined-project-generator.js.map +1 -1
- package/dist/utils/testing/runner/combined-project-generator.test.d.ts +2 -0
- package/dist/utils/testing/runner/combined-project-generator.test.d.ts.map +1 -0
- package/dist/utils/testing/runner/combined-project-generator.test.js +26 -0
- package/dist/utils/testing/runner/combined-project-generator.test.js.map +1 -0
- package/package.json +2 -2
- package/src/commands/batch-command/batch-test-command.ts +14 -0
- package/src/utils/build/place-recompression.test.ts +104 -0
- package/src/utils/build/place-recompression.ts +236 -0
- package/src/utils/job-context/batch-script-job-context.ts +140 -31
- package/src/utils/open-cloud/open-cloud-client.ts +58 -1
- package/src/utils/testing/batch-run.e2e.test.ts +93 -13
- package/src/utils/testing/runner/combined-project-generator.test.ts +30 -0
- package/src/utils/testing/runner/combined-project-generator.ts +198 -30
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -3,6 +3,12 @@
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
|
5
5
|
|
|
6
|
+
# [4.49.0](https://github.com/Quenty/NevermoreEngine/compare/@quenty/nevermore-cli@4.48.0...@quenty/nevermore-cli@4.49.0) (2026-10-26)
|
|
7
|
+
|
|
8
|
+
### Features
|
|
9
|
+
|
|
10
|
+
- **nevermore-cli:** Recompress place uploads and split oversized batches ([#850](https://github.com/Quenty/NevermoreEngine/issues/850)) ([a516061](https://github.com/Quenty/NevermoreEngine/commit/a516061dab439f4ad553a4707c9a64537271c42e))
|
|
11
|
+
|
|
6
12
|
# [4.48.0](https://github.com/Quenty/NevermoreEngine/compare/@quenty/nevermore-cli@4.47.2...@quenty/nevermore-cli@4.48.0) (2026-10-24)
|
|
7
13
|
|
|
8
14
|
### Bug Fixes
|
|
@@ -12,6 +12,7 @@ interface BatchTestArgs extends NevermoreGlobalArgs {
|
|
|
12
12
|
aggregated?: boolean;
|
|
13
13
|
batchPlaceId?: number;
|
|
14
14
|
batchUniverseId?: number;
|
|
15
|
+
maxPlaceMb?: number;
|
|
15
16
|
timeout?: number;
|
|
16
17
|
}
|
|
17
18
|
export declare const batchTestCommand: CommandModule<NevermoreGlobalArgs, BatchTestArgs>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-test-command.d.ts","sourceRoot":"","sources":["../../../src/commands/batch-command/batch-test-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAQtC,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAoChE,UAAU,aAAc,SAAQ,mBAAmB;IACjD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,eAAO,MAAM,gBAAgB,EAAE,aAAa,CAC1C,mBAAmB,EACnB,aAAa,
|
|
1
|
+
{"version":3,"file":"batch-test-command.d.ts","sourceRoot":"","sources":["../../../src/commands/batch-command/batch-test-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAQtC,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAoChE,UAAU,aAAc,SAAQ,mBAAmB;IACjD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,eAAO,MAAM,gBAAgB,EAAE,aAAa,CAC1C,mBAAmB,EACnB,aAAa,CAiGd,CAAC"}
|
|
@@ -65,6 +65,10 @@ export const batchTestCommand = {
|
|
|
65
65
|
.option('batch-universe-id', {
|
|
66
66
|
describe: 'Override universeId for the aggregated batch upload (--aggregated only)',
|
|
67
67
|
type: 'number',
|
|
68
|
+
})
|
|
69
|
+
.option('max-place-mb', {
|
|
70
|
+
describe: 'Size budget in MB for one combined place (--aggregated only). Larger batches are split into several places that run in parallel. Open Cloud rejects place uploads over 150 MB. (default: 90 with --cloud, unlimited locally)',
|
|
71
|
+
type: 'number',
|
|
68
72
|
})
|
|
69
73
|
.option('timeout', {
|
|
70
74
|
describe: 'Max script execution time in seconds for the whole batch. Sent to the Open Cloud API so Roblox cancels server-side on overrun. Max 300s (API limit). (default: 300)',
|
|
@@ -189,6 +193,13 @@ async function _runAsync(args) {
|
|
|
189
193
|
batchPlaceId: args.batchPlaceId,
|
|
190
194
|
batchUniverseId: args.batchUniverseId,
|
|
191
195
|
batchTimeoutMs: timeoutMs,
|
|
196
|
+
// Only an upload is size-limited, so a local run stays one place
|
|
197
|
+
// unless asked to split.
|
|
198
|
+
maxChunkBytes: args.maxPlaceMb !== undefined
|
|
199
|
+
? args.maxPlaceMb * 1024 * 1024
|
|
200
|
+
: cloud
|
|
201
|
+
? undefined
|
|
202
|
+
: Infinity,
|
|
192
203
|
reporter,
|
|
193
204
|
})
|
|
194
205
|
: undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-test-command.js","sourceRoot":"","sources":["../../../src/commands/batch-command/batch-test-command.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EACL,uBAAuB,GAIxB,MAAM,sCAAsC,CAAC;AAE9C,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,MAAM,mDAAmD,CAAC;AACvF,OAAO,EAAE,cAAc,EAAE,MAAM,iDAAiD,CAAC;AACjF,OAAO,EAAE,uBAAuB,EAAE,MAAM,kDAAkD,CAAC;AAC3F,OAAO,EAEL,qBAAqB,EACrB,eAAe,EACf,eAAe,GAChB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,6CAA6C,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,wCAAwC,CAAC;AACrE,OAAO,EACL,oCAAoC,EACpC,wCAAwC,GAEzC,MAAM,6CAA6C,CAAC;AACrD,OAAO,EAGL,iBAAiB,EACjB,0BAA0B,EAC1B,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,GACxB,MAAM,wCAAwC,CAAC;AAChD,OAAO,EACL,kBAAkB,GAEnB,MAAM,2CAA2C,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,oCAAoC,CAAC;
|
|
1
|
+
{"version":3,"file":"batch-test-command.js","sourceRoot":"","sources":["../../../src/commands/batch-command/batch-test-command.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EACL,uBAAuB,GAIxB,MAAM,sCAAsC,CAAC;AAE9C,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,MAAM,mDAAmD,CAAC;AACvF,OAAO,EAAE,cAAc,EAAE,MAAM,iDAAiD,CAAC;AACjF,OAAO,EAAE,uBAAuB,EAAE,MAAM,kDAAkD,CAAC;AAC3F,OAAO,EAEL,qBAAqB,EACrB,eAAe,EACf,eAAe,GAChB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,6CAA6C,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,wCAAwC,CAAC;AACrE,OAAO,EACL,oCAAoC,EACpC,wCAAwC,GAEzC,MAAM,6CAA6C,CAAC;AACrD,OAAO,EAGL,iBAAiB,EACjB,0BAA0B,EAC1B,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,GACxB,MAAM,wCAAwC,CAAC;AAChD,OAAO,EACL,kBAAkB,GAEnB,MAAM,2CAA2C,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,oCAAoC,CAAC;AAkB1D,MAAM,CAAC,MAAM,gBAAgB,GAGzB;IACF,OAAO,EAAE,MAAM;IACf,QAAQ,EAAE,kDAAkD;IAC5D,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;QACjB,OAAO,CACL,KAAK;aACF,MAAM,CAAC,OAAO,EAAE;YACf,QAAQ,EAAE,6CAA6C;YACvD,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;SACf,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACjB,QAAQ,EAAE,0CAA0C;YACpD,IAAI,EAAE,QAAQ;SACf,CAAC;aACD,MAAM,CAAC,KAAK,EAAE;YACb,QAAQ,EAAE,uDAAuD;YACjE,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;SACf,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACd,QAAQ,EAAE,8CAA8C;YACxD,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,aAAa;SACvB,CAAC;aACD,MAAM,CAAC,aAAa,EAAE;YACrB,QAAQ,EAAE,wDAAwD;YAClE,IAAI,EAAE,QAAQ;SACf,CAAC;aACD,MAAM,CAAC,QAAQ,EAAE;YAChB,QAAQ,EAAE,iCAAiC;YAC3C,IAAI,EAAE,QAAQ;SACf,CAAC;aACD,MAAM,CAAC,OAAO,EAAE;YACf,QAAQ,EAAE,sDAAsD;YAChE,IAAI,EAAE,QAAQ;SACf,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACd,QAAQ,EAAE,0DAA0D;YACpE,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;SACf,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YACpB,QAAQ,EACN,qEAAqE;YACvE,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,IAAI;SACd,CAAC;aACD,MAAM,CAAC,gBAAgB,EAAE;YACxB,QAAQ,EACN,sEAAsE;YACxE,IAAI,EAAE,QAAQ;SACf,CAAC;aACD,MAAM,CAAC,mBAAmB,EAAE;YAC3B,QAAQ,EACN,yEAAyE;YAC3E,IAAI,EAAE,QAAQ;SACf,CAAC;aACD,MAAM,CAAC,cAAc,EAAE;YACtB,QAAQ,EACN,8NAA8N;YAChO,IAAI,EAAE,QAAQ;SACf,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACjB,QAAQ,EACN,qKAAqK;YACvK,IAAI,EAAE,QAAQ;SACf,CAAC;YACF,uEAAuE;YACvE,yEAAyE;YACzE,qEAAqE;YACrE,4DAA4D;aAC3D,MAAM,CAAC,aAAa,EAAE;YACrB,MAAM,EAAE,IAAI;YACZ,IAAI,EAAE,QAAQ;SACf,CAAC;aACD,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE;YACd,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CACb,kGAAkG;oBAChG,4CAA4C;oBAC5C,wEAAwE,CAC3E,CAAC;YACJ,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CACL,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;QACtB,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,YAAY,CAAC,KAAK,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;CACF,CAAC;AAEF,KAAK,UAAU,SAAS,CAAC,IAAmB;IAC1C,IAAI,QAAQ,GAAG,IAAI,CAAC,GAAG;QACrB,CAAC,CAAC,MAAM,oCAAoC,EAAE;QAC9C,CAAC,CAAC,MAAM,wCAAwC,CAAC,IAAI,CAAC,IAAK,CAAC,CAAC;IAE/D,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QACjC,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,YAAY,CAAC,IAAI,CACf,2FAA2F;YACzF,6DAA6D,CAChE,CAAC;QACF,OAAO;IACT,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,YAAY,CAAC,IAAI,CACf,uBAAuB,QAAQ,CAAC,MAAM,cAAc,KAAK,EAAE,CAC5D,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC;IAClC,MAAM,WAAW,GACf,IAAI,CAAC,WAAW,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,KAAK,CAAC;QACtD,CAAC,CAAC,QAAQ;QACV,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC;IACvB,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;IAClE,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC;IAC3C,2EAA2E;IAC3E,0EAA0E;IAC1E,6EAA6E;IAC7E,MAAM,YAAY,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IACzE,IAAI,kBAAuD,CAAC;IAE5D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CACpC,YAAY,EACZ,CAAC,KAAuB,EAAE,EAAE;QAC1B,0EAA0E;QAC1E,uDAAuD;QACvD,IAAI,UAAU,EAAE,CAAC;YACf,kBAAkB,GAAG,IAAI,uBAAuB,CAAC,QAAQ,CAAC,MAAM,EAAE;gBAChE,UAAU,EAAE,SAAS;aACtB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAe;YAC5B,kBAAkB;gBAChB,CAAC,SAAS;oBACR,CAAC,CAAC,IAAI,eAAe,CAAC,KAAK,EAAE;wBACzB,QAAQ,EAAE,IAAI,CAAC,IAAI,IAAI,KAAK;wBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,UAAU,EAAE,SAAS;wBACrB,iBAAiB,EAAE,IAAI;qBACxB,CAAC;oBACJ,CAAC,CAAC,IAAI,eAAe,CAAC,KAAK,EAAE;wBACzB,QAAQ,EAAE,IAAI,CAAC,IAAI,IAAI,KAAK;wBAC5B,UAAU,EAAE,SAAS;qBACtB,CAAC,CAAC;YACT,qEAAqE;YACrE,uEAAuE;YACvE,wCAAwC;YACxC,IAAI,oBAAoB,CAAC,KAAK,EAAE,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC;SAC7D,CAAC;QACF,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,SAAS,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,IAAI,EAAE,EAAE,CAAC;YACX,SAAS,CAAC,IAAI,CACZ,IAAI,0BAA0B,CAC5B,KAAK,EACL,uBAAuB,EAAE,EACzB,WAAW,CACZ,CACF,CAAC;QACJ,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC,CACF,CAAC;IAEF,IAAI,MAAuB,CAAC;IAC5B,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,GAAG,IAAI,eAAe,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,WAAW,EAAE,EAAE,CAAC,CAAC;IAC3E,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,IAAI,eAAe,CAAC;YAC3B,MAAM,EAAE,GAAG,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC;YAClC,WAAW,EAAE,IAAI,WAAW,EAAE;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC;IAE/C,kFAAkF;IAClF,yEAAyE;IACzE,MAAM,aAAa,GAAa,UAAU;QACxC,CAAC,CAAC,wBAAwB,CAAC,QAAQ,EAAE,YAAY,CAAC;QAClD,CAAC,CAAC,QAAQ,CAAC;IAEb,0EAA0E;IAC1E,2EAA2E;IAC3E,+CAA+C;IAC/C,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAEhE,MAAM,YAAY,GAAe,KAAK;QACpC,CAAC,CAAC,IAAI,eAAe,CAAC,aAAa,EAAE,MAAM,EAAE,iBAAiB,CAAC;QAC/D,CAAC,CAAC,IAAI,eAAe,CAAC,aAAa,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAC;IAElE,MAAM,YAAY,GAAG,UAAU;QAC7B,CAAC,CAAC,IAAI,qBAAqB,CAAC,YAAY,EAAE,QAAQ,EAAE;YAChD,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,cAAc,EAAE,SAAS;YACzB,iEAAiE;YACjE,yBAAyB;YACzB,aAAa,EACX,IAAI,CAAC,UAAU,KAAK,SAAS;gBAC3B,CAAC,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI;gBAC/B,CAAC,CAAC,KAAK;oBACP,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,QAAQ;YACd,QAAQ;SACT,CAAC;QACJ,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,OAAO,GAAe,YAAY,IAAI,YAAY,CAAC;IAEzD,MAAM,QAAQ,CAAC,UAAU,EAAE,CAAC;IAE5B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,CAAC;QACH,uEAAuE;QACvE,0EAA0E;QAC1E,uEAAuE;QACvE,2EAA2E;QAC3E,wEAAwE;QACxE,wEAAwE;QACxE,iDAAiD;QACjD,IAAI,YAAY,IAAI,kBAAkB,EAAE,CAAC;YACvC,0EAA0E;YAC1E,yEAAyE;YACzE,0EAA0E;YAC1E,MAAM,OAAO,GAAG,MAAM,sBAAsB,CAAC,iBAAiB,EAAE,GAAG,EAAE,CACnE,YAAY,CAAC,wBAAwB,EAAE,CACxC,CAAC;YAEF,kBAAkB,CAAC,QAAQ,CAAC,kBAAkB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;YAEvE,KAAK,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;gBACrD,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,aAAa,CAA+B;YAChE,KAAK,EAAE,QAAQ;YACf,WAAW;YACX,QAAQ;YACR,YAAY,EAAE,SAAS;YACvB,YAAY,EAAE,QAAQ,CAAC,KAAK;YAC5B,YAAY,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;gBAC1B,MAAM,MAAM,GAAG,MAAM,kBAAkB,CACrC,GAAG,EACH,OAAO,EACP,SAAS,EACT,UAAU,CACX,CAAC;gBAEF,OAAO;oBACL,WAAW,EAAE,GAAG,CAAC,IAAI;oBACrB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,OAAO;oBAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,4DAA4D;oBAC5D,4DAA4D;oBAC5D,oDAAoD;oBACpD,UAAU,EAAE,MAAM,CAAC,UAAU;oBAC7B,eAAe,EAAE,MAAM,CAAC,UAAU;wBAChC,CAAC,CAAC,EAAE,IAAI,EAAE,aAAsB,EAAE,GAAG,MAAM,CAAC,UAAU,EAAE;wBACxD,CAAC,CAAC,SAAS;oBACb,qEAAqE;oBACrE,mEAAmE;oBACnE,oCAAoC;oBACpC,OAAO,EAAE,MAAM,CAAC,QAAQ;wBACtB,CAAC,CAAE,CAAC,WAAW,CAAW,CAAC,KAAK,EAAE;wBAClC,CAAC,CAAC,SAAS;oBACb,KAAK,EAAE,MAAM,CAAC,KAAK;iBACpB,CAAC;YACJ,CAAC;SACF,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,QAAQ,GAAG,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,QAAQ,GAAG,CAAC,CAAC;IACf,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;IAED,MAAM,QAAQ,CAAC,SAAS,EAAE,CAAC;IAE3B,uEAAuE;IACvE,uEAAuE;IACvE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AACzB,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,GAAgB,EAChB,OAAmB,EACnB,SAAiB,EACjB,UAAmB;IAEnB,MAAM,IAAI,GAAG;QACX,WAAW,EAAE,GAAG,CAAC,IAAI;QACrB,WAAW,EAAE,GAAG,CAAC,IAAI;QACrB,SAAS;QACT,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,0EAA0E;QAC1E,wDAAwD;QACxD,wBAAwB,EAAE,UAAU;KACrC,CAAC;IAEF,OAAO,sBAAsB,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,CAC3C,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,CAClC,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,sBAAsB,CACnC,KAAa,EACb,QAA0B;IAE1B,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAEjE,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;YACtE,YAAY,CAAC,IAAI,CAAC,GAAG,KAAK,kCAAkC,CAAC,CAAC;YAC9D,OAAO,MAAM,QAAQ,EAAE,CAAC;QAC1B,CAAC;QAED,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAC/B,MAAgB,EAChB,YAAsB;IAEtB,6EAA6E;IAC7E,4EAA4E;IAC5E,0EAA0E;IAC1E,oEAAoE;IACpE,IAAI,kBAAwC,CAAC;IAC7C,OAAO;QACL,UAAU,EAAE,KAAK,IAAI,EAAE,GAAE,CAAC;QAC1B,SAAS,EAAE,KAAK,IAAI,EAAE,GAAE,CAAC;QACzB,cAAc,EAAE,GAAG,EAAE,GAAE,CAAC;QACxB,eAAe,EAAE,GAAG,EAAE,GAAE,CAAC;QACzB,oBAAoB,EAAE,CAAC,KAAa,EAAE,KAAe,EAAE,EAAE;YACvD,IAAI,kBAAkB,KAAK,KAAK;gBAAE,OAAO;YACzC,kBAAkB,GAAG,KAAK,CAAC;YAC3B,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;gBAChC,MAAM,CAAC,oBAAoB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QACD,uBAAuB,EAAE,CAAC,KAAa,EAAE,QAAyB,EAAE,EAAE;YACpE,uEAAuE;YACvE,wEAAwE;YACxE,mEAAmE;YACnE,IAAI,kBAAkB,KAAK,WAAW;gBAAE,OAAO;YAC/C,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;gBAChC,MAAM,CAAC,uBAAuB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export type PlaceRecompressionResult = {
|
|
2
|
+
kind: 'recompressed';
|
|
3
|
+
data: Buffer;
|
|
4
|
+
originalBytes: number;
|
|
5
|
+
}
|
|
6
|
+
/** Not a binary place, or every compressed chunk is already zstd. */
|
|
7
|
+
| {
|
|
8
|
+
kind: 'unchanged';
|
|
9
|
+
data: Buffer;
|
|
10
|
+
} | {
|
|
11
|
+
kind: 'unsupported';
|
|
12
|
+
data: Buffer;
|
|
13
|
+
reason: string;
|
|
14
|
+
};
|
|
15
|
+
/** Whether this Node can recompress places at all. */
|
|
16
|
+
export declare function getZstdSupport(): {
|
|
17
|
+
supported: true;
|
|
18
|
+
} | {
|
|
19
|
+
supported: false;
|
|
20
|
+
reason: string;
|
|
21
|
+
};
|
|
22
|
+
export declare function recompressPlace(data: Buffer): PlaceRecompressionResult;
|
|
23
|
+
/** Decode one raw LZ4 block, the format's default chunk compression. */
|
|
24
|
+
export declare function decompressLz4Block(src: Buffer, uncompressedLength: number): Buffer;
|
|
25
|
+
//# sourceMappingURL=place-recompression.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"place-recompression.d.ts","sourceRoot":"","sources":["../../../src/utils/build/place-recompression.ts"],"names":[],"mappings":"AA8CA,MAAM,MAAM,wBAAwB,GAChC;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAE;AAC/D,qEAAqE;GACnE;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1D,sDAAsD;AACtD,wBAAgB,cAAc,IAC1B;IAAE,SAAS,EAAE,IAAI,CAAA;CAAE,GACnB;IAAE,SAAS,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAYvC;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,wBAAwB,CA+CtE;AA+DD,wEAAwE;AACxE,wBAAgB,kBAAkB,CAChC,GAAG,EAAE,MAAM,EACX,kBAAkB,EAAE,MAAM,GACzB,MAAM,CAoDR"}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import * as zlib from 'zlib';
|
|
2
|
+
/**
|
|
3
|
+
* Rewrite a binary place file's compressed chunks as long-window zstd.
|
|
4
|
+
*
|
|
5
|
+
* Every script's source lives in one `PROP` chunk per class, and Rojo and Lune
|
|
6
|
+
* compress chunks as LZ4, whose 64 KB window cannot reach a module the place
|
|
7
|
+
* already holds a copy of. A Nevermore place carries its dependencies by copy
|
|
8
|
+
* — a combined batch place a copy per package — so the same modules repeat
|
|
9
|
+
* many times over and LZ4 stores every one. zstd with a long window and
|
|
10
|
+
* long-distance matching finds them: a 40-package batch place measured 38.3 MB
|
|
11
|
+
* as LZ4 and 1.3 MB recompressed, and ran identically.
|
|
12
|
+
*
|
|
13
|
+
* The binary format picks the codec per chunk from its leading bytes, so this
|
|
14
|
+
* changes how the file is stored, not what it holds. Chunk payloads are
|
|
15
|
+
* decompressed and recompressed byte for byte; nothing is parsed.
|
|
16
|
+
*/
|
|
17
|
+
/** zstd frame magic; the format reads any other compressed payload as LZ4. */
|
|
18
|
+
const ZSTD_MAGIC = 0xfd2fb528;
|
|
19
|
+
const PLACE_MAGIC = Buffer.from('<roblox!', 'latin1');
|
|
20
|
+
const HEADER_LENGTH = 32;
|
|
21
|
+
const CHUNK_HEADER_LENGTH = 16;
|
|
22
|
+
/**
|
|
23
|
+
* 2^27, zstd's default decoder window limit. A larger window would need the
|
|
24
|
+
* decoder to opt in, and this is the size Roblox has been seen to decode.
|
|
25
|
+
*/
|
|
26
|
+
const WINDOW_LOG = 27;
|
|
27
|
+
/** Level 9 is within 15% of level 19's size here at a fourteenth of the time. */
|
|
28
|
+
const COMPRESSION_LEVEL = 9;
|
|
29
|
+
/** Whether this Node can recompress places at all. */
|
|
30
|
+
export function getZstdSupport() {
|
|
31
|
+
const z = zlib;
|
|
32
|
+
if (typeof z.zstdCompressSync === 'function' &&
|
|
33
|
+
typeof z.zstdDecompressSync === 'function') {
|
|
34
|
+
return { supported: true };
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
supported: false,
|
|
38
|
+
reason: `Node ${process.version} has no zstd (needs 22.15+ or 23.8+)`,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
export function recompressPlace(data) {
|
|
42
|
+
if (data.length < HEADER_LENGTH ||
|
|
43
|
+
!data.subarray(0, PLACE_MAGIC.length).equals(PLACE_MAGIC)) {
|
|
44
|
+
// An XML place, or not a place: nothing to recompress.
|
|
45
|
+
return { kind: 'unchanged', data };
|
|
46
|
+
}
|
|
47
|
+
const chunks = _readChunks(data);
|
|
48
|
+
if (!chunks.some((c) => c.compressedLength > 0 && !c.isZstd)) {
|
|
49
|
+
return { kind: 'unchanged', data };
|
|
50
|
+
}
|
|
51
|
+
const support = getZstdSupport();
|
|
52
|
+
if (!support.supported) {
|
|
53
|
+
return { kind: 'unsupported', data, reason: support.reason };
|
|
54
|
+
}
|
|
55
|
+
const z = zlib;
|
|
56
|
+
const params = {
|
|
57
|
+
[z.constants.ZSTD_c_compressionLevel]: COMPRESSION_LEVEL,
|
|
58
|
+
[z.constants.ZSTD_c_windowLog]: WINDOW_LOG,
|
|
59
|
+
[z.constants.ZSTD_c_enableLongDistanceMatching]: 1,
|
|
60
|
+
};
|
|
61
|
+
const parts = [data.subarray(0, HEADER_LENGTH)];
|
|
62
|
+
for (const chunk of chunks) {
|
|
63
|
+
if (chunk.compressedLength === 0 || chunk.isZstd) {
|
|
64
|
+
parts.push(data.subarray(chunk.start, chunk.end));
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
const raw = decompressLz4Block(chunk.payload, chunk.uncompressedLength);
|
|
68
|
+
const compressed = z.zstdCompressSync(raw, { params });
|
|
69
|
+
const header = Buffer.from(data.subarray(chunk.start, chunk.start + CHUNK_HEADER_LENGTH));
|
|
70
|
+
header.writeUInt32LE(compressed.length, 4);
|
|
71
|
+
parts.push(header, compressed);
|
|
72
|
+
}
|
|
73
|
+
return {
|
|
74
|
+
kind: 'recompressed',
|
|
75
|
+
data: Buffer.concat(parts),
|
|
76
|
+
originalBytes: data.length,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
function _readChunks(data) {
|
|
80
|
+
const chunks = [];
|
|
81
|
+
let pos = HEADER_LENGTH;
|
|
82
|
+
let sawEnd = false;
|
|
83
|
+
while (!sawEnd) {
|
|
84
|
+
if (pos + CHUNK_HEADER_LENGTH > data.length) {
|
|
85
|
+
throw new Error('Place file is truncated before its END chunk');
|
|
86
|
+
}
|
|
87
|
+
const name = data.toString('latin1', pos, pos + 4);
|
|
88
|
+
const compressedLength = data.readUInt32LE(pos + 4);
|
|
89
|
+
const uncompressedLength = data.readUInt32LE(pos + 8);
|
|
90
|
+
const payloadStart = pos + CHUNK_HEADER_LENGTH;
|
|
91
|
+
const end = payloadStart + (compressedLength || uncompressedLength);
|
|
92
|
+
if (end > data.length) {
|
|
93
|
+
throw new Error(`Place file is truncated inside its ${name.trim()} chunk`);
|
|
94
|
+
}
|
|
95
|
+
const payload = data.subarray(payloadStart, end);
|
|
96
|
+
chunks.push({
|
|
97
|
+
start: pos,
|
|
98
|
+
end,
|
|
99
|
+
compressedLength,
|
|
100
|
+
uncompressedLength,
|
|
101
|
+
payload,
|
|
102
|
+
isZstd: compressedLength >= 4 && payload.readUInt32LE(0) === ZSTD_MAGIC,
|
|
103
|
+
});
|
|
104
|
+
pos = end;
|
|
105
|
+
sawEnd = name === 'END\0';
|
|
106
|
+
}
|
|
107
|
+
// Anything after END is not the format's, but it is kept rather than dropped.
|
|
108
|
+
if (pos < data.length) {
|
|
109
|
+
chunks.push({
|
|
110
|
+
start: pos,
|
|
111
|
+
end: data.length,
|
|
112
|
+
compressedLength: 0,
|
|
113
|
+
uncompressedLength: data.length - pos,
|
|
114
|
+
payload: data.subarray(pos),
|
|
115
|
+
isZstd: false,
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
return chunks;
|
|
119
|
+
}
|
|
120
|
+
/** Decode one raw LZ4 block, the format's default chunk compression. */
|
|
121
|
+
export function decompressLz4Block(src, uncompressedLength) {
|
|
122
|
+
const out = Buffer.alloc(uncompressedLength);
|
|
123
|
+
let s = 0;
|
|
124
|
+
let d = 0;
|
|
125
|
+
while (s < src.length) {
|
|
126
|
+
const token = src[s++];
|
|
127
|
+
let literalLength = token >> 4;
|
|
128
|
+
if (literalLength === 15) {
|
|
129
|
+
let b;
|
|
130
|
+
do {
|
|
131
|
+
b = src[s++];
|
|
132
|
+
literalLength += b;
|
|
133
|
+
} while (b === 255);
|
|
134
|
+
}
|
|
135
|
+
src.copy(out, d, s, s + literalLength);
|
|
136
|
+
s += literalLength;
|
|
137
|
+
d += literalLength;
|
|
138
|
+
// The last sequence is literals only.
|
|
139
|
+
if (s >= src.length)
|
|
140
|
+
break;
|
|
141
|
+
const offset = src[s] | (src[s + 1] << 8);
|
|
142
|
+
s += 2;
|
|
143
|
+
if (offset === 0 || offset > d) {
|
|
144
|
+
throw new Error('Corrupt LZ4 block: match offset out of range');
|
|
145
|
+
}
|
|
146
|
+
let matchLength = token & 15;
|
|
147
|
+
if (matchLength === 15) {
|
|
148
|
+
let b;
|
|
149
|
+
do {
|
|
150
|
+
b = src[s++];
|
|
151
|
+
matchLength += b;
|
|
152
|
+
} while (b === 255);
|
|
153
|
+
}
|
|
154
|
+
matchLength += 4;
|
|
155
|
+
// Byte by byte: a match may overlap the bytes it is producing.
|
|
156
|
+
for (let i = 0; i < matchLength; i++) {
|
|
157
|
+
out[d] = out[d - offset];
|
|
158
|
+
d++;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (d !== uncompressedLength) {
|
|
162
|
+
throw new Error(`Corrupt LZ4 block: decoded ${d} bytes, expected ${uncompressedLength}`);
|
|
163
|
+
}
|
|
164
|
+
return out;
|
|
165
|
+
}
|
|
166
|
+
//# sourceMappingURL=place-recompression.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"place-recompression.js","sourceRoot":"","sources":["../../../src/utils/build/place-recompression.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B;;;;;;;;;;;;;;GAcG;AAEH,8EAA8E;AAC9E,MAAM,UAAU,GAAG,UAAU,CAAC;AAE9B,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;AACtD,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B;;;GAGG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC;AACtB,iFAAiF;AACjF,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAqB5B,sDAAsD;AACtD,MAAM,UAAU,cAAc;IAG5B,MAAM,CAAC,GAAG,IAAoC,CAAC;IAC/C,IACE,OAAO,CAAC,CAAC,gBAAgB,KAAK,UAAU;QACxC,OAAO,CAAC,CAAC,kBAAkB,KAAK,UAAU,EAC1C,CAAC;QACD,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC7B,CAAC;IACD,OAAO;QACL,SAAS,EAAE,KAAK;QAChB,MAAM,EAAE,QAAQ,OAAO,CAAC,OAAO,sCAAsC;KACtE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,IACE,IAAI,CAAC,MAAM,GAAG,aAAa;QAC3B,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,EACzD,CAAC;QACD,uDAAuD;QACvD,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IACrC,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IACrC,CAAC;IAED,MAAM,OAAO,GAAG,cAAc,EAAE,CAAC;IACjC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QACvB,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;IAC/D,CAAC;IACD,MAAM,CAAC,GAAG,IAA2B,CAAC;IACtC,MAAM,MAAM,GAAG;QACb,CAAC,CAAC,CAAC,SAAS,CAAC,uBAAuB,CAAC,EAAE,iBAAiB;QACxD,CAAC,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,EAAE,UAAU;QAC1C,CAAC,CAAC,CAAC,SAAS,CAAC,iCAAiC,CAAC,EAAE,CAAC;KACnD,CAAC;IAEF,MAAM,KAAK,GAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC;IAC1D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,gBAAgB,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YAClD,SAAS;QACX,CAAC;QAED,MAAM,GAAG,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,CAAC,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAEvD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,mBAAmB,CAAC,CAC9D,CAAC;QACF,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC3C,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,OAAO;QACL,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;QAC1B,aAAa,EAAE,IAAI,CAAC,MAAM;KAC3B,CAAC;AACJ,CAAC;AAaD,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,GAAG,GAAG,aAAa,CAAC;IACxB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,OAAO,CAAC,MAAM,EAAE,CAAC;QACf,IAAI,GAAG,GAAG,mBAAmB,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;QACnD,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QACpD,MAAM,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QACtD,MAAM,YAAY,GAAG,GAAG,GAAG,mBAAmB,CAAC;QAC/C,MAAM,GAAG,GAAG,YAAY,GAAG,CAAC,gBAAgB,IAAI,kBAAkB,CAAC,CAAC;QACpE,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CACb,sCAAsC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAC1D,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;QACjD,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,GAAG;YACV,GAAG;YACH,gBAAgB;YAChB,kBAAkB;YAClB,OAAO;YACP,MAAM,EAAE,gBAAgB,IAAI,CAAC,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,UAAU;SACxE,CAAC,CAAC;QAEH,GAAG,GAAG,GAAG,CAAC;QACV,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC;IAC5B,CAAC;IAED,8EAA8E;IAC9E,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACtB,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,GAAG;YACV,GAAG,EAAE,IAAI,CAAC,MAAM;YAChB,gBAAgB,EAAE,CAAC;YACnB,kBAAkB,EAAE,IAAI,CAAC,MAAM,GAAG,GAAG;YACrC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YAC3B,MAAM,EAAE,KAAK;SACd,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,kBAAkB,CAChC,GAAW,EACX,kBAA0B;IAE1B,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IAC7C,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,CAAC,GAAG,CAAC,CAAC;IAEV,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,EAAE,CAAE,CAAC;QAExB,IAAI,aAAa,GAAG,KAAK,IAAI,CAAC,CAAC;QAC/B,IAAI,aAAa,KAAK,EAAE,EAAE,CAAC;YACzB,IAAI,CAAS,CAAC;YACd,GAAG,CAAC;gBACF,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,CAAE,CAAC;gBACd,aAAa,IAAI,CAAC,CAAC;YACrB,CAAC,QAAQ,CAAC,KAAK,GAAG,EAAE;QACtB,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,CAAC;QACvC,CAAC,IAAI,aAAa,CAAC;QACnB,CAAC,IAAI,aAAa,CAAC;QAEnB,sCAAsC;QACtC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM;YAAE,MAAM;QAE3B,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,CAAE,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAE,IAAI,CAAC,CAAC,CAAC;QAC5C,CAAC,IAAI,CAAC,CAAC;QACP,IAAI,MAAM,KAAK,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,WAAW,GAAG,KAAK,GAAG,EAAE,CAAC;QAC7B,IAAI,WAAW,KAAK,EAAE,EAAE,CAAC;YACvB,IAAI,CAAS,CAAC;YACd,GAAG,CAAC;gBACF,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,CAAE,CAAC;gBACd,WAAW,IAAI,CAAC,CAAC;YACnB,CAAC,QAAQ,CAAC,KAAK,GAAG,EAAE;QACtB,CAAC;QACD,WAAW,IAAI,CAAC,CAAC;QAEjB,+DAA+D;QAC/D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,MAAM,CAAE,CAAC;YAC1B,CAAC,EAAE,CAAC;QACN,CAAC;IACH,CAAC;IAED,IAAI,CAAC,KAAK,kBAAkB,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CACb,8BAA8B,CAAC,oBAAoB,kBAAkB,EAAE,CACxE,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"place-recompression.test.d.ts","sourceRoot":"","sources":["../../../src/utils/build/place-recompression.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import * as zlib from 'zlib';
|
|
2
|
+
import { describe, it, expect } from 'vitest';
|
|
3
|
+
import { decompressLz4Block, getZstdSupport, recompressPlace, } from './place-recompression.js';
|
|
4
|
+
const zstd = zlib;
|
|
5
|
+
/** "abcd", then 8 bytes copied from 4 back (overlapping), then "xy". */
|
|
6
|
+
const LZ4_BLOCK = Buffer.concat([
|
|
7
|
+
Buffer.from([0x44]),
|
|
8
|
+
Buffer.from('abcd'),
|
|
9
|
+
Buffer.from([0x04, 0x00]),
|
|
10
|
+
Buffer.from([0x20]),
|
|
11
|
+
Buffer.from('xy'),
|
|
12
|
+
]);
|
|
13
|
+
const LZ4_DECODED = 'abcdabcdabcdxy';
|
|
14
|
+
function chunk(name, payload, uncompressedLength, compressed) {
|
|
15
|
+
const header = Buffer.alloc(16);
|
|
16
|
+
header.write(name, 0, 'latin1');
|
|
17
|
+
header.writeUInt32LE(compressed ? payload.length : 0, 4);
|
|
18
|
+
header.writeUInt32LE(uncompressedLength, 8);
|
|
19
|
+
return Buffer.concat([header, payload]);
|
|
20
|
+
}
|
|
21
|
+
function place(...chunks) {
|
|
22
|
+
const header = Buffer.alloc(32);
|
|
23
|
+
header.write('<roblox!', 0, 'latin1');
|
|
24
|
+
return Buffer.concat([header, ...chunks]);
|
|
25
|
+
}
|
|
26
|
+
const END = chunk('END\0', Buffer.from('</roblox>'), 9, false);
|
|
27
|
+
describe('decompressLz4Block', () => {
|
|
28
|
+
it('decodes literals and an overlapping match', () => {
|
|
29
|
+
expect(decompressLz4Block(LZ4_BLOCK, LZ4_DECODED.length).toString()).toBe(LZ4_DECODED);
|
|
30
|
+
});
|
|
31
|
+
it('rejects a block that decodes to the wrong length', () => {
|
|
32
|
+
expect(() => decompressLz4Block(LZ4_BLOCK, LZ4_DECODED.length + 1)).toThrow(/expected/);
|
|
33
|
+
});
|
|
34
|
+
it('rejects a match reaching before the start of the output', () => {
|
|
35
|
+
const bad = Buffer.from([0x14, 0x61, 0x09, 0x00, 0x00]);
|
|
36
|
+
expect(() => decompressLz4Block(bad, 13)).toThrow(/offset/);
|
|
37
|
+
});
|
|
38
|
+
});
|
|
39
|
+
describe.runIf(getZstdSupport().supported)('recompressPlace', () => {
|
|
40
|
+
it('rewrites LZ4 chunks as zstd holding the same bytes, and keeps the rest', () => {
|
|
41
|
+
const original = place(chunk('PROP', LZ4_BLOCK, LZ4_DECODED.length, true), END);
|
|
42
|
+
const result = recompressPlace(original);
|
|
43
|
+
expect(result.kind).toBe('recompressed');
|
|
44
|
+
const data = result.data;
|
|
45
|
+
expect(data.subarray(0, 32)).toEqual(original.subarray(0, 32));
|
|
46
|
+
expect(data.toString('latin1', 32, 36)).toBe('PROP');
|
|
47
|
+
const compressedLength = data.readUInt32LE(36);
|
|
48
|
+
expect(data.readUInt32LE(40)).toBe(LZ4_DECODED.length);
|
|
49
|
+
const payload = data.subarray(48, 48 + compressedLength);
|
|
50
|
+
expect(payload.readUInt32LE(0)).toBe(0xfd2fb528);
|
|
51
|
+
expect(zstd.zstdDecompressSync(payload).toString()).toBe(LZ4_DECODED);
|
|
52
|
+
// END is uncompressed, so it comes through untouched.
|
|
53
|
+
expect(data.subarray(48 + compressedLength)).toEqual(END);
|
|
54
|
+
});
|
|
55
|
+
it('leaves a place with nothing left to recompress alone', () => {
|
|
56
|
+
const once = recompressPlace(place(chunk('PROP', LZ4_BLOCK, LZ4_DECODED.length, true), END)).data;
|
|
57
|
+
expect(recompressPlace(once)).toEqual({ kind: 'unchanged', data: once });
|
|
58
|
+
});
|
|
59
|
+
it('leaves files that are not binary places alone', () => {
|
|
60
|
+
const xml = Buffer.from('<roblox version="4"></roblox>');
|
|
61
|
+
expect(recompressPlace(xml)).toEqual({ kind: 'unchanged', data: xml });
|
|
62
|
+
});
|
|
63
|
+
it('refuses a truncated place rather than uploading part of it', () => {
|
|
64
|
+
const truncated = place(chunk('PROP', LZ4_BLOCK, LZ4_DECODED.length, true)).subarray(0, 40);
|
|
65
|
+
expect(() => recompressPlace(truncated)).toThrow(/truncated/);
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
//# sourceMappingURL=place-recompression.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"place-recompression.test.js","sourceRoot":"","sources":["../../../src/utils/build/place-recompression.test.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EACL,kBAAkB,EAClB,cAAc,EACd,eAAe,GAChB,MAAM,0BAA0B,CAAC;AAElC,MAAM,IAAI,GAAG,IAA4D,CAAC;AAE1E,wEAAwE;AACxE,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC;IAC9B,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;IACnB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;IACnB,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACzB,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;IACnB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;CAClB,CAAC,CAAC;AACH,MAAM,WAAW,GAAG,gBAAgB,CAAC;AAErC,SAAS,KAAK,CACZ,IAAY,EACZ,OAAe,EACf,kBAA0B,EAC1B,UAAmB;IAEnB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IAChC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzD,MAAM,CAAC,aAAa,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC;IAC5C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,KAAK,CAAC,GAAG,MAAgB;IAChC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;AAE/D,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IAClC,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,CAAC,kBAAkB,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CACvE,WAAW,CACZ,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,CAAC,GAAG,EAAE,CAAC,kBAAkB,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CACzE,UAAU,CACX,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,GAAG,EAAE,CAAC,kBAAkB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC,SAAS,CAAC,CAAC,iBAAiB,EAAE,GAAG,EAAE;IACjE,EAAE,CAAC,wEAAwE,EAAE,GAAG,EAAE;QAChF,MAAM,QAAQ,GAAG,KAAK,CACpB,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,EAClD,GAAG,CACJ,CAAC;QAEF,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAEzC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC/D,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAErD,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,GAAG,gBAAgB,CAAC,CAAC;QACzD,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACjD,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEtE,sDAAsD;QACtD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,GAAG,gBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,IAAI,GAAG,eAAe,CAC1B,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,CAC/D,CAAC,IAAI,CAAC;QAEP,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QACzD,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACpE,MAAM,SAAS,GAAG,KAAK,CACrB,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CACnD,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClB,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -6,10 +6,13 @@ import { type BatchDiagnosticLevel, type BatchPackageResult } from '../testing/p
|
|
|
6
6
|
import { type LogFetchStats } from '../testing/log-fetch-stats.js';
|
|
7
7
|
import { type RenderableLogLine } from '../testing/parsers/batch-log-renderer.js';
|
|
8
8
|
/**
|
|
9
|
-
* JobContext wrapper that batches all packages into a single build,
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
9
|
+
* JobContext wrapper that batches all packages into a single build, and as few
|
|
10
|
+
* uploads and executions as the place size budget allows — one, unless the
|
|
11
|
+
* combined place is too big even recompressed, in which case it is split into
|
|
12
|
+
* chunks that each upload and run on their own, in parallel. Wraps an inner
|
|
13
|
+
* context (cloud or local) and intercepts all methods using the lazy-promise
|
|
14
|
+
* pattern: the first concurrent call triggers the shared operation, all others
|
|
15
|
+
* await the same promise.
|
|
13
16
|
*
|
|
14
17
|
* Note that a `basePlace` cannot be honoured here: every package is combined
|
|
15
18
|
* into one place, so there is no per-package place to merge Studio content
|
|
@@ -48,17 +51,22 @@ export declare class BatchScriptJobContext implements JobContext {
|
|
|
48
51
|
private _batchPlaceId?;
|
|
49
52
|
private _batchUniverseId?;
|
|
50
53
|
private _batchTimeoutMs;
|
|
54
|
+
private _maxChunkBytes?;
|
|
51
55
|
private _reporter?;
|
|
52
56
|
private _combinedBuildPromise?;
|
|
53
|
-
private
|
|
57
|
+
private _chunkRuns?;
|
|
54
58
|
private _executionPromise?;
|
|
55
59
|
private _combinedBuildContext?;
|
|
56
|
-
private _sharedDeployment?;
|
|
57
60
|
constructor(inner: JobContext, batchTargets: BatchTarget[], options?: {
|
|
58
61
|
repoRoot?: string;
|
|
59
62
|
batchPlaceId?: number;
|
|
60
63
|
batchUniverseId?: number;
|
|
61
64
|
batchTimeoutMs?: number;
|
|
65
|
+
/**
|
|
66
|
+
* Per-place size budget; see `DEFAULT_MAX_CHUNK_BYTES`. Only an upload
|
|
67
|
+
* is size-limited, so a local run passes `Infinity`.
|
|
68
|
+
*/
|
|
69
|
+
maxChunkBytes?: number;
|
|
62
70
|
reporter?: Reporter;
|
|
63
71
|
});
|
|
64
72
|
buildPlaceAsync(options: BuildPlaceOptions): Promise<BuiltPlace>;
|
|
@@ -78,9 +86,13 @@ export declare class BatchScriptJobContext implements JobContext {
|
|
|
78
86
|
disposeAsync(): Promise<void>;
|
|
79
87
|
private _getCombinedBuildAsync;
|
|
80
88
|
private _doCombinedBuildAsync;
|
|
81
|
-
private
|
|
82
|
-
private
|
|
89
|
+
private _getChunkRunsAsync;
|
|
90
|
+
private _getChunkRunAsync;
|
|
91
|
+
private _getChunkDeploymentAsync;
|
|
92
|
+
private _doChunkDeployAsync;
|
|
83
93
|
private _getBatchExecutionAsync;
|
|
84
94
|
private _doBatchExecutionAsync;
|
|
95
|
+
private _getChunkExecutionAsync;
|
|
96
|
+
private _doChunkExecutionAsync;
|
|
85
97
|
}
|
|
86
98
|
//# sourceMappingURL=batch-script-job-context.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-script-job-context.d.ts","sourceRoot":"","sources":["../../../src/utils/job-context/batch-script-job-context.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,sCAAsC,CAAC;AACrE,OAAO,EACL,KAAK,UAAU,EACf,KAAK,UAAU,EACf,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,OAAO,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,KAAK,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,oCAAoC,CAAC;
|
|
1
|
+
{"version":3,"file":"batch-script-job-context.d.ts","sourceRoot":"","sources":["../../../src/utils/job-context/batch-script-job-context.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,sCAAsC,CAAC;AACrE,OAAO,EACL,KAAK,UAAU,EACf,KAAK,UAAU,EACf,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,OAAO,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,KAAK,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,oCAAoC,CAAC;AAQtE,OAAO,EACL,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EAExB,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,0CAA0C,CAAC;AAkBlD;;;;;;;;;;;;;;GAcG;AACH;;;;;;;;;GASG;AACH,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACzC,gEAAgE;IAChE,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,sFAAsF;IACtF,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,sEAAsE;IACtE,WAAW,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,oBAAoB,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACrE,qEAAqE;IACrE,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACpC;AAED,qBAAa,qBAAsB,YAAW,UAAU;IACtD,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,cAAc,CAAC,CAAS;IAChC,OAAO,CAAC,SAAS,CAAC,CAAW;IAG7B,OAAO,CAAC,qBAAqB,CAAC,CAAiC;IAC/D,OAAO,CAAC,UAAU,CAAC,CAAa;IAChC,OAAO,CAAC,iBAAiB,CAAC,CAAiC;IAG3D,OAAO,CAAC,qBAAqB,CAAC,CAAe;gBAG3C,KAAK,EAAE,UAAU,EACjB,YAAY,EAAE,WAAW,EAAE,EAC3B,OAAO,CAAC,EAAE;QACR,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB;;;WAGG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,EAAE,QAAQ,CAAC;KACrB;IAwBG,eAAe,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,UAAU,CAAC;IAahE,qBAAqB,CACzB,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,UAAU,CAAC;IAOhB,cAAc,CAClB,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,eAAe,CAAC;IAwB3B;;;;;;OAMG;IACG,wBAAwB,IAAI,OAAO,CAAC,qBAAqB,CAAC;IAI1D,YAAY,CAAC,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC;IAatD,YAAY,CAAC,WAAW,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAIpD,sBAAsB,CAAC,WAAW,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAI9D,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBnC,OAAO,CAAC,sBAAsB;YAOhB,qBAAqB;YAgDrB,kBAAkB;YAKlB,iBAAiB;IAW/B,OAAO,CAAC,wBAAwB;YAWlB,mBAAmB;IAqBjC,OAAO,CAAC,uBAAuB;YAWjB,sBAAsB;IAyBpC,OAAO,CAAC,uBAAuB;YAajB,sBAAsB;CAmFrC"}
|