@quenty/nevermore-cli 4.48.0 → 4.50.0

package/src/utils/build/place-recompression.ts

@@ -0,0 +1,236 @@
+import * as zlib from 'zlib';
+
+/**
+ * Rewrite a binary place file's compressed chunks as long-window zstd.
+ *
+ * Every script's source lives in one `PROP` chunk per class, and Rojo and Lune
+ * compress chunks as LZ4, whose 64 KB window cannot reach a module the place
+ * already holds a copy of. A Nevermore place carries its dependencies by copy
+ * — a combined batch place a copy per package — so the same modules repeat
+ * many times over and LZ4 stores every one. zstd with a long window and
+ * long-distance matching finds them: a 40-package batch place measured 38.3 MB
+ * as LZ4 and 1.3 MB recompressed, and ran identically.
+ *
+ * The binary format picks the codec per chunk from its leading bytes, so this
+ * changes how the file is stored, not what it holds. Chunk payloads are
+ * decompressed and recompressed byte for byte; nothing is parsed.
+ */
+
+/** zstd frame magic; the format reads any other compressed payload as LZ4. */
+const ZSTD_MAGIC = 0xfd2fb528;
+
+const PLACE_MAGIC = Buffer.from('<roblox!', 'latin1');
+const HEADER_LENGTH = 32;
+const CHUNK_HEADER_LENGTH = 16;
+
+/**
+ * 2^27, zstd's default decoder window limit. A larger window would need the
+ * decoder to opt in, and this is the size Roblox has been seen to decode.
+ */
+const WINDOW_LOG = 27;
+/** Level 9 is within 15% of level 19's size here at a fourteenth of the time. */
+const COMPRESSION_LEVEL = 9;
+
+/**
+ * `zstdCompressSync` landed in Node 22.15 / 23.8 and the installed typings
+ * predate it, so it is looked up rather than imported.
+ */
+interface ZstdZlib {
+  zstdCompressSync(
+    buffer: Buffer,
+    options: { params: Record<number, number> }
+  ): Buffer;
+  zstdDecompressSync(buffer: Buffer): Buffer;
+  constants: Record<string, number>;
+}
+
+export type PlaceRecompressionResult =
+  | { kind: 'recompressed'; data: Buffer; originalBytes: number }
+  /** Not a binary place, or every compressed chunk is already zstd. */
+  | { kind: 'unchanged'; data: Buffer }
+  | { kind: 'unsupported'; data: Buffer; reason: string };
+
+/** Whether this Node can recompress places at all. */
+export function getZstdSupport():
+  | { supported: true }
+  | { supported: false; reason: string } {
+  const z = zlib as unknown as Partial<ZstdZlib>;
+  if (
+    typeof z.zstdCompressSync === 'function' &&
+    typeof z.zstdDecompressSync === 'function'
+  ) {
+    return { supported: true };
+  }
+  return {
+    supported: false,
+    reason: `Node ${process.version} has no zstd (needs 22.15+ or 23.8+)`,
+  };
+}
+
+export function recompressPlace(data: Buffer): PlaceRecompressionResult {
+  if (
+    data.length < HEADER_LENGTH ||
+    !data.subarray(0, PLACE_MAGIC.length).equals(PLACE_MAGIC)
+  ) {
+    // An XML place, or not a place: nothing to recompress.
+    return { kind: 'unchanged', data };
+  }
+
+  const chunks = _readChunks(data);
+  if (!chunks.some((c) => c.compressedLength > 0 && !c.isZstd)) {
+    return { kind: 'unchanged', data };
+  }
+
+  const support = getZstdSupport();
+  if (!support.supported) {
+    return { kind: 'unsupported', data, reason: support.reason };
+  }
+  const z = zlib as unknown as ZstdZlib;
+  const params = {
+    [z.constants.ZSTD_c_compressionLevel]: COMPRESSION_LEVEL,
+    [z.constants.ZSTD_c_windowLog]: WINDOW_LOG,
+    [z.constants.ZSTD_c_enableLongDistanceMatching]: 1,
+  };
+
+  const parts: Buffer[] = [data.subarray(0, HEADER_LENGTH)];
+  for (const chunk of chunks) {
+    if (chunk.compressedLength === 0 || chunk.isZstd) {
+      parts.push(data.subarray(chunk.start, chunk.end));
+      continue;
+    }
+
+    const raw = decompressLz4Block(chunk.payload, chunk.uncompressedLength);
+    const compressed = z.zstdCompressSync(raw, { params });
+
+    const header = Buffer.from(
+      data.subarray(chunk.start, chunk.start + CHUNK_HEADER_LENGTH)
+    );
+    header.writeUInt32LE(compressed.length, 4);
+    parts.push(header, compressed);
+  }
+
+  return {
+    kind: 'recompressed',
+    data: Buffer.concat(parts),
+    originalBytes: data.length,
+  };
+}
+
+interface PlaceChunk {
+  /** Offset of the chunk header. */
+  start: number;
+  /** Offset just past the payload. */
+  end: number;
+  compressedLength: number;
+  uncompressedLength: number;
+  payload: Buffer;
+  isZstd: boolean;
+}
+
+function _readChunks(data: Buffer): PlaceChunk[] {
+  const chunks: PlaceChunk[] = [];
+  let pos = HEADER_LENGTH;
+  let sawEnd = false;
+
+  while (!sawEnd) {
+    if (pos + CHUNK_HEADER_LENGTH > data.length) {
+      throw new Error('Place file is truncated before its END chunk');
+    }
+
+    const name = data.toString('latin1', pos, pos + 4);
+    const compressedLength = data.readUInt32LE(pos + 4);
+    const uncompressedLength = data.readUInt32LE(pos + 8);
+    const payloadStart = pos + CHUNK_HEADER_LENGTH;
+    const end = payloadStart + (compressedLength || uncompressedLength);
+    if (end > data.length) {
+      throw new Error(
+        `Place file is truncated inside its ${name.trim()} chunk`
+      );
+    }
+
+    const payload = data.subarray(payloadStart, end);
+    chunks.push({
+      start: pos,
+      end,
+      compressedLength,
+      uncompressedLength,
+      payload,
+      isZstd: compressedLength >= 4 && payload.readUInt32LE(0) === ZSTD_MAGIC,
+    });
+
+    pos = end;
+    sawEnd = name === 'END\0';
+  }
+
+  // Anything after END is not the format's, but it is kept rather than dropped.
+  if (pos < data.length) {
+    chunks.push({
+      start: pos,
+      end: data.length,
+      compressedLength: 0,
+      uncompressedLength: data.length - pos,
+      payload: data.subarray(pos),
+      isZstd: false,
+    });
+  }
+
+  return chunks;
+}
+
+/** Decode one raw LZ4 block, the format's default chunk compression. */
+export function decompressLz4Block(
+  src: Buffer,
+  uncompressedLength: number
+): Buffer {
+  const out = Buffer.alloc(uncompressedLength);
+  let s = 0;
+  let d = 0;
+
+  while (s < src.length) {
+    const token = src[s++]!;
+
+    let literalLength = token >> 4;
+    if (literalLength === 15) {
+      let b: number;
+      do {
+        b = src[s++]!;
+        literalLength += b;
+      } while (b === 255);
+    }
+    src.copy(out, d, s, s + literalLength);
+    s += literalLength;
+    d += literalLength;
+
+    // The last sequence is literals only.
+    if (s >= src.length) break;
+
+    const offset = src[s]! | (src[s + 1]! << 8);
+    s += 2;
+    if (offset === 0 || offset > d) {
+      throw new Error('Corrupt LZ4 block: match offset out of range');
+    }
+
+    let matchLength = token & 15;
+    if (matchLength === 15) {
+      let b: number;
+      do {
+        b = src[s++]!;
+        matchLength += b;
+      } while (b === 255);
+    }
+    matchLength += 4;
+
+    // Byte by byte: a match may overlap the bytes it is producing.
+    for (let i = 0; i < matchLength; i++) {
+      out[d] = out[d - offset]!;
+      d++;
+    }
+  }
+
+  if (d !== uncompressedLength) {
+    throw new Error(
+      `Corrupt LZ4 block: decoded ${d} bytes, expected ${uncompressedLength}`
+    );
+  }
+  return out;
+}

package/src/utils/job-context/batch-script-job-context.ts

@@ -18,7 +18,9 @@ import { type BuildPlaceOptions } from '../build/build.js';
 import { type BatchTarget } from '../batch/changed-packages-utils.js';
 import {
   type CombinedBuildProgress,
+  type CombinedPlaceChunk,
   type CombinedProjectResult,
+  formatMegabytes,
   generateCombinedProjectAsync,
 } from '../testing/runner/combined-project-generator.js';
 import {
@@ -40,16 +42,22 @@ class BatchDeployment implements Deployment {
   ) {}
 }
 
-interface CombinedBuildState {
-  combinedResult: CombinedProjectResult;
-  rbxlPath: string;
+/** One combined place's upload and execution, each started at most once. */
+interface ChunkRun {
+  chunk: CombinedPlaceChunk;
+  deployPromise?: Promise<Deployment>;
+  deployment?: Deployment;
+  executionPromise?: Promise<BatchExecutionOutcome>;
 }
 
 /**
- * JobContext wrapper that batches all packages into a single build, upload, and
- * execution. Wraps an inner context (cloud or local) and intercepts all methods
- * using the lazy-promise pattern: the first concurrent call triggers the shared
- * operation, all others await the same promise.
+ * JobContext wrapper that batches all packages into a single build, and as few
+ * uploads and executions as the place size budget allows — one, unless the
+ * combined place is too big even recompressed, in which case it is split into
+ * chunks that each upload and run on their own, in parallel. Wraps an inner
+ * context (cloud or local) and intercepts all methods using the lazy-promise
+ * pattern: the first concurrent call triggers the shared operation, all others
+ * await the same promise.
  *
  * Note that a `basePlace` cannot be honoured here: every package is combined
  * into one place, so there is no per-package place to merge Studio content
@@ -86,16 +94,16 @@ export class BatchScriptJobContext implements JobContext {
   private _batchPlaceId?: number;
   private _batchUniverseId?: number;
   private _batchTimeoutMs: number;
+  private _maxChunkBytes?: number;
   private _reporter?: Reporter;
 
   // Lazy-promise state
-  private _combinedBuildPromise?: Promise<CombinedBuildState>;
-  private _deployPromise?: Promise<Deployment>;
+  private _combinedBuildPromise?: Promise<CombinedProjectResult>;
+  private _chunkRuns?: ChunkRun[];
   private _executionPromise?: Promise<BatchExecutionOutcome>;
 
   // State for cleanup
   private _combinedBuildContext?: BuildContext;
-  private _sharedDeployment?: Deployment;
 
   constructor(
     inner: JobContext,
@@ -105,6 +113,11 @@ export class BatchScriptJobContext implements JobContext {
       batchPlaceId?: number;
       batchUniverseId?: number;
       batchTimeoutMs?: number;
+      /**
+       * Per-place size budget; see `DEFAULT_MAX_CHUNK_BYTES`. Only an upload
+       * is size-limited, so a local run passes `Infinity`.
+       */
+      maxChunkBytes?: number;
       reporter?: Reporter;
     }
   ) {
@@ -114,6 +127,7 @@ export class BatchScriptJobContext implements JobContext {
     this._batchPlaceId = options?.batchPlaceId;
     this._batchUniverseId = options?.batchUniverseId;
     this._batchTimeoutMs = options?.batchTimeoutMs ?? 300_000;
+    this._maxChunkBytes = options?.maxChunkBytes;
     this._reporter = options?.reporter;
 
     const withBasePlace = batchTargets.filter((t) => t.target.basePlace);
@@ -130,12 +144,14 @@ export class BatchScriptJobContext implements JobContext {
   }
 
   async buildPlaceAsync(options: BuildPlaceOptions): Promise<BuiltPlace> {
-    const buildState = await this._getCombinedBuildAsync();
+    const chunkRun = options.packageName
+      ? await this._getChunkRunAsync(options.packageName)
+      : (await this._getChunkRunsAsync())[0]!;
     // The script content baked into BuiltPlace.target is read by
     // runSingleTestAsync but discarded in aggregated batch mode — we just
     // echo back the caller's resolved place.
     return {
-      rbxlPath: buildState.rbxlPath,
+      rbxlPath: chunkRun.chunk.rbxlPath,
       target: options.target,
     };
   }
@@ -143,7 +159,8 @@ export class BatchScriptJobContext implements JobContext {
   async deployBuiltPlaceAsync(
     options: DeployPlaceOptions
   ): Promise<Deployment> {
-    const innerDeployment = await this._getSharedDeploymentAsync();
+    const chunkRun = await this._getChunkRunAsync(options.packageName);
+    const innerDeployment = await this._getChunkDeploymentAsync(chunkRun);
 
     return new BatchDeployment(options.packageName, innerDeployment);
   }
@@ -208,10 +225,11 @@ export class BatchScriptJobContext implements JobContext {
   }
 
   async disposeAsync(): Promise<void> {
-    // Release shared inner deployment
-    if (this._sharedDeployment) {
+    // Release each chunk's inner deployment
+    for (const { deployment } of this._chunkRuns ?? []) {
+      if (!deployment) continue;
       try {
-        await this._inner.releaseAsync(this._sharedDeployment);
+        await this._inner.releaseAsync(deployment);
       } catch {
         // best effort
       }
@@ -231,14 +249,14 @@ export class BatchScriptJobContext implements JobContext {
 
   // ---- Lazy-promise internals ----
 
-  private _getCombinedBuildAsync(): Promise<CombinedBuildState> {
+  private _getCombinedBuildAsync(): Promise<CombinedProjectResult> {
     if (!this._combinedBuildPromise) {
       this._combinedBuildPromise = this._doCombinedBuildAsync();
     }
     return this._combinedBuildPromise;
   }
 
-  private async _doCombinedBuildAsync(): Promise<CombinedBuildState> {
+  private async _doCombinedBuildAsync(): Promise<CombinedProjectResult> {
     OutputHelper.verbose('Building combined batch place...');
 
     // Set all packages to "waiting" — they're queued for building
@@ -276,37 +294,61 @@ export class BatchScriptJobContext implements JobContext {
       repoRoot: this._repoRoot,
       batchPlaceId: this._batchPlaceId,
       batchUniverseId: this._batchUniverseId,
+      maxChunkBytes: this._maxChunkBytes,
       progress,
     });
 
     this._combinedBuildContext = combinedResult.buildContext;
+    this._chunkRuns = combinedResult.chunks.map((chunk) => ({ chunk }));
 
-    return { combinedResult, rbxlPath: combinedResult.rbxlPath };
+    return combinedResult;
   }
 
-  private _getSharedDeploymentAsync(): Promise<Deployment> {
-    if (!this._deployPromise) {
-      this._deployPromise = this._doSharedDeployAsync();
+  private async _getChunkRunsAsync(): Promise<ChunkRun[]> {
+    await this._getCombinedBuildAsync();
+    return this._chunkRuns!;
+  }
+
+  private async _getChunkRunAsync(packageName: string): Promise<ChunkRun> {
+    const chunkRuns = await this._getChunkRunsAsync();
+    const chunkRun = chunkRuns.find((run) =>
+      run.chunk.slugMap.has(packageName)
+    );
+    if (!chunkRun) {
+      throw new Error(`${packageName} is not part of this batch`);
     }
-    return this._deployPromise;
+    return chunkRun;
   }
 
-  private async _doSharedDeployAsync(): Promise<Deployment> {
-    const buildState = await this._getCombinedBuildAsync();
-    const { primaryTarget } = buildState.combinedResult;
+  private _getChunkDeploymentAsync(chunkRun: ChunkRun): Promise<Deployment> {
+    if (!chunkRun.deployPromise) {
+      chunkRun.deployPromise = this._doChunkDeployAsync(chunkRun);
+      // A failed upload is retried with its chunk, not remembered as failed
+      chunkRun.deployPromise.catch(() => {
+        chunkRun.deployPromise = undefined;
+      });
+    }
+    return chunkRun.deployPromise;
+  }
+
+  private async _doChunkDeployAsync(chunkRun: ChunkRun): Promise<Deployment> {
+    const { primaryTarget } = await this._getCombinedBuildAsync();
 
     const builtPlace: BuiltPlace = {
-      rbxlPath: buildState.rbxlPath,
+      rbxlPath: chunkRun.chunk.rbxlPath,
       target: primaryTarget,
     };
 
+    // Every chunk uploads to the same place. Each upload is its own version and
+    // each execution is pinned to the version it was given, so a later upload
+    // cannot change what an earlier chunk runs against.
     const deployment = await this._inner.deployBuiltPlaceAsync({
       builtPlace,
       packageName: '_batch_',
       packagePath: this._repoRoot,
     });
 
-    this._sharedDeployment = deployment;
+    chunkRun.deployment = deployment;
     return deployment;
   }
 
@@ -322,9 +364,48 @@ export class BatchScriptJobContext implements JobContext {
   }
 
   private async _doBatchExecutionAsync(): Promise<BatchExecutionOutcome> {
-    const buildState = await this._getCombinedBuildAsync();
-    const deployment = await this._getSharedDeploymentAsync();
-    const { slugMap } = buildState.combinedResult;
+    const chunkRuns = await this._getChunkRunsAsync();
+
+    if (chunkRuns.length > 1) {
+      const sizes = chunkRuns
+        .map(
+          ({ chunk }) =>
+            `${chunk.slugMap.size} packages, ${formatMegabytes(
+              chunk.sizeBytes
+            )}`
+        )
+        .join('; ');
+      OutputHelper.info(`Running ${chunkRuns.length} places: ${sizes}`);
+    }
+
+    // All at once: every upload is its own version and each execution is
+    // pinned to the version it was given, so the places cannot interfere.
+    // Promise.all keeps chunk order, so the merged log stays in package order.
+    const outcomes = await Promise.all(
+      chunkRuns.map((chunkRun) => this._getChunkExecutionAsync(chunkRun))
+    );
+
+    return _mergeOutcomes(outcomes);
+  }
+
+  private _getChunkExecutionAsync(
+    chunkRun: ChunkRun
+  ): Promise<BatchExecutionOutcome> {
+    if (!chunkRun.executionPromise) {
+      chunkRun.executionPromise = this._doChunkExecutionAsync(chunkRun);
+      // Cleared on rejection, so a retry re-runs only the chunks that failed
+      chunkRun.executionPromise.catch(() => {
+        chunkRun.executionPromise = undefined;
+      });
+    }
+    return chunkRun.executionPromise;
+  }
+
+  private async _doChunkExecutionAsync(
+    chunkRun: ChunkRun
+  ): Promise<BatchExecutionOutcome> {
+    const deployment = await this._getChunkDeploymentAsync(chunkRun);
+    const { slugMap } = chunkRun.chunk;
 
     // Build the batch Luau script from the template
     const templatePath = resolveTemplatePath(
@@ -404,3 +485,31 @@ export class BatchScriptJobContext implements JobContext {
     };
   }
 }
+
+/** Fold each chunk's outcome into one, in chunk order. */
+function _mergeOutcomes(
+  outcomes: BatchExecutionOutcome[]
+): BatchExecutionOutcome {
+  if (outcomes.length === 1) {
+    return outcomes[0]!;
+  }
+
+  const allStats = outcomes.map((o) => o.stats);
+  const stats = allStats.every((s): s is LogFetchStats => s !== undefined)
+    ? allStats.reduce((sum, s) => ({
+        requests: sum.requests + s.requests,
+        pages: sum.pages + s.pages,
+        entries: sum.entries + s.entries,
+        messages: sum.messages + s.messages,
+        chars: sum.chars + s.chars,
+      }))
+    : undefined;
+
+  return {
+    results: new Map(outcomes.flatMap((o) => [...o.results])),
+    lines: outcomes.flatMap((o) => o.lines),
+    stats,
+    diagnostics: outcomes.flatMap((o) => o.diagnostics),
+    slugToPackage: new Map(outcomes.flatMap((o) => [...o.slugToPackage])),
+  };
+}

package/src/utils/open-cloud/open-cloud-client.ts

@@ -13,6 +13,20 @@ import {
   type ParsedTestLogs,
 } from '../testing/test-log-parser.js';
 import { type LogFetchStats } from '../testing/log-fetch-stats.js';
+import { recompressPlace } from '../build/place-recompression.js';
+
+/**
+ * The largest place file the place version endpoint accepts, measured: 150 MiB
+ * exactly uploads, one byte more is refused from `Content-Length` before the
+ * body is sent, as `400 InvalidRequest: "Request upload was interrupted before
+ * completion."` — and somewhere past ~200 MB as `413 Payload Too Large`.
+ * Neither says anything about size, so it is checked here first.
+ *
+ * Not the 100 MB Roblox documents as supported
+ * (https://create.roblox.com/docs/projects/place-files); that is where Studio
+ * saving "might fail", not where the API cuts off.
+ */
+export const PLACE_UPLOAD_MAX_BYTES = 150 * 1024 * 1024;
 
 /**
  * One engine message, with the severity Open Cloud reported for it.
@@ -276,8 +290,20 @@ export class OpenCloudClient {
     );
 
     const apiKey = await this._resolveApiKeyAsync();
-    const fileBuffer = await fs.readFile(rbxlPath);
+    const fileBuffer = _recompressForUpload(
+      rbxlPath,
+      await fs.readFile(rbxlPath)
+    );
     const totalBytes = fileBuffer.length;
+    if (totalBytes > PLACE_UPLOAD_MAX_BYTES) {
+      throw new Error(
+        `Upload failed: ${path.basename(rbxlPath)} is ${_formatMegabytes(
+          totalBytes
+        )}, over the Open Cloud ${_formatMegabytes(
+          PLACE_UPLOAD_MAX_BYTES
+        )} place upload limit`
+      );
+    }
     const versionType = publish ? 'Published' : 'Saved';
     const url = `https://apis.roblox.com/universes/v1/${universeId}/places/${placeId}/versions?versionType=${versionType}`;
 
@@ -1097,3 +1123,34 @@ async function _readBodyWithProgressAsync(
 
   return Buffer.concat(chunks);
 }
+
+/**
+ * The bytes to upload for a place file: recompressed when that is possible,
+ * the file as read when it is not. An upload never fails for want of zstd —
+ * it only goes up larger.
+ */
+function _recompressForUpload(rbxlPath: string, fileBuffer: Buffer): Buffer {
+  const name = path.basename(rbxlPath);
+  const result = recompressPlace(fileBuffer);
+
+  if (result.kind === 'recompressed') {
+    OutputHelper.verbose(
+      `Recompressed ${name}: ${_formatMegabytes(
+        result.originalBytes
+      )} -> ${_formatMegabytes(result.data.length)}`
+    );
+  } else if (result.kind === 'unsupported') {
+    OutputHelper.warn(
+      `Uploading ${name} as built (${_formatMegabytes(
+        fileBuffer.length
+      )}) without recompressing it: ${result.reason}`
+    );
+  }
+
+  return result.data;
+}
+
+/** In Roblox's MB, which are 1024 * 1024 bytes. */
+function _formatMegabytes(bytes: number): string {
+  return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
+}