@quenty/nevermore-cli 4.50.1 → 4.52.0

package/src/commands/batch-command/batch-bundle-command.ts

@@ -0,0 +1,236 @@
+import * as path from 'path';
+import { CommandModule } from 'yargs';
+import { OutputHelper } from '@quenty/cli-output-helpers';
+import { BaseReporter } from '@quenty/cli-output-helpers/reporting';
+import { NevermoreGlobalArgs } from '../../args/global-args.js';
+import {
+  BatchScriptJobContext,
+  LocalJobContext,
+} from '../../utils/job-context/index.js';
+import {
+  discoverAllTestableBatchTargetsAsync,
+  discoverChangedTestableBatchTargetsAsync,
+  discoverDeployBatchTargetsAsync,
+} from '../../utils/batch/changed-packages-utils.js';
+import { gatherGitDeployInfo } from '../../utils/deploy/deploy-metadata.js';
+import {
+  DEPLOY_BUNDLE_FILE_SUFFIX,
+  type PrebuiltDeployPlace,
+  writePrebuiltDeployBundleAsync,
+} from '../../utils/deploy/prebuilt-deploy-bundle.js';
+import { readPackageVersionAsync } from '../../utils/nevermore-cli-utils.js';
+import { prepareTestAsync } from '../../utils/testing/runner/test-runner.js';
+import {
+  BUNDLE_FILE_SUFFIX,
+  writePrebuiltBatchBundleAsync,
+} from '../../utils/testing/runner/prebuilt-batch-bundle.js';
+
+interface BatchBundleArgs extends NevermoreGlobalArgs {
+  file: string;
+  test?: boolean;
+  deploy?: boolean;
+  target?: string;
+  base?: string;
+  all?: boolean;
+  limit?: number;
+  maxPlaceMb?: number;
+}
+
+export const batchBundleCommand: CommandModule<
+  NevermoreGlobalArgs,
+  BatchBundleArgs
+> = {
+  command: 'bundle <file>',
+  describe:
+    'Build what "batch test" or "batch deploy" would run into one bundle file, without an API key. ' +
+    'Run it later with "batch test --bundle <file>" or "batch deploy --bundle <file>".',
+  builder: (yargs) => {
+    return yargs
+      .positional('file', {
+        describe: `File to write, conventionally <name>${BUNDLE_FILE_SUFFIX} or <name>${DEPLOY_BUNDLE_FILE_SUFFIX}`,
+        type: 'string',
+        demandOption: true,
+      })
+      .option('test', {
+        describe: 'Bundle the combined test places "batch test" would run',
+        type: 'boolean',
+      })
+      .option('deploy', {
+        describe:
+          'Bundle the places "batch deploy" would deploy, without their base place, which the deploying run merges',
+        type: 'boolean',
+      })
+      .option('target', {
+        describe: 'Deploy target name in deploy.nevermore.json (--deploy only)',
+        type: 'string',
+      })
+      .option('all', {
+        describe: 'Bundle every package, not just changed ones',
+        type: 'boolean',
+        default: false,
+      })
+      .option('base', {
+        describe: 'Git ref to diff against for change detection',
+        type: 'string',
+        default: 'origin/main',
+      })
+      .option('limit', {
+        describe: 'Max number of packages to bundle (for debugging)',
+        type: 'number',
+      })
+      .option('max-place-mb', {
+        describe:
+          'Size budget in MB for one combined place (--test only). Larger batches are split into several places. (default: 90, under the Open Cloud upload limit)',
+        type: 'number',
+      })
+      .check((args) => {
+        if (args.test === args.deploy || (!args.test && !args.deploy)) {
+          throw new Error('Pass exactly one of --test or --deploy.');
+        }
+        if (args.test && args.target !== undefined) {
+          throw new Error(
+            '--target picks a deploy target; it applies to --deploy only.'
+          );
+        }
+        if (args.deploy && args.maxPlaceMb !== undefined) {
+          throw new Error(
+            '--max-place-mb sizes combined test places; it applies to --test only.'
+          );
+        }
+        return true;
+      }) as any;
+  },
+  handler: async (args) => {
+    try {
+      if (args.deploy) {
+        await _writeDeployBundleAsync(args);
+      } else {
+        await _writeTestBundleAsync(args);
+      }
+    } catch (err) {
+      OutputHelper.error(err instanceof Error ? err.message : String(err));
+      process.exit(1);
+    }
+  },
+};
+
+async function _writeTestBundleAsync(args: BatchBundleArgs): Promise<void> {
+  let packages = args.all
+    ? await discoverAllTestableBatchTargetsAsync()
+    : await discoverChangedTestableBatchTargetsAsync(args.base!);
+  if (args.limit && args.limit > 0) {
+    packages = packages.slice(0, args.limit);
+  }
+
+  if (args.dryrun) {
+    OutputHelper.info(
+      `[DRYRUN] Would bundle ${packages.length} packages: ${packages
+        .map((p) => p.name)
+        .join(', ')}`
+    );
+    return;
+  }
+
+  if (packages.length === 0) {
+    OutputHelper.warn('No testable packages found; writing an empty bundle.');
+    await writePrebuiltBatchBundleAsync(args.file, [], new Map());
+    return;
+  }
+
+  // No client: building never uploads, so there is no key to need.
+  const batchContext = new BatchScriptJobContext(
+    new LocalJobContext(new BaseReporter()),
+    packages,
+    {
+      maxChunkBytes:
+        args.maxPlaceMb !== undefined
+          ? args.maxPlaceMb * 1024 * 1024
+          : undefined,
+    }
+  );
+
+  try {
+    // Prepared one at a time so every package's deploy-metadata stamp lands in
+    // the shared place before it is packed, and the last package's stamp wins.
+    // A live aggregated run stamps only after the shared place has uploaded,
+    // so its specs never see a stamp at all.
+    const hasJestConfig = new Map<string, boolean>();
+    for (const pkg of packages) {
+      const prepared = await prepareTestAsync(batchContext, {
+        packagePath: pkg.path,
+        packageName: pkg.name,
+        target: pkg.target,
+      });
+      hasJestConfig.set(pkg.name, prepared.hasJestConfig);
+    }
+
+    const chunks = await batchContext.getBuiltChunksAsync();
+    await writePrebuiltBatchBundleAsync(args.file, chunks, hasJestConfig);
+    OutputHelper.info(
+      `Wrote ${packages.length} package${packages.length === 1 ? '' : 's'} in ${
+        chunks.length
+      } place${chunks.length === 1 ? '' : 's'} to ${args.file}.`
+    );
+  } finally {
+    await batchContext.disposeAsync();
+  }
+}
+
+async function _writeDeployBundleAsync(args: BatchBundleArgs): Promise<void> {
+  const targetName = args.target ?? 'test';
+  const batchTargets = await discoverDeployBatchTargetsAsync({
+    targetName,
+    all: args.all,
+    base: args.base!,
+    limit: args.limit,
+  });
+
+  if (args.dryrun) {
+    OutputHelper.info(
+      `[DRYRUN] Would bundle ${batchTargets.length} places: ${batchTargets
+        .map((p) => p.name)
+        .join(', ')}`
+    );
+    return;
+  }
+
+  if (batchTargets.length === 0) {
+    OutputHelper.warn(
+      `No changed packages have a "${targetName}" target; writing an empty bundle.`
+    );
+  }
+
+  const context = new LocalJobContext(new BaseReporter());
+  try {
+    const places: PrebuiltDeployPlace[] = [];
+    for (const buildTarget of batchTargets) {
+      const built = await context.buildPlaceAsync({
+        target: buildTarget.target,
+        packagePath: buildTarget.path,
+        packageName: buildTarget.name,
+        outputFileName: 'deploy.rbxl',
+        deferBasePlaceMerge: true,
+      });
+      places.push({
+        name: buildTarget.name,
+        rbxlPath: built.rbxlPath,
+        projectPath: path.resolve(buildTarget.path, buildTarget.target.project),
+        packageVersion: await readPackageVersionAsync(buildTarget.path),
+      });
+    }
+
+    await writePrebuiltDeployBundleAsync(
+      args.file,
+      targetName,
+      gatherGitDeployInfo(),
+      places
+    );
+    OutputHelper.info(
+      `Wrote ${places.length} place${places.length === 1 ? '' : 's'} to ${
+        args.file
+      }.`
+    );
+  } finally {
+    await context.disposeAsync();
+  }
+}

package/src/commands/batch-command/batch-deploy-command.ts

@@ -24,7 +24,13 @@ import {
   buildDeployMetadataAttributes,
   gatherGitDeployInfo,
   injectDeployMetadataAsync,
+  type GitDeployInfo,
 } from '../../utils/deploy/deploy-metadata.js';
+import {
+  type PrebuiltDeployBundle,
+  type PrebuiltDeployPlace,
+  readPrebuiltDeployBundleAsync,
+} from '../../utils/deploy/prebuilt-deploy-bundle.js';
 import { OpenCloudClient } from '../../utils/open-cloud/open-cloud-client.js';
 import {
   RateLimiter,
@@ -38,7 +44,7 @@ import {
 import {
   type BatchTarget,
   discoverAllTargetPackagesAsync,
-  discoverChangedTargetPackagesAsync,
+  discoverDeployBatchTargetsAsync,
   flattenToBatchTargets,
 } from '../../utils/batch/changed-packages-utils.js';
 import {
@@ -77,6 +83,8 @@ interface BatchDeployArgs extends NevermoreGlobalArgs {
   watch?: string;
   watchShareApiKey?: boolean;
   watchUseGhAuth?: boolean;
+  bundle?: string;
+  bundleCommit?: string;
 }
 
 export const batchDeployCommand: CommandModule<
@@ -157,6 +165,36 @@ export const batchDeployCommand: CommandModule<
           'Off by default — the key is stored by the service.',
         type: 'boolean',
         default: false,
+      })
+      .option('bundle', {
+        describe:
+          'Deploy the places in a *.nevermore.deploy.tar written by "batch bundle --deploy" instead of ' +
+          'finding and building packages. The bundle only names places: where each one deploys, ' +
+          "and the base place it merges onto, come from this checkout's deploy.nevermore.json.",
+        type: 'string',
+      })
+      .option('bundle-commit', {
+        describe:
+          'Fail unless the --bundle was built from this full commit sha, e.g. the PR head GitHub reports',
+        type: 'string',
+      })
+      .check((args) => {
+        if (args.bundleCommit !== undefined && args.bundle === undefined) {
+          throw new Error('--bundle-commit checks a --bundle; pass one.');
+        }
+        if (args.bundle !== undefined) {
+          if (args.all || args.limit !== undefined) {
+            throw new Error(
+              '--bundle deploys every place in the bundle; drop --all and --limit.'
+            );
+          }
+          if (args.watch !== undefined) {
+            throw new Error(
+              "--bundle deploys another checkout's build, which must not become a watch baseline; drop --watch."
+            );
+          }
+        }
+        return true;
       });
   },
   handler: async (args) => {
@@ -170,18 +208,58 @@ export const batchDeployCommand: CommandModule<
 };
 
 async function _runAsync(args: BatchDeployArgs): Promise<void> {
+  const bundle =
+    args.bundle !== undefined
+      ? await readPrebuiltDeployBundleAsync(args.bundle)
+      : undefined;
+  let exitCode: number;
+  try {
+    exitCode = await _runDeployAsync(args, bundle);
+  } finally {
+    await bundle?.buildContext.cleanupAsync();
+  }
+  process.exit(exitCode);
+}
+
+async function _runDeployAsync(
+  args: BatchDeployArgs,
+  bundle: PrebuiltDeployBundle | undefined
+): Promise<number> {
   const targetName = args.target ?? 'test';
   // Parsed up front so a typo'd duration fails before anything is deployed.
   const watchOption =
     args.watch == null ? undefined : parseWatchOption(args.watch);
 
-  const discovered = args.all
-    ? await discoverAllTargetPackagesAsync(targetName)
-    : await discoverChangedTargetPackagesAsync(args.base!, targetName);
-  let batchTargets = flattenToBatchTargets(discovered);
+  let batchTargets: BatchTarget[];
+  let prebuiltByName: Map<string, PrebuiltDeployPlace> | undefined;
+  if (
+    bundle &&
+    args.bundleCommit !== undefined &&
+    bundle.git.version !== args.bundleCommit
+  ) {
+    throw new Error(
+      `The bundle was built from ${bundle.git.version}, not ${args.bundleCommit}.`
+    );
+  }
+  if (bundle) {
+    ({ batchTargets, prebuiltByName } = await _resolveBundleTargetsAsync(
+      bundle,
+      targetName
+    ));
+  } else {
+    batchTargets = await discoverDeployBatchTargetsAsync({
+      targetName,
+      all: args.all,
+      base: args.base!,
+      limit: args.limit,
+    });
+  }
 
-  if (args.limit && args.limit > 0) {
-    batchTargets = batchTargets.slice(0, args.limit);
+  if (batchTargets.length === 0 && bundle) {
+    OutputHelper.warn(
+      `The bundle holds no places; nothing changed in a "${targetName}" target.`
+    );
+    return 0;
   }
 
   if (batchTargets.length === 0) {
@@ -217,7 +295,7 @@ async function _runAsync(args: BatchDeployArgs): Promise<void> {
         dryrun: args.dryrun,
       });
     }
-    return;
+    return 0;
   }
 
   if (args.dryrun) {
@@ -247,7 +325,7 @@ async function _runAsync(args: BatchDeployArgs): Promise<void> {
         dryrun: true,
       });
     }
-    return;
+    return 0;
   }
 
   const concurrency = args.concurrency ?? 3;
@@ -306,7 +384,8 @@ async function _runAsync(args: BatchDeployArgs): Promise<void> {
   const context = new CloudJobContext(reporter, client, basePlaceResolver);
 
   // Gathered once so every deployed package shares one timestamp and commit.
-  const deployGitInfo = gatherGitDeployInfo();
+  // A bundle's places carry the commit that built them, not this checkout's.
+  const deployGitInfo: GitDeployInfo = bundle?.git ?? gatherGitDeployInfo();
   const deployTimestamp = new Date().toISOString();
 
   await reporter.startAsync();
@@ -319,11 +398,18 @@ async function _runAsync(args: BatchDeployArgs): Promise<void> {
       reporter,
       bufferOutput: isGrouped,
       executeAsync: async (buildTarget, pkgReporter) => {
+        const prebuilt = prebuiltByName?.get(buildTarget.name);
         const builtPlace = await context.buildPlaceAsync({
           target: buildTarget.target,
           outputFileName: publish ? 'publish.rbxl' : 'deploy.rbxl',
           packagePath: buildTarget.path,
           packageName: buildTarget.name,
+          overrides: prebuilt
+            ? {
+                placeFile: prebuilt.rbxlPath,
+                projectFile: prebuilt.projectPath,
+              }
+            : undefined,
         });
 
         const injected = await injectDeployMetadataAsync(
@@ -336,7 +422,9 @@ async function _runAsync(args: BatchDeployArgs): Promise<void> {
               timestamp: deployTimestamp,
               universeId: buildTarget.target.universeId,
               placeId: buildTarget.target.placeId,
-              packageVersion: await readPackageVersionAsync(buildTarget.path),
+              packageVersion: prebuilt
+                ? prebuilt.packageVersion
+                : await readPackageVersionAsync(buildTarget.path),
               basePlaceId: buildTarget.target.basePlace?.placeId,
               basePlaceVersion: builtPlace.basePlaceVersion,
             },
@@ -360,6 +448,18 @@ async function _runAsync(args: BatchDeployArgs): Promise<void> {
         // Eagerly release build artifacts after upload
         await context.releaseBuiltPlaceAsync(builtPlace);
 
+        const deployed = {
+          packageName: buildTarget.name,
+          placeId: buildTarget.target.placeId,
+          universeId: buildTarget.target.universeId,
+          target: targetName,
+          version,
+          published: publish,
+          basePlaceVersion: builtPlace.basePlaceVersion,
+          commit: deployGitInfo.version,
+          branch: deployGitInfo.branch,
+        };
+
         let logs: string;
         if (args.smokeTest && buildTarget.target.basePlace) {
           OutputHelper.verbose('Running post-deploy smoke test...');
@@ -374,8 +474,7 @@ async function _runAsync(args: BatchDeployArgs): Promise<void> {
           logs = smokeResult.logs;
           if (!smokeResult.success) {
             return {
-              packageName: buildTarget.name,
-              placeId: buildTarget.target.placeId,
+              ...deployed,
               success: false,
               logs: _annotateSmokeTestFailure(logs),
               failureLabel: 'SMOKE TEST FAILED',
@@ -387,8 +486,7 @@ async function _runAsync(args: BatchDeployArgs): Promise<void> {
         }
 
         return {
-          packageName: buildTarget.name,
-          placeId: buildTarget.target.placeId,
+          ...deployed,
           success: true,
           logs,
           progressSummary: {
@@ -425,7 +523,53 @@ async function _runAsync(args: BatchDeployArgs): Promise<void> {
   }
 
   await reporter.stopAsync();
-  process.exit(exitCode);
+  return exitCode;
+}
+
+/**
+ * Match each bundled place to this checkout's target of the same name.
+ *
+ * The bundle is untrusted, so it only gets to say which places it built. Place
+ * ids, universe ids and the base place all come from this checkout, which is
+ * what a same-repo PR's run would have used. A place this checkout does not
+ * know — a fork adding a new game — has nowhere trusted to go, so it fails
+ * rather than deploying to ids the fork chose.
+ */
+async function _resolveBundleTargetsAsync(
+  bundle: PrebuiltDeployBundle,
+  targetName: string
+): Promise<{
+  batchTargets: BatchTarget[];
+  prebuiltByName: Map<string, PrebuiltDeployPlace>;
+}> {
+  if (bundle.target !== targetName) {
+    throw new Error(
+      `The bundle was built for the "${bundle.target}" target, not "${targetName}". ` +
+        `Pass --target ${bundle.target}.`
+    );
+  }
+
+  const known = new Map(
+    flattenToBatchTargets(await discoverAllTargetPackagesAsync(targetName)).map(
+      (t) => [t.name, t]
+    )
+  );
+  const unknown = bundle.places
+    .map((p) => p.name)
+    .filter((name) => !known.has(name));
+  if (unknown.length > 0) {
+    throw new Error(
+      `The bundle holds places this checkout has no "${targetName}" target for: ` +
+        `${unknown.join(
+          ', '
+        )}. A new place deploys once its deploy.nevermore.json is on this branch.`
+    );
+  }
+
+  return {
+    batchTargets: bundle.places.map((p) => known.get(p.name)!),
+    prebuiltByName: new Map(bundle.places.map((p) => [p.name, p])),
+  };
 }
 
 /**