@atlaspack/core 2.16.2-canary.27 → 2.16.2-canary.270

package/src/requests/{ValidationRequest.js → ValidationRequest.ts}

@@ -1,4 +1,3 @@
-// @flow strict-local
 import type {Async} from '@atlaspack/types';
 import type {SharedReference} from '@atlaspack/workers';
 import type {StaticRunOpts} from '../RequestTracker';
@@ -12,22 +11,21 @@ import Validation from '../Validation';
 import createAtlaspackConfigRequest from './AtlaspackConfigRequest';
 import {requestTypes} from '../RequestTracker';
 
-type ValidationRequest = {|
-  id: string,
-  +type: typeof requestTypes.validation_request,
-  run: (RunOpts<void>) => Async<void>,
-  input: ValidationRequestInput,
-|};
+type ValidationRequest = {
+  id: string;
+  readonly type: typeof requestTypes.validation_request;
+  run: (arg1: RunOpts<undefined>) => Async<void>;
+  input: ValidationRequestInput;
+};
 
-type RunOpts<TResult> = {|
-  input: ValidationRequestInput,
-  ...StaticRunOpts<TResult>,
-|};
+type RunOpts<TResult> = {
+  input: ValidationRequestInput;
+} & StaticRunOpts<TResult>;
 
-type ValidationRequestInput = {|
-  assetRequests: Array<AssetGroup>,
-  optionsRef: SharedReference,
-|};
+type ValidationRequestInput = {
+  assetRequests: Array<AssetGroup>;
+  optionsRef: SharedReference;
+};
 
 export default function createValidationRequest(
   input: ValidationRequestInput,
@@ -44,17 +42,18 @@ export default function createValidationRequest(
 
       let config = new AtlaspackConfig(processedConfig, options);
       let trackedRequestsDesc = assetRequests.filter((request) => {
+        // @ts-expect-error TS2345
         return config.getValidatorNames(request.filePath).length > 0;
       });
 
       // Schedule validations on workers for all plugins that implement the one-asset-at-a-time "validate" method.
       let promises = trackedRequestsDesc.map(
         async (request) =>
-          ((await farm.createHandle('runValidate'))({
+          (await farm.createHandle('runValidate'))({
             requests: [request],
             optionsRef: optionsRef,
             configCachePath: cachePath,
-          }): void),
+          }) as undefined,
       );
 
       // Skip sending validation requests if no validators were configured
@@ -64,10 +63,12 @@ export default function createValidationRequest(
 
       // Schedule validations on the main thread for all validation plugins that implement "validateAll".
       promises.push(
+        // @ts-expect-error TS2345
         new Validation({
           requests: trackedRequestsDesc,
           options,
           config,
+          // @ts-expect-error TS2322
           report,
           dedicatedThread: true,
         }).run(),

package/src/requests/{WriteBundleRequest.js → WriteBundleRequest.ts}

@@ -1,5 +1,3 @@
-// @flow strict-local
-
 import type {FileSystem, FileOptions} from '@atlaspack/fs';
 import type {ContentKey} from '@atlaspack/graph';
 import type {Async, FilePath, Compressor} from '@atlaspack/types';
@@ -40,30 +38,31 @@ import {AtlaspackConfig} from '../AtlaspackConfig';
 import ThrowableDiagnostic, {errorToDiagnostic} from '@atlaspack/diagnostic';
 import {PluginTracer, tracer} from '@atlaspack/profiler';
 import {requestTypes} from '../RequestTracker';
+import {getFeatureFlag} from '@atlaspack/feature-flags';
+import {fromEnvironmentId} from '../EnvironmentManager';
 
 const HASH_REF_PREFIX_LEN = HASH_REF_PREFIX.length;
 const BOUNDARY_LENGTH = HASH_REF_PREFIX.length + 32 - 1;
 
-type WriteBundleRequestInput = {|
-  bundleGraph: BundleGraph,
-  bundle: Bundle,
-  info: BundleInfo,
-  hashRefToNameHash: Map<string, string>,
-|};
+type WriteBundleRequestInput = {
+  bundleGraph: BundleGraph;
+  bundle: Bundle;
+  info: BundleInfo;
+  hashRefToNameHash: Map<string, string>;
+};
 
 export type WriteBundleRequestResult = PackagedBundleInfo;
 
-type RunInput<TResult> = {|
-  input: WriteBundleRequestInput,
-  ...StaticRunOpts<TResult>,
-|};
+type RunInput<TResult> = {
+  input: WriteBundleRequestInput;
+} & StaticRunOpts<TResult>;
 
-export type WriteBundleRequest = {|
-  id: ContentKey,
-  +type: typeof requestTypes.write_bundle_request,
-  run: (RunInput<PackagedBundleInfo>) => Async<PackagedBundleInfo>,
-  input: WriteBundleRequestInput,
-|};
+export type WriteBundleRequest = {
+  id: ContentKey;
+  readonly type: typeof requestTypes.write_bundle_request;
+  run: (arg1: RunInput<PackagedBundleInfo>) => Async<PackagedBundleInfo>;
+  input: WriteBundleRequestInput;
+};
 
 /**
  * Writes a bundle to the dist directory, replacing hash references with the final content hashes.
@@ -83,6 +82,7 @@ export default function createWriteBundleRequest(
   };
 }
 
+// @ts-expect-error TS7031
 async function run({input, options, api}) {
   let {bundleGraph, bundle, info, hashRefToNameHash} = input;
   let {inputFS, outputFS} = options;
@@ -110,7 +110,9 @@ async function run({input, options, api}) {
   let cacheKeys = info.cacheKeys;
   let mapKey = cacheKeys.map;
   let fullPath = fromProjectPath(options.projectRoot, filePath);
-  if (mapKey && bundle.env.sourceMap && !bundle.env.sourceMap.inline) {
+  const env = fromEnvironmentId(bundle.env);
+
+  if (mapKey && env.sourceMap && !env.sourceMap.inline) {
     api.invalidateOnFileDelete(
       toProjectPath(options.projectRoot, fullPath + '.map'),
     );
@@ -139,12 +141,14 @@ async function run({input, options, api}) {
   }
   let size = 0;
   contentStream = contentStream.pipe(
-    new TapStream((buf) => {
+    // @ts-expect-error TS2554
+    new TapStream((buf: Buffer) => {
       size += buf.length;
     }),
   );
 
   let configResult = nullthrows(
+    // @ts-expect-error TS2347
     await api.runRequest<null, ConfigAndCachePath>(
       createAtlaspackConfigRequest(),
     ),
@@ -167,14 +171,15 @@ async function run({input, options, api}) {
     api,
   );
 
-  if (
-    mapKey &&
-    bundle.env.sourceMap &&
-    !bundle.env.sourceMap.inline &&
-    (await options.cache.has(mapKey))
-  ) {
+  const hasSourceMap = getFeatureFlag('cachePerformanceImprovements')
+    ? await options.cache.hasLargeBlob(mapKey)
+    : await options.cache.has(mapKey);
+  if (mapKey && env.sourceMap && !env.sourceMap.inline && hasSourceMap) {
+    const mapEntry = getFeatureFlag('cachePerformanceImprovements')
+      ? await options.cache.getLargeBlob(mapKey)
+      : await options.cache.getBlob(mapKey);
     await writeFiles(
-      blobToStream(await options.cache.getBlob(mapKey)),
+      blobToStream(mapEntry),
       info,
       hashRefToNameHash,
       options,
@@ -189,6 +194,7 @@ async function run({input, options, api}) {
 
   let res = {
     filePath,
+    bundleId: bundle.id,
     type: info.type,
     stats: {
       size,
@@ -201,14 +207,15 @@ async function run({input, options, api}) {
 }
 
 async function writeFiles(
-  inputStream: stream$Readable,
+  // @ts-expect-error TS2503
+  inputStream: stream.Readable,
   info: BundleInfo,
   hashRefToNameHash: Map<string, string>,
   options: AtlaspackOptions,
   config: AtlaspackConfig,
   outputFS: FileSystem,
   filePath: ProjectPath,
-  writeOptions: ?FileOptions,
+  writeOptions: FileOptions | null | undefined,
   devDeps: Map<string, string>,
   api: RunAPI<PackagedBundleInfo>,
 ) {
@@ -221,9 +228,10 @@ async function writeFiles(
     ? inputStream.pipe(replaceStream(hashRefToNameHash))
     : inputStream;
 
-  let promises = [];
+  let promises: Array<Promise<undefined>> = [];
   for (let compressor of compressors) {
     promises.push(
+      // @ts-expect-error TS2345
       runCompressor(
         compressor,
         cloneStream(stream),
@@ -242,11 +250,12 @@ async function writeFiles(
 
 async function runCompressor(
   compressor: LoadedPlugin<Compressor>,
-  stream: stream$Readable,
+  // @ts-expect-error TS2503
+  stream: stream.Readable,
   options: AtlaspackOptions,
   outputFS: FileSystem,
   filePath: FilePath,
-  writeOptions: ?FileOptions,
+  writeOptions: FileOptions | null | undefined,
   devDeps: Map<string, string>,
   api: RunAPI<PackagedBundleInfo>,
 ) {
@@ -265,21 +274,26 @@ async function runCompressor(
     });
 
     if (res != null) {
-      await new Promise((resolve, reject) =>
-        pipeline(
-          res.stream,
-          outputFS.createWriteStream(
-            filePath + (res.type != null ? '.' + res.type : ''),
-            writeOptions,
+      await new Promise(
+        (
+          resolve: (result: Promise<undefined> | undefined) => void,
+          reject: (error?: any) => void,
+        ) =>
+          pipeline(
+            res.stream,
+            outputFS.createWriteStream(
+              filePath + (res.type != null ? '.' + res.type : ''),
+              writeOptions,
+            ),
+            (err) => {
+              if (err) reject(err);
+              // @ts-expect-error TS2794
+              else resolve();
+            },
           ),
-          (err) => {
-            if (err) reject(err);
-            else resolve();
-          },
-        ),
       );
     }
-  } catch (err) {
+  } catch (err: any) {
     throw new ThrowableDiagnostic({
       diagnostic: errorToDiagnostic(err, {
         origin: compressor.name,
@@ -300,11 +314,18 @@ async function runCompressor(
   }
 }
 
-function replaceStream(hashRefToNameHash) {
+function replaceStream(hashRefToNameHash: Map<string, string>) {
   let boundaryStr = Buffer.alloc(0);
   let replaced = Buffer.alloc(0);
   return new Transform({
-    transform(chunk, encoding, cb) {
+    transform(
+      chunk: Buffer | string,
+      encoding: string,
+      cb: (
+        error?: Error | null | undefined,
+        data?: Buffer | string | null | undefined,
+      ) => void,
+    ) {
       let str = Buffer.concat([boundaryStr, Buffer.from(chunk)]);
       let lastMatchI = 0;
       if (replaced.length < str.byteLength) {
@@ -345,18 +366,25 @@ function replaceStream(hashRefToNameHash) {
       cb(null, strUpToBoundary);
     },
 
-    flush(cb) {
+    flush(
+      cb: (
+        error?: Error | null | undefined,
+        data?: Buffer | string | null | undefined,
+      ) => void,
+    ) {
       cb(null, boundaryStr);
     },
   });
 }
 
-function cloneStream(readable) {
+// @ts-expect-error TS2503
+function cloneStream(readable: stream.Readable | stream.Transform) {
   let res = new Readable();
-  // $FlowFixMe
   res._read = () => {};
+  // @ts-expect-error TS7006
   readable.on('data', (chunk) => res.push(chunk));
   readable.on('end', () => res.push(null));
+  // @ts-expect-error TS7006
   readable.on('error', (err) => res.emit('error', err));
   return res;
 }

package/src/requests/{WriteBundlesRequest.js → WriteBundlesRequest.ts}

@@ -1,14 +1,18 @@
-// @flow strict-local
-
 import type {ContentKey} from '@atlaspack/graph';
 import type {Async} from '@atlaspack/types';
 import {getFeatureFlag} from '@atlaspack/feature-flags';
 import type {SharedReference} from '@atlaspack/workers';
 import type {StaticRunOpts} from '../RequestTracker';
 import {requestTypes} from '../RequestTracker';
-import type {PackagedBundleInfo} from '../types';
+import {
+  BundleBehavior,
+  type PackagedBundleInfo,
+  type Bundle,
+  type AtlaspackOptions,
+} from '../types';
 import type BundleGraph from '../BundleGraph';
 import type {BundleInfo} from '../PackagerRunner';
+import {report} from '../ReporterRunner';
 
 import {HASH_REF_PREFIX} from '../constants';
 import {joinProjectPath} from '../projectPath';
@@ -16,27 +20,49 @@ import nullthrows from 'nullthrows';
 import {hashString} from '@atlaspack/rust';
 import {createPackageRequest} from './PackageRequest';
 import createWriteBundleRequest from './WriteBundleRequest';
+import {debugTools} from '@atlaspack/utils';
 
-type WriteBundlesRequestInput = {|
-  bundleGraph: BundleGraph,
-  optionsRef: SharedReference,
-|};
+type WriteBundlesRequestInput = {
+  bundleGraph: BundleGraph;
+  optionsRef: SharedReference;
+};
 
-export type WriteBundlesRequestResult = Map<string, PackagedBundleInfo>;
+export type WriteBundlesRequestResult = {
+  bundleInfo: Map<string, PackagedBundleInfo>;
+  scopeHoistingStats?: {
+    totalAssets: number;
+    wrappedAssets: number;
+  };
+};
 
-type RunInput<TResult> = {|
-  input: WriteBundlesRequestInput,
-  ...StaticRunOpts<TResult>,
-|};
+type RunInput<TResult> = {
+  input: WriteBundlesRequestInput;
+} & StaticRunOpts<TResult>;
 
-export type WriteBundlesRequest = {|
-  id: ContentKey,
-  +type: typeof requestTypes.write_bundles_request,
+export type WriteBundlesRequest = {
+  id: ContentKey;
+  readonly type: typeof requestTypes.write_bundles_request;
   run: (
-    RunInput<WriteBundlesRequestResult>,
-  ) => Async<WriteBundlesRequestResult>,
-  input: WriteBundlesRequestInput,
-|};
+    arg1: RunInput<WriteBundlesRequestResult>,
+  ) => Async<WriteBundlesRequestResult>;
+  input: WriteBundlesRequestInput;
+};
+
+function reportPackagingProgress(
+  completeBundles: number,
+  totalBundles: number,
+) {
+  if (!getFeatureFlag('cliProgressReportingImprovements')) {
+    return;
+  }
+
+  report({
+    type: 'buildProgress',
+    phase: 'packagingAndOptimizing',
+    totalBundles,
+    completeBundles,
+  });
+}
 
 /**
  * Packages, optimizes, and writes all bundles to the dist directory.
@@ -52,17 +78,22 @@ export default function createWriteBundlesRequest(
   };
 }
 
-async function run({input, api, farm, options}) {
+async function run({
+  input,
+  api,
+  farm,
+  options,
+}: RunInput<WriteBundlesRequestResult>) {
   let {bundleGraph, optionsRef} = input;
   let {ref, dispose} = await farm.createSharedReference(bundleGraph);
 
   api.invalidateOnOptionChange('shouldContentHash');
 
   let res = new Map();
-  let bundleInfoMap: {|
-    [string]: BundleInfo,
-  |} = {};
-  let writeEarlyPromises = {};
+  let bundleInfoMap: {
+    [key: string]: BundleInfo;
+  } = {};
+  let writeEarlyPromises: Record<string, Promise<PackagedBundleInfo>> = {};
   let hashRefToNameHash = new Map();
 
   // Include inline bundles so that non-inline bundles referenced from inline bundles are written to
@@ -71,7 +102,11 @@ async function run({input, api, farm, options}) {
     includeInline: getFeatureFlag('inlineBundlesSourceMapFixes'),
   });
   const bundles = allBundles
-    .filter((bundle) => bundle.bundleBehavior !== 'inline')
+    .filter(
+      (bundle) =>
+        bundle.bundleBehavior !== BundleBehavior.inline &&
+        bundle.bundleBehavior !== BundleBehavior.inlineIsolated,
+    )
     .filter((bundle) => {
       // Do not package and write placeholder bundles to disk. We just
       // need to update the name so other bundles can reference it.
@@ -84,6 +119,7 @@ async function run({input, api, farm, options}) {
         ).replace(bundle.hashReference, hash);
         res.set(bundle.id, {
           filePath: joinProjectPath(bundle.target.distDir, name),
+          bundleId: bundle.id,
           type: bundle.type, // FIXME: this is wrong if the packager changes the type...
           stats: {
             time: 0,
@@ -96,14 +132,19 @@ async function run({input, api, farm, options}) {
       return true;
     });
 
+  let cachedBundles = new Set(
+    bundles.filter((b) => api.canSkipSubrequest(bundleGraph.getHash(b))),
+  );
+
   // Package on the main thread if there is only one bundle to package.
   // This avoids the cost of serializing the bundle graph for single file change builds.
   let useMainThread =
-    bundles.length === 1 ||
-    bundles.filter((b) => !api.canSkipSubrequest(bundleGraph.getHash(b)))
-      .length === 1;
+    bundles.length === 1 || bundles.length - cachedBundles.size <= 1;
 
   try {
+    let completeBundles = cachedBundles.size;
+    reportPackagingProgress(completeBundles, bundles.length);
+
     await Promise.all(
       bundles.map(async (bundle) => {
         let request = createPackageRequest({
@@ -116,6 +157,11 @@ async function run({input, api, farm, options}) {
 
         let info = await api.runRequest(request);
 
+        if (!cachedBundles.has(bundle)) {
+          completeBundles++;
+          reportPackagingProgress(completeBundles, bundles.length);
+        }
+
         if (!useMainThread) {
           // Force a refresh of the cache to avoid a race condition
           // between threaded reads and writes that can result in an LMDB cache miss:
@@ -167,18 +213,42 @@ async function run({input, api, farm, options}) {
       }),
     );
 
-    api.storeResult(res);
-    return res;
+    let result: WriteBundlesRequestResult = {bundleInfo: res};
+
+    if (debugTools['scope-hoisting-stats']) {
+      // Aggregate scope hoisting stats from all bundles
+      let aggregatedScopeHoistingStats = {
+        totalAssets: 0,
+        wrappedAssets: 0,
+      };
+
+      for (let bundle of bundles) {
+        let bundleInfo = bundleInfoMap[bundle.id];
+        if (bundleInfo?.scopeHoistingStats) {
+          aggregatedScopeHoistingStats.totalAssets +=
+            bundleInfo.scopeHoistingStats.totalAssets;
+          aggregatedScopeHoistingStats.wrappedAssets +=
+            bundleInfo.scopeHoistingStats.wrappedAssets;
+        }
+      }
+
+      result.scopeHoistingStats = aggregatedScopeHoistingStats;
+    }
+
+    api.storeResult(result);
+    return result;
   } finally {
     await dispose();
   }
 }
 
 function assignComplexNameHashes(
-  hashRefToNameHash,
-  bundles,
-  bundleInfoMap,
-  options,
+  hashRefToNameHash: Map<string, string>,
+  bundles: Array<Bundle>,
+  bundleInfoMap: {
+    [key: string]: BundleInfo;
+  },
+  options: AtlaspackOptions,
 ) {
   for (let bundle of bundles) {
     if (hashRefToNameHash.get(bundle.hashReference) != null) {
@@ -198,9 +268,11 @@ function assignComplexNameHashes(
 }
 
 function getBundlesIncludedInHash(
-  bundleId,
-  bundleInfoMap,
-  included = new Set(),
+  bundleId: ContentKey | string,
+  bundleInfoMap: {
+    [key: string]: BundleInfo;
+  },
+  included = new Set<string>(),
 ) {
   included.add(bundleId);
   for (let hashRef of bundleInfoMap[bundleId]?.hashReferences ?? []) {

package/src/requests/{asset-graph-diff.js → asset-graph-diff.ts}

@@ -1,15 +1,12 @@
 /* eslint-disable no-console */
-// @flow strict-local
-
-// $FlowFixMe
+// @ts-expect-error missing types
 import deepClone from 'rfdc/default';
-// $FlowFixMe
 import {diff} from 'jest-diff';
 import AssetGraph from '../AssetGraph';
 import type {AssetGraphNode} from '../types';
-import {fromProjectPathRelative, toProjectPath} from '../projectPath';
+import {toProjectPath} from '../projectPath';
 
-function filterNode(node) {
+function filterNode(node: any) {
   let clone = deepClone(node);
 
   // Clean up anything you don't want to see in the diff
@@ -35,24 +32,26 @@ function filterNode(node) {
   return clone;
 }
 
+// @ts-expect-error missing return type
 function compactDeep(
-  obj: mixed,
+  obj: unknown,
   ignoredPatterns: Array<string> = [],
   currentPath: string = '$',
 ) {
   if (obj instanceof Map) {
-    const copy = {};
-    Array.from(obj.entries()).forEach(([k, v]) => {
+    const copy: Record<string, any> = {};
+    Array.from(obj.entries()).forEach(([k, v]: [any, any]) => {
       if (v != null) {
         copy[k] = compactDeep(v, ignoredPatterns, `${currentPath}.${k}`);
       }
     });
     return copy;
   } else if (Array.isArray(obj)) {
+    // @ts-expect-error implicit any
     return obj.map((v) => compactDeep(v, ignoredPatterns, `${currentPath}[]`));
   } else if (typeof obj === 'object') {
-    const copy = {};
-    Object.entries(obj ?? {}).forEach(([key, value]) => {
+    const copy: Record<string, any> = {};
+    Object.entries(obj ?? {}).forEach(([key, value]: [any, any]) => {
       const path = `${currentPath}.${key}`;
       if (ignoredPatterns.some((pattern) => path.includes(pattern))) {
         return;
@@ -72,21 +71,26 @@ function compactDeep(
   }
 }
 
-function assetGraphDiff(jsAssetGraph: AssetGraph, rustAssetGraph: AssetGraph) {
-  const getNodes = (graph) => {
-    let nodes = {};
+function assetGraphDiff(
+  jsAssetGraph: AssetGraph,
+  rustAssetGraph: AssetGraph,
+  projectRoot: string,
+) {
+  const getNodes = (graph: any) => {
+    let nodes: Record<string, any> = {};
 
+    // @ts-expect-error implicit any
     graph.traverse((nodeId) => {
       let node: AssetGraphNode | null = graph.getNode(nodeId) ?? null;
       if (!node) return;
 
       if (node.type === 'dependency') {
-        let sourcePath = node.value.sourcePath ?? toProjectPath('', 'entry');
-        nodes[
-          `dep:${fromProjectPathRelative(sourcePath)}:${node.value.specifier}`
-        ] = filterNode(node);
+        let sourcePath = node.value.sourcePath
+          ? toProjectPath(projectRoot, node.value.sourcePath)
+          : toProjectPath(projectRoot, 'entry');
+        nodes[`dep:${sourcePath}:${node.value.specifier}`] = filterNode(node);
       } else if (node.type === 'asset') {
-        nodes[`asset:${fromProjectPathRelative(node.value.filePath)}`] =
+        nodes[`asset:${toProjectPath(projectRoot, node.value.filePath)}`] =
           filterNode(node);
       }
     });
@@ -98,8 +102,8 @@ function assetGraphDiff(jsAssetGraph: AssetGraph, rustAssetGraph: AssetGraph) {
   const rustNodes = getNodes(rustAssetGraph);
 
   const all = new Set([...Object.keys(jsNodes), ...Object.keys(rustNodes)]);
-  const missing = [];
-  const extra = [];
+  const missing: Array<any | string> = [];
+  const extra: Array<any | string> = [];
 
   for (const key of all.keys()) {
     if (process.env.NATIVE_COMPARE !== 'true') {