@atlaspack/core 2.12.1-dev.3401 → 2.12.1-dev.3443

package/src/public/BundleGraph.js

@@ -332,4 +332,110 @@ export default class BundleGraph<TBundle: IBundle>
       targetToInternalTarget(target),
     );
   }
+
+  // Given a set of dependencies, return any conditions where those dependencies are either
+  // the true or false dependency for those conditions. This is currently used to work out which
+  // conditions belong to a bundle in packaging.
+  getConditionsForDependencies(deps: Array<IDependency>): Set<{|
+    publicId: string,
+    key: string,
+    ifTrueDependency: IDependency,
+    ifFalseDependency: IDependency,
+    ifTrueAssetId: string,
+    ifFalseAssetId: string,
+  |}> {
+    const conditions = new Set();
+    const depIds = deps.map(dep => dep.id);
+    for (const condition of this.#graph._conditions.values()) {
+      if (
+        depIds.includes(condition.ifTrueDependency.id) ||
+        depIds.includes(condition.ifFalseDependency.id)
+      ) {
+        const [trueAsset, falseAsset] = [
+          condition.ifTrueDependency,
+          condition.ifFalseDependency,
+        ].map(dep => {
+          const resolved = nullthrows(this.#graph.resolveAsyncDependency(dep));
+          if (resolved.type === 'asset') {
+            return resolved.value;
+          } else {
+            return this.#graph.getAssetById(resolved.value.entryAssetId);
+          }
+        });
+
+        conditions.add({
+          publicId: condition.publicId,
+          key: condition.key,
+          ifTrueDependency: nullthrows(
+            deps.find(dep => dep.id === condition.ifTrueDependency.id),
+            'ifTrueDependency was null',
+          ),
+          ifFalseDependency: nullthrows(
+            deps.find(dep => dep.id === condition.ifFalseDependency.id),
+            'ifFalseDependency was null',
+          ),
+          ifTrueAssetId: this.#graph.getAssetPublicId(trueAsset),
+          ifFalseAssetId: this.#graph.getAssetPublicId(falseAsset),
+        });
+      }
+    }
+
+    return conditions;
+  }
+
+  // This is used to generate information for building a manifest that can
+  // be used by a webserver to understand which conditions are used by which bundles,
+  // and which bundles those conditions require depending on what they evaluate to.
+  getConditionalBundleMapping(): Map<
+    TBundle,
+    Map<
+      string,
+      {|
+        ifTrueBundles: Array<TBundle>,
+        ifFalseBundles: Array<TBundle>,
+      |},
+    >,
+  > {
+    let bundleConditions = new Map();
+
+    // Convert the internal references in conditions to public API references
+    for (const cond of this.#graph._conditions.values()) {
+      let assets = Array.from(cond.assets).map(asset =>
+        nullthrows(this.getAssetById(asset.id)),
+      );
+      let bundles = new Set<TBundle>();
+      let ifTrueBundles = [];
+      let ifFalseBundles = [];
+      for (const asset of assets) {
+        const bundlesWithAsset = this.getBundlesWithAsset(asset);
+        for (const bundle of bundlesWithAsset) {
+          bundles.add(bundle);
+        }
+        const assetDeps = this.getDependencies(asset);
+        const depToBundles = dep => {
+          const publicDep = nullthrows(
+            assetDeps.find(assetDep => dep.id === assetDep.id),
+          );
+          const resolved = nullthrows(this.resolveAsyncDependency(publicDep));
+          invariant(resolved.type === 'bundle_group');
+          return this.getBundlesInBundleGroup(resolved.value);
+        };
+        ifTrueBundles.push(...depToBundles(cond.ifTrueDependency));
+        ifFalseBundles.push(...depToBundles(cond.ifFalseDependency));
+      }
+
+      for (let bundle of bundles) {
+        const conditions = bundleConditions.get(bundle) ?? new Map();
+
+        conditions.set(cond.key, {
+          ifTrueBundles,
+          ifFalseBundles,
+        });
+
+        bundleConditions.set(bundle, conditions);
+      }
+    }
+
+    return bundleConditions;
+  }
 }

package/src/requests/AssetGraphRequestRust.js

@@ -10,6 +10,7 @@ import type {AtlaspackV3} from '../atlaspack-v3';
 import {toProjectPath} from '../projectPath';
 import {requestTypes, type StaticRunOpts} from '../RequestTracker';
 import {propagateSymbols} from '../SymbolPropagation';
+import type {Environment} from '../types';
 
 import type {
   AssetGraphRequestInput,
@@ -95,9 +96,58 @@ function getAssetGraph(serializedGraph, options) {
 
   graph.safeToIncrementallyBundle = false;
 
+  function mapSymbols({exported, ...symbol}) {
+    let jsSymbol = {
+      local: symbol.local ?? undefined,
+      loc: symbol.loc ?? undefined,
+      meta: undefined,
+      isWeak: symbol.isWeak,
+    };
+
+    if (symbol.exported) {
+      // $FlowFixMe
+      jsSymbol.exported = symbol.exported;
+    }
+
+    if (symbol.isEsmExport) {
+      // $FlowFixMe
+      jsSymbol.meta = {
+        isEsm: true,
+      };
+    }
+    return [exported, jsSymbol];
+  }
+
+  // See crates/atlaspack_core/src/types/environment.rs
   let cachedAssets = new Map();
   let changedAssets = new Map();
   let entry = 0;
+
+  let envs = new Map();
+  let getEnvId = (env: Environment) => {
+    let envKey = [
+      env.context,
+      env.engines.atlaspack,
+      env.engines.browsers,
+      env.engines.electron,
+      env.engines.node,
+      env.includeNodeModules,
+      env.isLibrary,
+      env.outputFormat,
+      env.shouldScopeHoist,
+      env.shouldOptimize,
+      env.sourceType,
+    ].join(':');
+
+    let envId = envs.get(envKey);
+    if (envId == null) {
+      envId = envs.size;
+      envs.set(envKey, envId);
+    }
+
+    return envId;
+  };
+
   for (let node of serializedGraph.nodes) {
     if (node.type === 'root') {
       let index = graph.addNodeByContentKey('@@root', {
@@ -116,20 +166,38 @@ function getAssetGraph(serializedGraph, options) {
         value: null,
       });
     } else if (node.type === 'asset') {
-      let id = node.value.id;
-      let asset = node.value.asset;
+      let asset = node.value;
+      let id = asset.id;
+
+      asset.meta.id = id;
 
       asset = {
         ...asset,
-        id,
+        uniqueKey: asset.uniqueKey ?? undefined,
+        pipeline: asset.pipeline ?? undefined,
+        range: asset.range ?? undefined,
+        resolveFrom: asset.resolveFrom ?? undefined,
+        target: asset.target ?? undefined,
+        plugin: asset.plugin ?? undefined,
+        query: asset.query ?? undefined,
+        configPath: asset.configPath ?? undefined,
+        configKeyPath: asset.configKeyPath ?? undefined,
+        isLargeBlob: asset.isLargeBlob ?? false,
+        isSource: asset.isSource ?? false,
+        sourcePath: asset.sourcePath ?? undefined,
+        env: {
+          ...asset.env,
+          loc: asset.env.loc ?? undefined,
+          id: getEnvId(asset.env),
+          sourceType: asset.env.sourceType,
+        },
+        bundleBehavior:
+          asset.bundleBehavior === 255 ? null : asset.bundleBehavior,
         committed: true,
         contentKey: id,
         filePath: toProjectPath(options.projectRoot, asset.filePath),
-        symbols: asset.hasSymbols
-          ? new Map(
-              asset.symbols.map(({exported, ...symbol}) => [exported, symbol]),
-            )
-          : null,
+        symbols:
+          asset.symbols != null ? new Map(asset.symbols.map(mapSymbols)) : null,
       };
 
       cachedAssets.set(id, asset.code);
@@ -150,18 +218,38 @@ function getAssetGraph(serializedGraph, options) {
       dependency = {
         ...dependency,
         id,
+        env: {
+          ...dependency.env,
+          id: getEnvId(dependency.env),
+          sourceType: dependency.env.sourceType,
+          loc: dependency.env.loc ?? undefined,
+        },
+        pipeline: dependency.pipeline ?? undefined,
+        range: dependency.range ?? undefined,
+        resolveFrom: dependency.resolveFrom ?? undefined,
+        target: dependency.target ?? undefined,
+        bundleBehavior:
+          dependency.bundleBehavior === 255 ? null : dependency.bundleBehavior,
         contentKey: id,
+        loc: dependency.loc
+          ? {
+              ...dependency.loc,
+              filePath: toProjectPath(
+                options.projectRoot,
+                dependency.loc.filePath,
+              ),
+            }
+          : undefined,
         sourcePath: dependency.sourcePath
           ? toProjectPath(options.projectRoot, dependency.sourcePath)
-          : null,
-        symbols: dependency.hasSymbols
-          ? new Map(
-              dependency.symbols.map(({exported, ...symbol}) => [
-                exported,
-                symbol,
-              ]),
-            )
-          : null,
+          : undefined,
+        symbols:
+          // Dependency.symbols are always set to an empty map when scope hoisting
+          // is enabled. Some tests will fail if this is not the case. We should
+          // make this consistant when we re-visit packaging.
+          dependency.symbols != null || dependency.env.shouldScopeHoist
+            ? new Map(dependency.symbols?.map(mapSymbols))
+            : undefined,
       };
       let usedSymbolsDown = new Set();
       let usedSymbolsUp = new Map();

package/src/requests/BundleGraphRequest.js

@@ -118,6 +118,24 @@ export default function createBundleGraphRequest(
         },
       );
 
+      // if (input.rustAtlaspack && process.env.NATIVE_COMPARE === 'true') {
+      //   let {assetGraph: jsAssetGraph} = await api.runRequest(
+      //     createAssetGraphRequestJS({
+      //       name: 'Main',
+      //       entries: options.entries,
+      //       optionsRef,
+      //       shouldBuildLazily: options.shouldBuildLazily,
+      //       lazyIncludes: options.lazyIncludes,
+      //       lazyExcludes: options.lazyExcludes,
+      //       requestedAssetIds,
+      //     }),
+      //     {
+      //       force: true,
+      //     },
+      //   );
+      //   require('./asset-graph-diff.js')(jsAssetGraph, assetGraph);
+      // }
+
       measurement && measurement.end();
       assertSignalNotAborted(signal);
 

package/src/requests/WriteBundleRequest.js

@@ -3,6 +3,7 @@
 import type {FileSystem, FileOptions} from '@atlaspack/fs';
 import type {ContentKey} from '@atlaspack/graph';
 import type {Async, FilePath, Compressor} from '@atlaspack/types';
+import {replaceHashReferences} from '@atlaspack/rust';
 
 import type {RunAPI, StaticRunOpts} from '../RequestTracker';
 import type {Bundle, PackagedBundleInfo, AtlaspackOptions} from '../types';
@@ -16,7 +17,7 @@ import {HASH_REF_HASH_LEN, HASH_REF_PREFIX} from '../constants';
 import nullthrows from 'nullthrows';
 import path from 'path';
 import {NamedBundle} from '../public/Bundle';
-import {blobToStream, TapStream} from '@atlaspack/utils';
+import {blobToStream} from '@atlaspack/utils';
 import {Readable, Transform, pipeline} from 'stream';
 import {
   fromProjectPath,
@@ -129,20 +130,8 @@ async function run({input, options, api}) {
       : {
           mode: (await inputFS.stat(mainEntry.filePath)).mode,
         };
-  let contentStream: Readable;
-  if (info.isLargeBlob) {
-    contentStream = options.cache.getStream(cacheKeys.content);
-  } else {
-    contentStream = blobToStream(
-      await options.cache.getBlob(cacheKeys.content),
-    );
-  }
-  let size = 0;
-  contentStream = contentStream.pipe(
-    new TapStream(buf => {
-      size += buf.length;
-    }),
-  );
+  const contents: Buffer = await options.cache.getBlob(cacheKeys.content);
+  const size = contents.byteLength;
 
   let configResult = nullthrows(
     await api.runRequest<null, ConfigAndCachePath>(
@@ -155,7 +144,7 @@ async function run({input, options, api}) {
   invalidateDevDeps(invalidDevDeps, options, config);
 
   await writeFiles(
-    contentStream,
+    contents,
     info,
     hashRefToNameHash,
     options,
@@ -174,7 +163,7 @@ async function run({input, options, api}) {
     (await options.cache.has(mapKey))
   ) {
     await writeFiles(
-      blobToStream(await options.cache.getBlob(mapKey)),
+      await options.cache.getBlob(mapKey),
       info,
       hashRefToNameHash,
       options,
@@ -201,7 +190,7 @@ async function run({input, options, api}) {
 }
 
 async function writeFiles(
-  inputStream: stream$Readable,
+  contents: Buffer,
   info: BundleInfo,
   hashRefToNameHash: Map<string, string>,
   options: AtlaspackOptions,
@@ -217,16 +206,19 @@ async function writeFiles(
   );
   let fullPath = fromProjectPath(options.projectRoot, filePath);
 
-  let stream = info.hashReferences.length
-    ? inputStream.pipe(replaceStream(hashRefToNameHash))
-    : inputStream;
+  const stream = info.hashReferences.length
+    ? replaceHashReferences(
+        contents,
+        Object.fromEntries(hashRefToNameHash.entries()),
+      )
+    : contents;
 
   let promises = [];
   for (let compressor of compressors) {
     promises.push(
       runCompressor(
         compressor,
-        cloneStream(stream),
+        blobToStream(stream),
         options,
         outputFS,
         fullPath,
@@ -300,7 +292,9 @@ async function runCompressor(
   }
 }
 
-function replaceStream(hashRefToNameHash) {
+export function replaceStream(
+  hashRefToNameHash: Map<string, string>,
+): Transform {
   let boundaryStr = Buffer.alloc(0);
   let replaced = Buffer.alloc(0);
   return new Transform({

package/src/requests/asset-graph-diff.js

@@ -0,0 +1,145 @@
+/* eslint-disable no-console */
+// @flow strict-local
+
+// $FlowFixMe
+import deepClone from 'rfdc/default';
+// $FlowFixMe
+import {diff} from 'jest-diff';
+import AssetGraph from '../AssetGraph';
+import type {AssetGraphNode} from '../types';
+import {fromProjectPathRelative, toProjectPath} from '../projectPath';
+
+function filterNode(node) {
+  let clone = deepClone(node);
+
+  // Clean up anything you don't want to see in the diff
+  // delete clone.id;
+  delete clone.value.id;
+  delete clone.value.meta.id;
+  delete clone.value.sourceAssetId;
+  delete clone.value.env.id;
+  delete clone.value.isEsm;
+  delete clone.value.shouldWrap;
+  delete clone.value.contentKey;
+  delete clone.value.placeholder;
+  delete clone.value.code;
+  delete clone.value.hasCjsExports;
+  delete clone.value.staticExports;
+  delete clone.value.isConstantModule;
+  delete clone.value.hasNodeReplacements;
+  delete clone.value.stats;
+  delete clone.value.astKey;
+  delete clone.value.astGenerator;
+  delete clone.value.dependencies;
+
+  return clone;
+}
+
+function compactDeep(
+  obj: mixed,
+  ignoredPatterns: Array<string> = [],
+  currentPath: string = '$',
+) {
+  if (obj instanceof Map) {
+    const copy = {};
+    Array.from(obj.entries()).forEach(([k, v]) => {
+      if (v != null) {
+        copy[k] = compactDeep(v, ignoredPatterns, `${currentPath}.${k}`);
+      }
+    });
+    return copy;
+  } else if (Array.isArray(obj)) {
+    return obj.map(v => compactDeep(v, ignoredPatterns, `${currentPath}[]`));
+  } else if (typeof obj === 'object') {
+    const copy = {};
+    Object.entries(obj ?? {}).forEach(([key, value]) => {
+      const path = `${currentPath}.${key}`;
+      if (ignoredPatterns.some(pattern => path.includes(pattern))) {
+        return;
+      }
+      // Equivalent false == null
+      if (key === 'isWeak' && value === false) {
+        return;
+      }
+
+      if (value != null) {
+        copy[key] = compactDeep(value, ignoredPatterns, path);
+      }
+    });
+    return copy;
+  } else if (obj != null) {
+    return obj;
+  }
+}
+
+function assetGraphDiff(jsAssetGraph: AssetGraph, rustAssetGraph: AssetGraph) {
+  const getNodes = graph => {
+    let nodes = {};
+
+    graph.traverse(nodeId => {
+      let node: AssetGraphNode | null = graph.getNode(nodeId) ?? null;
+      if (!node) return;
+
+      if (node.type === 'dependency') {
+        let sourcePath = node.value.sourcePath ?? toProjectPath('', 'entry');
+        nodes[
+          `dep:${fromProjectPathRelative(sourcePath)}:${node.value.specifier}`
+        ] = filterNode(node);
+      } else if (node.type === 'asset') {
+        nodes[`asset:${fromProjectPathRelative(node.value.filePath)}`] =
+          filterNode(node);
+      }
+    });
+
+    return nodes;
+  };
+
+  const jsNodes = getNodes(jsAssetGraph);
+  const rustNodes = getNodes(rustAssetGraph);
+
+  const all = new Set([...Object.keys(jsNodes), ...Object.keys(rustNodes)]);
+  const missing = [];
+  const extra = [];
+
+  for (const key of all.keys()) {
+    if (process.env.NATIVE_COMPARE !== 'true') {
+      continue;
+    }
+    let jsNode = jsNodes[key];
+    let rustNode = rustNodes[key];
+
+    if (!rustNode) {
+      missing.push(key);
+      continue;
+    }
+    if (!jsNode) {
+      extra.push(key);
+      continue;
+    }
+
+    console.log(key);
+    const ignoredPatterns = [
+      // ignored because we don't copy the environment ID back from rust
+      // in the target value
+      '$.value.target.env.id',
+      // ignore asset.mapKey because we don't do persistence on rust yet
+      '$.value.mapKey',
+      // ignore this because it's just the output hash. We don't need to compute
+      // this yet
+      '$.value.outputHash',
+      // ignore correspondingRequest from all nodes
+      '$.correspondingRequest',
+    ];
+    console.log(
+      diff(
+        compactDeep(rustNode, ignoredPatterns),
+        compactDeep(jsNode, ignoredPatterns),
+      ),
+    );
+  }
+
+  console.log('Missing', missing);
+  console.log('Extra', extra);
+}
+
+module.exports = assetGraphDiff;

package/src/resolveOptions.js

@@ -12,7 +12,8 @@ import type {AtlaspackOptions} from './types';
 import path from 'path';
 import {hashString} from '@atlaspack/rust';
 import {NodeFS} from '@atlaspack/fs';
-import {LMDBCache, FSCache} from '@atlaspack/cache';
+import {LMDBCache, LMDBLiteCache, FSCache} from '@atlaspack/cache';
+import {getFeatureFlag} from '@atlaspack/feature-flags';
 import {NodePackageManager} from '@atlaspack/package-manager';
 import {
   getRootDir,
@@ -108,10 +109,17 @@ export default async function resolveOptions(
       ? path.resolve(initialOptions.watchDir)
       : projectRoot;
 
+  const makeLMDBCache = () => {
+    if (getFeatureFlag('useLmdbJsLite')) {
+      return new LMDBLiteCache(cacheDir);
+    }
+    return new LMDBCache(cacheDir);
+  };
+
   let cache =
     initialOptions.cache ??
     (outputFS instanceof NodeFS
-      ? new LMDBCache(cacheDir)
+      ? makeLMDBCache()
       : new FSCache(outputFS, cacheDir));
 
   let mode = initialOptions.mode ?? 'development';
@@ -223,6 +231,8 @@ export default async function resolveOptions(
       outputFormat: initialOptions?.defaultTargetOptions?.outputFormat,
       isLibrary: initialOptions?.defaultTargetOptions?.isLibrary,
     },
+    // unused, feature-flags are set above this to allow this function to use
+    // feature-flags
     featureFlags: {...DEFAULT_FEATURE_FLAGS, ...initialOptions?.featureFlags},
     parcelVersion: ATLASPACK_VERSION,
   };

package/src/types.js

@@ -115,6 +115,7 @@ export const Priority = {
   sync: 0,
   parallel: 1,
   lazy: 2,
+  conditional: 3,
 };
 
 // Must match package_json.rs in node-resolver-rs.
@@ -542,6 +543,7 @@ export type Bundle = {|
   displayName: ?string,
   pipeline: ?string,
   manualSharedBundle?: ?string,
+  conditions?: Map<string, string>,
 |};
 
 export type BundleNode = {|
@@ -580,3 +582,11 @@ export type ValidationOpts = {|
 |};
 
 export type ReportFn = (event: ReporterEvent) => void | Promise<void>;
+
+export type Condition = {|
+  publicId: string,
+  assets: Set<Asset>,
+  key: string,
+  ifTrueDependency: Dependency,
+  ifFalseDependency: Dependency,
+|};