@atlaspack/core 2.12.1-dev.3401 → 2.12.1-dev.3450

package/src/requests/WriteBundleRequest.js

@@ -3,6 +3,7 @@
 import type {FileSystem, FileOptions} from '@atlaspack/fs';
 import type {ContentKey} from '@atlaspack/graph';
 import type {Async, FilePath, Compressor} from '@atlaspack/types';
+import {replaceHashReferences} from '@atlaspack/rust';
 
 import type {RunAPI, StaticRunOpts} from '../RequestTracker';
 import type {Bundle, PackagedBundleInfo, AtlaspackOptions} from '../types';
@@ -16,7 +17,7 @@ import {HASH_REF_HASH_LEN, HASH_REF_PREFIX} from '../constants';
 import nullthrows from 'nullthrows';
 import path from 'path';
 import {NamedBundle} from '../public/Bundle';
-import {blobToStream, TapStream} from '@atlaspack/utils';
+import {blobToStream} from '@atlaspack/utils';
 import {Readable, Transform, pipeline} from 'stream';
 import {
   fromProjectPath,
@@ -129,20 +130,8 @@ async function run({input, options, api}) {
       : {
           mode: (await inputFS.stat(mainEntry.filePath)).mode,
         };
-  let contentStream: Readable;
-  if (info.isLargeBlob) {
-    contentStream = options.cache.getStream(cacheKeys.content);
-  } else {
-    contentStream = blobToStream(
-      await options.cache.getBlob(cacheKeys.content),
-    );
-  }
-  let size = 0;
-  contentStream = contentStream.pipe(
-    new TapStream(buf => {
-      size += buf.length;
-    }),
-  );
+  const contents: Buffer = await options.cache.getBlob(cacheKeys.content);
+  const size = contents.byteLength;
 
   let configResult = nullthrows(
     await api.runRequest<null, ConfigAndCachePath>(
@@ -155,7 +144,7 @@ async function run({input, options, api}) {
   invalidateDevDeps(invalidDevDeps, options, config);
 
   await writeFiles(
-    contentStream,
+    contents,
     info,
     hashRefToNameHash,
     options,
@@ -174,7 +163,7 @@ async function run({input, options, api}) {
     (await options.cache.has(mapKey))
   ) {
     await writeFiles(
-      blobToStream(await options.cache.getBlob(mapKey)),
+      await options.cache.getBlob(mapKey),
       info,
       hashRefToNameHash,
       options,
@@ -201,7 +190,7 @@ async function run({input, options, api}) {
 }
 
 async function writeFiles(
-  inputStream: stream$Readable,
+  contents: Buffer,
   info: BundleInfo,
   hashRefToNameHash: Map<string, string>,
   options: AtlaspackOptions,
@@ -217,16 +206,19 @@ async function writeFiles(
   );
   let fullPath = fromProjectPath(options.projectRoot, filePath);
 
-  let stream = info.hashReferences.length
-    ? inputStream.pipe(replaceStream(hashRefToNameHash))
-    : inputStream;
+  const stream = info.hashReferences.length
+    ? replaceHashReferences(
+        contents,
+        Object.fromEntries(hashRefToNameHash.entries()),
+      )
+    : contents;
 
   let promises = [];
   for (let compressor of compressors) {
     promises.push(
       runCompressor(
         compressor,
-        cloneStream(stream),
+        blobToStream(stream),
         options,
         outputFS,
         fullPath,
@@ -300,7 +292,9 @@ async function runCompressor(
   }
 }
 
-function replaceStream(hashRefToNameHash) {
+export function replaceStream(
+  hashRefToNameHash: Map<string, string>,
+): Transform {
   let boundaryStr = Buffer.alloc(0);
   let replaced = Buffer.alloc(0);
   return new Transform({

package/src/requests/asset-graph-diff.js

@@ -0,0 +1,145 @@
+/* eslint-disable no-console */
+// @flow strict-local
+
+// $FlowFixMe
+import deepClone from 'rfdc/default';
+// $FlowFixMe
+import {diff} from 'jest-diff';
+import AssetGraph from '../AssetGraph';
+import type {AssetGraphNode} from '../types';
+import {fromProjectPathRelative, toProjectPath} from '../projectPath';
+
+function filterNode(node) {
+  let clone = deepClone(node);
+
+  // Clean up anything you don't want to see in the diff
+  // delete clone.id;
+  delete clone.value.id;
+  delete clone.value.meta.id;
+  delete clone.value.sourceAssetId;
+  delete clone.value.env.id;
+  delete clone.value.isEsm;
+  delete clone.value.shouldWrap;
+  delete clone.value.contentKey;
+  delete clone.value.placeholder;
+  delete clone.value.code;
+  delete clone.value.hasCjsExports;
+  delete clone.value.staticExports;
+  delete clone.value.isConstantModule;
+  delete clone.value.hasNodeReplacements;
+  delete clone.value.stats;
+  delete clone.value.astKey;
+  delete clone.value.astGenerator;
+  delete clone.value.dependencies;
+
+  return clone;
+}
+
+function compactDeep(
+  obj: mixed,
+  ignoredPatterns: Array<string> = [],
+  currentPath: string = '$',
+) {
+  if (obj instanceof Map) {
+    const copy = {};
+    Array.from(obj.entries()).forEach(([k, v]) => {
+      if (v != null) {
+        copy[k] = compactDeep(v, ignoredPatterns, `${currentPath}.${k}`);
+      }
+    });
+    return copy;
+  } else if (Array.isArray(obj)) {
+    return obj.map(v => compactDeep(v, ignoredPatterns, `${currentPath}[]`));
+  } else if (typeof obj === 'object') {
+    const copy = {};
+    Object.entries(obj ?? {}).forEach(([key, value]) => {
+      const path = `${currentPath}.${key}`;
+      if (ignoredPatterns.some(pattern => path.includes(pattern))) {
+        return;
+      }
+      // Equivalent false == null
+      if (key === 'isWeak' && value === false) {
+        return;
+      }
+
+      if (value != null) {
+        copy[key] = compactDeep(value, ignoredPatterns, path);
+      }
+    });
+    return copy;
+  } else if (obj != null) {
+    return obj;
+  }
+}
+
+function assetGraphDiff(jsAssetGraph: AssetGraph, rustAssetGraph: AssetGraph) {
+  const getNodes = graph => {
+    let nodes = {};
+
+    graph.traverse(nodeId => {
+      let node: AssetGraphNode | null = graph.getNode(nodeId) ?? null;
+      if (!node) return;
+
+      if (node.type === 'dependency') {
+        let sourcePath = node.value.sourcePath ?? toProjectPath('', 'entry');
+        nodes[
+          `dep:${fromProjectPathRelative(sourcePath)}:${node.value.specifier}`
+        ] = filterNode(node);
+      } else if (node.type === 'asset') {
+        nodes[`asset:${fromProjectPathRelative(node.value.filePath)}`] =
+          filterNode(node);
+      }
+    });
+
+    return nodes;
+  };
+
+  const jsNodes = getNodes(jsAssetGraph);
+  const rustNodes = getNodes(rustAssetGraph);
+
+  const all = new Set([...Object.keys(jsNodes), ...Object.keys(rustNodes)]);
+  const missing = [];
+  const extra = [];
+
+  for (const key of all.keys()) {
+    if (process.env.NATIVE_COMPARE !== 'true') {
+      continue;
+    }
+    let jsNode = jsNodes[key];
+    let rustNode = rustNodes[key];
+
+    if (!rustNode) {
+      missing.push(key);
+      continue;
+    }
+    if (!jsNode) {
+      extra.push(key);
+      continue;
+    }
+
+    console.log(key);
+    const ignoredPatterns = [
+      // ignored because we don't copy the environment ID back from rust
+      // in the target value
+      '$.value.target.env.id',
+      // ignore asset.mapKey because we don't do persistence on rust yet
+      '$.value.mapKey',
+      // ignore this because it's just the output hash. We don't need to compute
+      // this yet
+      '$.value.outputHash',
+      // ignore correspondingRequest from all nodes
+      '$.correspondingRequest',
+    ];
+    console.log(
+      diff(
+        compactDeep(rustNode, ignoredPatterns),
+        compactDeep(jsNode, ignoredPatterns),
+      ),
+    );
+  }
+
+  console.log('Missing', missing);
+  console.log('Extra', extra);
+}
+
+module.exports = assetGraphDiff;

package/src/resolveOptions.js

@@ -12,7 +12,8 @@ import type {AtlaspackOptions} from './types';
 import path from 'path';
 import {hashString} from '@atlaspack/rust';
 import {NodeFS} from '@atlaspack/fs';
-import {LMDBCache, FSCache} from '@atlaspack/cache';
+import {LMDBCache, LMDBLiteCache, FSCache} from '@atlaspack/cache';
+import {getFeatureFlag} from '@atlaspack/feature-flags';
 import {NodePackageManager} from '@atlaspack/package-manager';
 import {
   getRootDir,
@@ -108,10 +109,17 @@ export default async function resolveOptions(
       ? path.resolve(initialOptions.watchDir)
       : projectRoot;
 
+  const makeLMDBCache = () => {
+    if (getFeatureFlag('useLmdbJsLite')) {
+      return new LMDBLiteCache(cacheDir);
+    }
+    return new LMDBCache(cacheDir);
+  };
+
   let cache =
     initialOptions.cache ??
     (outputFS instanceof NodeFS
-      ? new LMDBCache(cacheDir)
+      ? makeLMDBCache()
       : new FSCache(outputFS, cacheDir));
 
   let mode = initialOptions.mode ?? 'development';
@@ -223,6 +231,8 @@ export default async function resolveOptions(
       outputFormat: initialOptions?.defaultTargetOptions?.outputFormat,
       isLibrary: initialOptions?.defaultTargetOptions?.isLibrary,
     },
+    // unused, feature-flags are set above this to allow this function to use
+    // feature-flags
     featureFlags: {...DEFAULT_FEATURE_FLAGS, ...initialOptions?.featureFlags},
     parcelVersion: ATLASPACK_VERSION,
   };

package/src/types.js

@@ -115,6 +115,7 @@ export const Priority = {
   sync: 0,
   parallel: 1,
   lazy: 2,
+  conditional: 3,
 };
 
 // Must match package_json.rs in node-resolver-rs.
@@ -542,6 +543,7 @@ export type Bundle = {|
   displayName: ?string,
   pipeline: ?string,
   manualSharedBundle?: ?string,
+  conditions?: Map<string, string>,
 |};
 
 export type BundleNode = {|
@@ -580,3 +582,11 @@ export type ValidationOpts = {|
 |};
 
 export type ReportFn = (event: ReporterEvent) => void | Promise<void>;
+
+export type Condition = {|
+  publicId: string,
+  assets: Set<Asset>,
+  key: string,
+  ifTrueDependency: Dependency,
+  ifFalseDependency: Dependency,
+|};

package/test/AssetGraph.test.js

@@ -9,13 +9,16 @@ import AssetGraph, {
   nodeFromAsset,
 } from '../src/AssetGraph';
 import {createDependency as _createDependency} from '../src/Dependency';
-import {createAsset as _createAsset} from '../src/assetUtils';
+import {
+  createAsset as _createAsset,
+  type AssetOptions,
+} from '../src/assetUtils';
 import {DEFAULT_ENV, DEFAULT_TARGETS} from './test-utils';
 import {toProjectPath as _toProjectPath} from '../src/projectPath';
 
 const stats = {size: 0, time: 0};
 
-function createAsset(opts) {
+function createAsset(opts: AssetOptions) {
   return _createAsset('/', opts);
 }
 
@@ -134,7 +137,7 @@ describe('AssetGraph', () => {
       graph.hasContentKey(
         createDependency({
           specifier: 'path/to/index1/src/main.js',
-          specifierType: 'esm',
+          specifierType: 'url',
           target: DEFAULT_TARGETS[0],
           env: DEFAULT_ENV,
         }).id,
@@ -144,7 +147,7 @@ describe('AssetGraph', () => {
       graph.hasContentKey(
         createDependency({
           specifier: 'path/to/index2/src/main.js',
-          specifierType: 'esm',
+          specifierType: 'url',
           target: DEFAULT_TARGETS[0],
           env: DEFAULT_ENV,
         }).id,
@@ -191,7 +194,7 @@ describe('AssetGraph', () => {
         to: graph.getNodeIdByContentKey(
           createDependency({
             specifier: 'path/to/index1/src/main.js',
-            specifierType: 'esm',
+            specifierType: 'url',
             target: DEFAULT_TARGETS[0],
             env: DEFAULT_ENV,
           }).id,
@@ -208,7 +211,7 @@ describe('AssetGraph', () => {
         to: graph.getNodeIdByContentKey(
           createDependency({
             specifier: 'path/to/index2/src/main.js',
-            specifierType: 'esm',
+            specifierType: 'url',
             target: DEFAULT_TARGETS[0],
             env: DEFAULT_ENV,
           }).id,
@@ -246,7 +249,7 @@ describe('AssetGraph', () => {
 
     let dep = createDependency({
       specifier: 'path/to/index/src/main.js',
-      specifierType: 'esm',
+      specifierType: 'url',
       target: DEFAULT_TARGETS[0],
       env: DEFAULT_ENV,
     });
@@ -310,7 +313,7 @@ describe('AssetGraph', () => {
 
     let dep = createDependency({
       specifier: 'path/to/index/src/main.js',
-      specifierType: 'esm',
+      specifierType: 'url',
       target: DEFAULT_TARGETS[0],
       env: DEFAULT_ENV,
       sourcePath: '',
@@ -322,6 +325,7 @@ describe('AssetGraph', () => {
     let assets = [
       createAsset({
         id: '1',
+        code: null,
         filePath,
         type: 'js',
         isSource: true,
@@ -341,6 +345,7 @@ describe('AssetGraph', () => {
       }),
       createAsset({
         id: '2',
+        code: null,
         filePath,
         type: 'js',
         isSource: true,
@@ -360,6 +365,7 @@ describe('AssetGraph', () => {
       }),
       createAsset({
         id: '3',
+        code: null,
         filePath,
         type: 'js',
         isSource: true,
@@ -400,6 +406,7 @@ describe('AssetGraph', () => {
     let assets2 = [
       createAsset({
         id: '1',
+        code: null,
         filePath,
         type: 'js',
         isSource: true,
@@ -419,6 +426,7 @@ describe('AssetGraph', () => {
       }),
       createAsset({
         id: '2',
+        code: null,
         filePath,
         type: 'js',
         isSource: true,
@@ -473,7 +481,7 @@ describe('AssetGraph', () => {
 
     let dep = createDependency({
       specifier: 'path/to/index/src/main.js',
-      specifierType: 'esm',
+      specifierType: 'url',
       env: DEFAULT_ENV,
       target: DEFAULT_TARGETS[0],
     });
@@ -496,6 +504,7 @@ describe('AssetGraph', () => {
     let assets = [
       createAsset({
         id: '1',
+        code: null,
         filePath,
         type: 'js',
         isSource: true,
@@ -505,6 +514,7 @@ describe('AssetGraph', () => {
       }),
       createAsset({
         id: '2',
+        code: null,
         uniqueKey: 'dependent-asset-1',
         filePath,
         type: 'js',
@@ -515,6 +525,7 @@ describe('AssetGraph', () => {
       }),
       createAsset({
         id: '3',
+        code: null,
         uniqueKey: 'dependent-asset-2',
         filePath,
         type: 'js',
@@ -572,6 +583,7 @@ describe('AssetGraph', () => {
     });
     let indexAsset = createAsset({
       id: 'assetIndex',
+      code: null,
       filePath: toProjectPath('/index.js'),
       type: 'js',
       isSource: true,
@@ -600,6 +612,7 @@ describe('AssetGraph', () => {
     let fooUtilsDepNode = nodeFromDep(fooUtilsDep);
     let fooAsset = createAsset({
       id: 'assetFoo',
+      code: null,
       filePath: toProjectPath('/foo.js'),
       type: 'js',
       isSource: true,
@@ -642,6 +655,7 @@ describe('AssetGraph', () => {
     });
     let barAsset = createAsset({
       id: 'assetBar',
+      code: null,
       filePath: toProjectPath('/bar.js'),
       type: 'js',
       isSource: true,

package/test/AtlaspackConfigRequest.test.js

@@ -13,107 +13,11 @@ import {
   resolveAtlaspackConfig,
   processConfig,
 } from '../src/requests/AtlaspackConfigRequest';
-import {validatePackageName} from '../src/AtlaspackConfig.schema';
 import {DEFAULT_OPTIONS, relative} from './test-utils';
 import {toProjectPath} from '../src/projectPath';
 
 describe('AtlaspackConfigRequest', () => {
-  describe('validatePackageName', () => {
-    it('should error on an invalid official package', () => {
-      assert.throws(() => {
-        validatePackageName('@atlaspack/foo-bar', 'transform', 'transformers');
-      }, /Official atlaspack transform packages must be named according to "@atlaspack\/transform-{name}"/);
-
-      assert.throws(() => {
-        validatePackageName(
-          '@atlaspack/transformer',
-          'transform',
-          'transformers',
-        );
-      }, /Official atlaspack transform packages must be named according to "@atlaspack\/transform-{name}"/);
-    });
-
-    it('should succeed on a valid official package', () => {
-      validatePackageName(
-        '@atlaspack/transform-bar',
-        'transform',
-        'transformers',
-      );
-    });
-
-    it('should error on an invalid community package', () => {
-      assert.throws(() => {
-        validatePackageName('foo-bar', 'transform', 'transformers');
-      }, /Atlaspack transform packages must be named according to "parcel-transform-{name}"/);
-
-      assert.throws(() => {
-        validatePackageName('parcel-foo-bar', 'transform', 'transformers');
-      }, /Atlaspack transform packages must be named according to "parcel-transform-{name}"/);
-
-      assert.throws(() => {
-        validatePackageName('parcel-transform', 'transform', 'transformers');
-      }, /Atlaspack transform packages must be named according to "parcel-transform-{name}"/);
-    });
-
-    it('should succeed on a valid community package', () => {
-      validatePackageName('parcel-transform-bar', 'transform', 'transformers');
-    });
-
-    // Skipping this while the migration to parcel occurs
-    it.skip('should error on an invalid scoped package', () => {
-      assert.throws(() => {
-        validatePackageName('@test/foo-bar', 'transform', 'transformers');
-      }, /Scoped atlaspack transform packages must be named according to "@test\/parcel-transform\[-{name}\]"/);
-
-      assert.throws(() => {
-        validatePackageName(
-          '@test/atlaspack-foo-bar',
-          'transform',
-          'transformers',
-        );
-      }, /Scoped atlaspack transform packages must be named according to "@test\/parcel-transform\[-{name}\]"/);
-    });
-
-    it('should succeed on a valid scoped package', () => {
-      validatePackageName(
-        '@test/atlaspack-transform-bar',
-        'transform',
-        'transformers',
-      );
-
-      validatePackageName(
-        '@test/atlaspack-transform',
-        'transform',
-        'transformers',
-      );
-    });
-
-    it('should succeed on a local package', () => {
-      validatePackageName(
-        './atlaspack-transform-bar',
-        'transform',
-        'transformers',
-      );
-      validatePackageName('./bar', 'transform', 'transformers');
-    });
-  });
-
   describe('validateConfigFile', () => {
-    it('should throw on invalid config', () => {
-      assert.throws(() => {
-        validateConfigFile(
-          {
-            filePath: '.parcelrc',
-            extends: 'parcel-config-foo',
-            transformers: {
-              '*.js': ['parcel-invalid-plugin'],
-            },
-          },
-          '.parcelrc',
-        );
-      });
-    });
-
     it('should require pipeline to be an array', () => {
       assert.throws(() => {
         validateConfigFile(
@@ -140,18 +44,6 @@ describe('AtlaspackConfigRequest', () => {
       });
     });
 
-    it('should require package names to be valid', () => {
-      assert.throws(() => {
-        validateConfigFile(
-          {
-            filePath: '.parcelrc',
-            resolvers: ['parcel-foo-bar'],
-          },
-          '.parcelrc',
-        );
-      });
-    });
-
     it('should succeed with an array of valid package names', () => {
       validateConfigFile(
         {
@@ -185,21 +77,6 @@ describe('AtlaspackConfigRequest', () => {
       });
     });
 
-    it('should trigger the validator function for each key', () => {
-      assert.throws(() => {
-        validateConfigFile(
-          {
-            filePath: '.parcelrc',
-            transformers: {
-              'types:*.{ts,tsx}': ['@atlaspack/transformer-typescript-types'],
-              'bundle-text:*': ['-inline-string', '...'],
-            },
-          },
-          '.parcelrc',
-        );
-      });
-    });
-
     it('should require extends to be a string or array of strings', () => {
       assert.throws(() => {
         validateConfigFile(
@@ -242,44 +119,6 @@ describe('AtlaspackConfigRequest', () => {
       );
     });
 
-    it('should validate package names', () => {
-      assert.throws(() => {
-        validateConfigFile(
-          {
-            filePath: '.parcelrc',
-            extends: 'foo',
-          },
-          '.parcelrc',
-        );
-      });
-
-      assert.throws(() => {
-        validateConfigFile(
-          {
-            filePath: '.parcelrc',
-            extends: ['foo', 'bar'],
-          },
-          '.parcelrc',
-        );
-      });
-
-      validateConfigFile(
-        {
-          filePath: '.parcelrc',
-          extends: 'parcel-config-foo',
-        },
-        '.parcelrc',
-      );
-
-      validateConfigFile(
-        {
-          filePath: '.parcelrc',
-          extends: ['parcel-config-foo', 'parcel-config-bar'],
-        },
-        '.parcelrc',
-      );
-    });
-
     it('should throw for invalid top level keys', () => {
       assert.throws(
         () => {