@atlaspack/core 2.12.1-dev.3443 → 2.12.1-dev.3460

package/lib/AssetGraph.js

@@ -153,7 +153,11 @@ class AssetGraph extends _graph().ContentGraph {
         nodes.push(node);
       }
     } else if (assetGroups) {
-      nodes.push(...assetGroups.map(assetGroup => nodeFromAssetGroup(assetGroup)));
+      for (const assetGroup of assetGroups) {
+        // Adding nodes individually ensures we do not encounter a range stack size exceeded error
+        // when there are >100000 asset groups
+        nodes.push(nodeFromAssetGroup(assetGroup));
+      }
     }
     this.replaceNodeIdsConnectedTo((0, _nullthrows().default)(this.rootNodeId), nodes.map(node => this.addNode(node)));
   }

package/lib/Atlaspack.js

@@ -187,7 +187,7 @@ class Atlaspack {
         threads: process.env.NODE_ENV === 'test' ? 2 : undefined,
         entries: Array.isArray(entries) ? entries : entries == null ? undefined : [entries],
         env: resolvedOptions.env,
-        fs: inputFS && (0, _atlaspackV.toFileSystemV3)(inputFS),
+        fs: inputFS && new _atlaspackV.FileSystemV3(inputFS),
         defaultTargetOptions: {
           // $FlowFixMe projectPath is just a string
           distDir: resolvedOptions.defaultTargetOptions.distDir,
@@ -604,6 +604,6 @@ function createWorkerFarm(options = {}) {
   return new (_workers().default)({
     ...options,
     // $FlowFixMe
-    workerPath: process.browser ? '@atlaspack/core/src/worker.js' : require.resolve('./worker')
+    workerPath: process.browser ? '@atlaspack/core/worker' : require.resolve('./worker')
   });
 }

package/lib/Dependency.js

@@ -40,7 +40,7 @@ function createDependencyId({
   const params = {
     sourceAssetId,
     specifier,
-    env,
+    environmentId: env.id,
     target,
     pipeline,
     specifierType: _types.SpecifierType[specifierType],

package/lib/PackagerRunner.js

@@ -11,6 +11,13 @@ function _assert() {
   };
   return data;
 }
+function _utils() {
+  const data = require("@atlaspack/utils");
+  _utils = function () {
+    return data;
+  };
+  return data;
+}
 function _logger() {
   const data = require("@atlaspack/logger");
   _logger = function () {
@@ -25,6 +32,13 @@ function _diagnostic() {
   };
   return data;
 }
+function _stream() {
+  const data = require("stream");
+  _stream = function () {
+    return data;
+  };
+  return data;
+}
 function _nullthrows() {
   const data = _interopRequireDefault(require("nullthrows"));
   _nullthrows = function () {
@@ -300,7 +314,7 @@ class PackagerRunner {
     let bundle = _Bundle.NamedBundle.get(internalBundle, internalBundleGraph, this.options);
     let bundleGraph = new _BundleGraph.default(internalBundleGraph, _Bundle.NamedBundle.get.bind(_Bundle.NamedBundle), this.options);
     let optimizers = await this.config.getOptimizers(bundle.name, internalBundle.pipeline);
-    if (optimizers.length === 0) {
+    if (!optimizers.length) {
       return {
         type: bundle.type,
         contents,
@@ -433,13 +447,40 @@ class PackagerRunner {
     }
     return devDepHashes;
   }
+  async readFromCache(cacheKey) {
+    let contentKey = PackagerRunner.getContentKey(cacheKey);
+    let mapKey = PackagerRunner.getMapKey(cacheKey);
+    let isLargeBlob = await this.options.cache.hasLargeBlob(contentKey);
+    let contentExists = isLargeBlob || (await this.options.cache.has(contentKey));
+    if (!contentExists) {
+      return null;
+    }
+    let mapExists = await this.options.cache.has(mapKey);
+    return {
+      contents: isLargeBlob ? this.options.cache.getStream(contentKey) : (0, _utils().blobToStream)(await this.options.cache.getBlob(contentKey)),
+      map: mapExists ? (0, _utils().blobToStream)(await this.options.cache.getBlob(mapKey)) : null
+    };
+  }
   async writeToCache(cacheKeys, type, contents, map) {
     let size = 0;
     let hash;
     let hashReferences = [];
+    let isLargeBlob = false;
 
     // TODO: don't replace hash references in binary files??
-    if (typeof contents === 'string') {
+    if (contents instanceof _stream().Readable) {
+      isLargeBlob = true;
+      let boundaryStr = '';
+      let h = new (_rust().Hash)();
+      await this.options.cache.setStream(cacheKeys.content, (0, _utils().blobToStream)(contents).pipe(new (_utils().TapStream)(buf => {
+        let str = boundaryStr + buf.toString();
+        hashReferences = hashReferences.concat(str.match(_constants.HASH_REF_REGEX) ?? []);
+        size += buf.length;
+        h.writeBuffer(buf);
+        boundaryStr = str.slice(str.length - BOUNDARY_LENGTH);
+      })));
+      hash = h.finish();
+    } else if (typeof contents === 'string') {
       let buffer = Buffer.from(contents);
       size = buffer.byteLength;
       hash = (0, _rust().hashBuffer)(buffer);
@@ -459,7 +500,8 @@ class PackagerRunner {
       size,
       hash,
       hashReferences,
-      cacheKeys
+      cacheKeys,
+      isLargeBlob
     };
     await this.options.cache.set(cacheKeys.info, info);
     return info;

package/lib/SymbolPropagation.js

@@ -438,15 +438,7 @@ function propagateSymbolsDown(assetGraph, changedAssets, assetGroupsWithRemovedP
       wasNodeDirty = node.usedSymbolsDownDirty;
       node.usedSymbolsDownDirty = false;
     } else if (node.type === 'asset' && node.usedSymbolsDownDirty) {
-      visit(node, assetGraph.getIncomingDependencies(node.value).map(d => {
-        let dep = assetGraph.getNodeByContentKey(d.id);
-        (0, _assert().default)(dep && dep.type === 'dependency');
-        return dep;
-      }), outgoing.map(dep => {
-        let depNode = (0, _nullthrows().default)(assetGraph.getNode(dep));
-        (0, _assert().default)(depNode.type === 'dependency');
-        return depNode;
-      }));
+      visit(node, incomingDependencyNodesFromAsset(assetGraph, node.value), dependencyNodesFromIds(assetGraph, outgoing));
       node.usedSymbolsDownDirty = false;
     }
     for (let child of outgoing) {
@@ -494,23 +486,15 @@ function propagateSymbolsUp(assetGraph, changedAssets, changedDepsUsedSymbolsUpD
     let rootNodeId = (0, _nullthrows().default)(assetGraph.rootNodeId, 'A root node is required to traverse');
     const nodeVisitor = nodeId => {
       let node = (0, _nullthrows().default)(assetGraph.getNode(nodeId));
-      let outgoing = assetGraph.getNodeIdsConnectedFrom(nodeId);
-      for (let childId of outgoing) {
-        let child = (0, _nullthrows().default)(assetGraph.getNode(childId));
-        if (node.type === 'asset') {
-          (0, _assert().default)(child.type === 'dependency');
+      if (node.type === 'asset') {
+        let outgoing = outgoingDependencyNodesFromAsset(assetGraph, nodeId);
+        for (let child of outgoing) {
           if (child.usedSymbolsUpDirtyUp) {
             node.usedSymbolsUpDirty = true;
             child.usedSymbolsUpDirtyUp = false;
           }
         }
-      }
-      if (node.type === 'asset') {
-        let incoming = assetGraph.getIncomingDependencies(node.value).map(d => {
-          let n = assetGraph.getNodeByContentKey(d.id);
-          (0, _assert().default)(n && n.type === 'dependency');
-          return n;
-        });
+        let incoming = incomingDependencyNodesFromAsset(assetGraph, node.value);
         for (let dep of incoming) {
           if (dep.usedSymbolsUpDirtyDown) {
             dep.usedSymbolsUpDirtyDown = false;
@@ -518,11 +502,7 @@ function propagateSymbolsUp(assetGraph, changedAssets, changedDepsUsedSymbolsUpD
           }
         }
         if (node.usedSymbolsUpDirty) {
-          let e = visit(node, incoming, outgoing.map(depNodeId => {
-            let depNode = (0, _nullthrows().default)(assetGraph.getNode(depNodeId));
-            (0, _assert().default)(depNode.type === 'dependency');
-            return depNode;
-          }));
+          let e = visit(node, incoming, outgoing);
           if (e.length > 0) {
             node.usedSymbolsUpDirty = true;
             errors.set(nodeId, e);
@@ -548,22 +528,14 @@ function propagateSymbolsUp(assetGraph, changedAssets, changedDepsUsedSymbolsUpD
     let queuedNodeId = setPop(queue);
     let node = (0, _nullthrows().default)(assetGraph.getNode(queuedNodeId));
     if (node.type === 'asset') {
-      let incoming = assetGraph.getIncomingDependencies(node.value).map(dep => {
-        let depNode = assetGraph.getNodeByContentKey(dep.id);
-        (0, _assert().default)(depNode && depNode.type === 'dependency');
-        return depNode;
-      });
+      let incoming = incomingDependencyNodesFromAsset(assetGraph, node.value);
       for (let dep of incoming) {
         if (dep.usedSymbolsUpDirtyDown) {
           dep.usedSymbolsUpDirtyDown = false;
           node.usedSymbolsUpDirty = true;
         }
       }
-      let outgoing = assetGraph.getNodeIdsConnectedFrom(queuedNodeId).map(depNodeId => {
-        let depNode = (0, _nullthrows().default)(assetGraph.getNode(depNodeId));
-        (0, _assert().default)(depNode.type === 'dependency');
-        return depNode;
-      });
+      let outgoing = outgoingDependencyNodesFromAsset(assetGraph, queuedNodeId);
       for (let dep of outgoing) {
         if (dep.usedSymbolsUpDirtyUp) {
           node.usedSymbolsUpDirty = true;
@@ -622,4 +594,21 @@ function setPop(set) {
   let v = (0, _nullthrows().default)(set.values().next().value);
   set.delete(v);
   return v;
+}
+function outgoingDependencyNodesFromAsset(assetGraph, assetNode) {
+  return dependencyNodesFromIds(assetGraph, assetGraph.getNodeIdsConnectedFrom(assetNode));
+}
+function dependencyNodesFromIds(assetGraph, dependencyIds) {
+  return dependencyIds.map(depNodeId => {
+    let depNode = (0, _nullthrows().default)(assetGraph.getNode(depNodeId));
+    (0, _assert().default)(depNode.type === 'dependency');
+    return depNode;
+  });
+}
+function incomingDependencyNodesFromAsset(assetGraph, assetNodeValue) {
+  return assetGraph.getIncomingDependencies(assetNodeValue).map(d => {
+    let n = assetGraph.getNodeByContentKey(d.id);
+    (0, _assert().default)(n && n.type === 'dependency');
+    return n;
+  });
 }

package/lib/assetUtils.js

@@ -62,7 +62,7 @@ function _profiler() {
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 function createAssetIdFromOptions(options) {
   return (0, _rust().createAssetId)({
-    env: options.env,
+    environmentId: options.env.id,
     filePath: options.filePath,
     code: options.code,
     pipeline: options.pipeline,

package/lib/atlaspack-v3/fs.js

@@ -3,35 +3,37 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.toFileSystemV3 = toFileSystemV3;
+exports.FileSystemV3 = void 0;
 var _jsCallable = require("./jsCallable");
-// Move to @atlaspack/utils or a dedicated v3 / migration package later
-function toFileSystemV3(fs) {
-  return {
-    canonicalize: (0, _jsCallable.jsCallable)(path => fs.realpathSync(path)),
-    createDirectory: (0, _jsCallable.jsCallable)(path => fs.mkdirp(path)),
-    cwd: (0, _jsCallable.jsCallable)(() => fs.cwd()),
-    readFile: (0, _jsCallable.jsCallable)((path, encoding) => {
-      if (!encoding) {
-        // $FlowFixMe
-        return [...fs.readFileSync(path)];
-      } else {
-        return fs.readFileSync(path, encoding);
-      }
-    }),
-    isFile: path => {
-      try {
-        return fs.statSync(path).isFile();
-      } catch {
-        return false;
-      }
-    },
-    isDir: path => {
-      try {
-        return fs.statSync(path).isDirectory();
-      } catch {
-        return false;
-      }
+class FileSystemV3 {
+  #fs;
+  constructor(fs) {
+    this.#fs = fs;
+  }
+  canonicalize = (0, _jsCallable.jsCallable)(path => this.#fs.realpathSync(path));
+  createDirectory = (0, _jsCallable.jsCallable)(path => this.#fs.mkdirp(path));
+  cwd = (0, _jsCallable.jsCallable)(() => this.#fs.cwd());
+  isFile = path => {
+    try {
+      return this.#fs.statSync(path).isFile();
+    } catch {
+      return false;
     }
   };
-}
+  isDir = path => {
+    try {
+      return this.#fs.statSync(path).isDirectory();
+    } catch {
+      return false;
+    }
+  };
+  readFile = (0, _jsCallable.jsCallable)((path, encoding) => {
+    if (!encoding) {
+      // $FlowFixMe
+      return [...this.#fs.readFileSync(path)];
+    } else {
+      return this.#fs.readFileSync(path, encoding);
+    }
+  });
+}
+exports.FileSystemV3 = FileSystemV3;

package/lib/atlaspack-v3/index.js

@@ -9,10 +9,10 @@ Object.defineProperty(exports, "AtlaspackV3", {
     return _AtlaspackV.AtlaspackV3;
   }
 });
-Object.defineProperty(exports, "toFileSystemV3", {
+Object.defineProperty(exports, "FileSystemV3", {
   enumerable: true,
   get: function () {
-    return _fs.toFileSystemV3;
+    return _fs.FileSystemV3;
   }
 });
 var _fs = require("./fs");

package/lib/atlaspack-v3/jsCallable.js

@@ -5,11 +5,16 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.jsCallable = jsCallable;
 function jsCallable(fn) {
-  return (...args) => {
+  return async (...args) => {
     try {
-      return fn(...args);
-    } catch (err) {
-      return err;
+      // $FlowFixMe "await" is needed to log the error from
+      // Promises and is safe to use on non-promises
+      return await fn(...args);
+      // $FlowFixMe
+    } catch (error) {
+      // eslint-disable-next-line no-console
+      console.error(error);
+      throw error;
     }
   };
 }

package/lib/atlaspack-v3/worker/compat/asset-symbols.js

@@ -0,0 +1,192 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.MutableDependencySymbols = exports.MutableAssetSymbols = exports.AssetSymbols = void 0;
+class MutableDependencySymbols {
+  #symbols;
+  #locals;
+  get isCleared() {
+    return this.#symbols.size === 0;
+  }
+  constructor(inner) {
+    this.#symbols = new Map();
+    this.#locals = new Set();
+    for (const {
+      exported,
+      local,
+      loc,
+      isWeak
+    } of inner || []) {
+      this.set(exported, local, loc, isWeak);
+    }
+  }
+  ensure() {
+    throw new Error('MutableDependencySymbols.ensure()');
+  }
+  get(exportSymbol) {
+    return this.#symbols.get(exportSymbol);
+  }
+  hasExportSymbol(exportSymbol) {
+    return this.#symbols.has(exportSymbol);
+  }
+  hasLocalSymbol(local) {
+    return this.#locals.has(local);
+  }
+  exportSymbols() {
+    return this.#symbols.keys();
+  }
+  set(exportSymbol, local, loc, isWeak) {
+    this.#locals.add(local);
+    this.#symbols.set(exportSymbol, {
+      local,
+      loc,
+      meta: {},
+      isWeak: isWeak ?? false
+    });
+  }
+  delete(exportSymbol) {
+    let existing = this.#symbols.get(exportSymbol);
+    if (!existing) return;
+    this.#locals.delete(existing.local);
+    this.#symbols.delete(exportSymbol);
+  }
+
+  /*::  @@iterator(): Iterator<[Symbol, DependencyAssetSymbol]> { return ({}: any); } */
+  // $FlowFixMe Flow can't do computed properties
+  [Symbol.iterator]() {
+    return this.#symbols.values();
+  }
+}
+exports.MutableDependencySymbols = MutableDependencySymbols;
+class AssetSymbols {
+  #symbols;
+  #locals;
+  get isCleared() {
+    return this.#symbols.size === 0;
+  }
+  constructor(inner) {
+    this.#symbols = new Map();
+    this.#locals = new Set();
+    for (const {
+      exported,
+      local,
+      loc
+    } of inner || []) {
+      this.#set(exported, local, loc);
+    }
+  }
+  #set(exportSymbol, local, loc) {
+    this.#locals.add(local);
+    this.#symbols.set(exportSymbol, {
+      local,
+      loc,
+      meta: {}
+    });
+  }
+  get(exportSymbol) {
+    return this.#symbols.get(exportSymbol);
+  }
+  hasExportSymbol(exportSymbol) {
+    return this.#symbols.has(exportSymbol);
+  }
+  hasLocalSymbol(local) {
+    return this.#locals.has(local);
+  }
+  exportSymbols() {
+    return this.#symbols.keys();
+  }
+  intoNapi() {
+    return [];
+  }
+
+  /*::  @@iterator(): Iterator<[Symbol, AssetSymbol]> { return ({}: any); } */
+  // $FlowFixMe Flow can't do computed properties
+  [Symbol.iterator]() {
+    return this.#symbols.values();
+  }
+}
+exports.AssetSymbols = AssetSymbols;
+class MutableAssetSymbols {
+  #symbols;
+  #locals;
+  get isCleared() {
+    return this.#symbols.size === 0;
+  }
+  constructor(inner) {
+    this.#symbols = new Map();
+    this.#locals = new Set();
+    for (const {
+      exported,
+      local,
+      loc
+    } of inner || []) {
+      this.set(exported, local, loc);
+    }
+  }
+  intoNapi() {
+    const results = [];
+    for (const [exportSymbol, {
+      local,
+      loc
+    }] of this.#symbols.entries()) {
+      results.push({
+        exported: exportSymbol,
+        local,
+        loc: loc ? {
+          filePath: loc.filePath,
+          start: {
+            line: loc.start.line,
+            column: loc.start.column
+          },
+          end: {
+            line: loc.end.line,
+            column: loc.end.column
+          }
+        } : undefined,
+        // TODO
+        isWeak: false,
+        isEsmExport: false,
+        selfReferenced: false
+      });
+    }
+    return results;
+  }
+  ensure() {
+    throw new Error('MutableAssetSymbols.ensure()');
+  }
+  set(exportSymbol, local, loc) {
+    this.#locals.add(local);
+    this.#symbols.set(exportSymbol, {
+      local,
+      loc,
+      meta: {}
+    });
+  }
+  get(exportSymbol) {
+    return this.#symbols.get(exportSymbol);
+  }
+  hasExportSymbol(exportSymbol) {
+    return this.#symbols.has(exportSymbol);
+  }
+  hasLocalSymbol(local) {
+    return this.#locals.has(local);
+  }
+  exportSymbols() {
+    return this.#symbols.keys();
+  }
+  delete(exportSymbol) {
+    let existing = this.#symbols.get(exportSymbol);
+    if (!existing) return;
+    this.#locals.delete(existing.local);
+    this.#symbols.delete(exportSymbol);
+  }
+
+  /*::  @@iterator(): Iterator<[Symbol, AssetSymbol]> { return ({}: any); } */
+  // $FlowFixMe Flow can't do computed properties
+  [Symbol.iterator]() {
+    return this.#symbols.values();
+  }
+}
+exports.MutableAssetSymbols = MutableAssetSymbols;

package/lib/atlaspack-v3/worker/compat/bitflags.js

@@ -0,0 +1,84 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.specifierTypeMap = exports.packageConditionsMap = exports.dependencyPriorityMap = exports.bundleBehaviorMap = exports.BitFlags = void 0;
+/// BitFlags is used to map number/string types from napi types
+class BitFlags {
+  #kv;
+  #vk;
+  constructor(source) {
+    this.#kv = source;
+    this.#vk = Object.fromEntries(
+    // $FlowFixMe
+    Object.entries(source).map(a => a.reverse()));
+  }
+  into(key) {
+    const found = this.#kv[key];
+    if (found === undefined) {
+      // $FlowFixMe
+      throw new Error(`Invalid BundleBehavior(${key})`);
+    }
+    return found;
+  }
+  intoNullable(key) {
+    if (key === undefined || key === null) {
+      return undefined;
+    }
+    return this.into(key);
+  }
+  intoArray(keys) {
+    return keys.map(key => this.into(key));
+  }
+  from(key) {
+    const found = this.#vk[key];
+    if (found === undefined) {
+      // $FlowFixMe
+      throw new Error(`Invalid BundleBehavior(${key})`);
+    }
+    return found;
+  }
+  fromNullable(key) {
+    if (key === undefined || key === null) {
+      return undefined;
+    }
+    return this.from(key);
+  }
+  fromArray(keys) {
+    return keys.map(key => this.from(key));
+  }
+}
+exports.BitFlags = BitFlags;
+const bundleBehaviorMap = exports.bundleBehaviorMap = new BitFlags({
+  inline: 0,
+  isolated: 1
+});
+const dependencyPriorityMap = exports.dependencyPriorityMap = new BitFlags({
+  sync: 0,
+  parallel: 1,
+  lazy: 2,
+  conditional: 3
+});
+const packageConditionsMap = exports.packageConditionsMap = new BitFlags({
+  import: 0,
+  require: 1,
+  module: 2,
+  node: 3,
+  browser: 4,
+  worker: 5,
+  worklet: 6,
+  electron: 7,
+  development: 8,
+  production: 9,
+  types: 10,
+  default: 11,
+  style: 12,
+  sass: 13
+});
+const specifierTypeMap = exports.specifierTypeMap = new BitFlags({
+  esm: 0,
+  commonjs: 1,
+  url: 2,
+  custom: 3
+});

package/lib/atlaspack-v3/worker/compat/dependency.js

@@ -0,0 +1,44 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.Dependency = void 0;
+var _target = require("./target");
+var _assetSymbols = require("./asset-symbols");
+var _bitflags = require("./bitflags");
+class Dependency {
+  get id() {
+    throw new Error('Dependency.id');
+  }
+  constructor(inner, env) {
+    this.env = env;
+    this.meta = inner.meta || {};
+    this.target = undefined;
+    if (inner.target) {
+      this.target = new _target.Target(inner.target, this.env);
+    }
+    if (!inner.bundleBehavior) {
+      this.bundleBehavior = undefined;
+    } else {
+      this.bundleBehavior = _bitflags.bundleBehaviorMap.from(inner.bundleBehavior);
+    }
+    this.bundleBehavior = undefined;
+    this.symbols = new _assetSymbols.MutableDependencySymbols(inner.symbols || []);
+    this.specifier = inner.specifier;
+    this.specifierType = _bitflags.specifierTypeMap.from(inner.specifierType);
+    this.priority = _bitflags.dependencyPriorityMap.from(inner.priority);
+    this.needsStableName = inner.needsStableName;
+    this.isOptional = inner.isOptional;
+    this.isEntry = inner.isEntry;
+    this.loc = inner.loc;
+    this.packageConditions = _bitflags.packageConditionsMap.fromArray(inner.packageConditions || []);
+    this.sourceAssetId = inner.sourceAssetId;
+    this.sourcePath = inner.sourcePath;
+    this.sourceAssetType = inner.sourceAssetType;
+    this.resolveFrom = inner.resolveFrom;
+    this.range = inner.range;
+    this.pipeline = inner.pipeline;
+  }
+}
+exports.Dependency = Dependency;