@atlaspack/core 2.12.1-dev.3450 → 2.12.1-dev.3466

package/lib/AssetGraph.js

@@ -153,7 +153,11 @@ class AssetGraph extends _graph().ContentGraph {
         nodes.push(node);
       }
     } else if (assetGroups) {
-      nodes.push(...assetGroups.map(assetGroup => nodeFromAssetGroup(assetGroup)));
+      for (const assetGroup of assetGroups) {
+        // Adding nodes individually ensures we do not encounter a range stack size exceeded error
+        // when there are >100000 asset groups
+        nodes.push(nodeFromAssetGroup(assetGroup));
+      }
     }
     this.replaceNodeIdsConnectedTo((0, _nullthrows().default)(this.rootNodeId), nodes.map(node => this.addNode(node)));
   }

package/lib/Atlaspack.js

@@ -187,7 +187,7 @@ class Atlaspack {
         threads: process.env.NODE_ENV === 'test' ? 2 : undefined,
         entries: Array.isArray(entries) ? entries : entries == null ? undefined : [entries],
         env: resolvedOptions.env,
-        fs: inputFS && (0, _atlaspackV.toFileSystemV3)(inputFS),
+        fs: inputFS && new _atlaspackV.FileSystemV3(inputFS),
         defaultTargetOptions: {
           // $FlowFixMe projectPath is just a string
           distDir: resolvedOptions.defaultTargetOptions.distDir,
@@ -604,6 +604,6 @@ function createWorkerFarm(options = {}) {
   return new (_workers().default)({
     ...options,
     // $FlowFixMe
-    workerPath: process.browser ? '@atlaspack/core/src/worker.js' : require.resolve('./worker')
+    workerPath: process.browser ? '@atlaspack/core/worker' : require.resolve('./worker')
   });
 }

package/lib/CommittedAsset.js

@@ -11,9 +11,9 @@ function _stream() {
   };
   return data;
 }
-function _sourceMap() {
-  const data = _interopRequireDefault(require("@parcel/source-map"));
-  _sourceMap = function () {
+function _buildCache() {
+  const data = require("@atlaspack/build-cache");
+  _buildCache = function () {
     return data;
   };
   return data;
@@ -25,8 +25,14 @@ function _utils() {
   };
   return data;
 }
+function _sourceMap() {
+  const data = _interopRequireDefault(require("@parcel/source-map"));
+  _sourceMap = function () {
+    return data;
+  };
+  return data;
+}
 var _assetUtils = require("./assetUtils");
-var _serializer = require("./serializer");
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 class CommittedAsset {
   constructor(value, options) {
@@ -122,7 +128,7 @@ class CommittedAsset {
       return Promise.resolve(null);
     }
     if (this.ast == null) {
-      this.ast = this.options.cache.getBlob(this.value.astKey).then(serializedAst => (0, _serializer.deserializeRaw)(serializedAst));
+      this.ast = this.options.cache.getBlob(this.value.astKey).then(serializedAst => (0, _buildCache().deserializeRaw)(serializedAst));
     }
     return this.ast;
   }

package/lib/Dependency.js

@@ -40,7 +40,7 @@ function createDependencyId({
   const params = {
     sourceAssetId,
     specifier,
-    env,
+    environmentId: env.id,
     target,
     pipeline,
     specifierType: _types.SpecifierType[specifierType],

package/lib/PackagerRunner.js

@@ -11,6 +11,13 @@ function _assert() {
   };
   return data;
 }
+function _utils() {
+  const data = require("@atlaspack/utils");
+  _utils = function () {
+    return data;
+  };
+  return data;
+}
 function _logger() {
   const data = require("@atlaspack/logger");
   _logger = function () {
@@ -25,6 +32,13 @@ function _diagnostic() {
   };
   return data;
 }
+function _stream() {
+  const data = require("stream");
+  _stream = function () {
+    return data;
+  };
+  return data;
+}
 function _nullthrows() {
   const data = _interopRequireDefault(require("nullthrows"));
   _nullthrows = function () {
@@ -62,7 +76,13 @@ var _projectPath = require("./projectPath");
 var _InternalConfig = require("./InternalConfig");
 var _ConfigRequest = require("./requests/ConfigRequest");
 var _DevDepRequest = require("./requests/DevDepRequest");
-var _buildCache = require("./buildCache");
+function _buildCache() {
+  const data = require("@atlaspack/build-cache");
+  _buildCache = function () {
+    return data;
+  };
+  return data;
+}
 var _assetUtils = require("./assetUtils");
 var _utils2 = require("./utils");
 function _profiler() {
@@ -79,7 +99,7 @@ const BOUNDARY_LENGTH = _constants.HASH_REF_PREFIX.length + 32 - 1;
 
 // Packager/optimizer configs are not bundle-specific, so we only need to
 // load them once per build.
-const pluginConfigs = (0, _buildCache.createBuildCache)();
+const pluginConfigs = (0, _buildCache().createBuildCache)();
 class PackagerRunner {
   constructor({
     config,
@@ -300,7 +320,7 @@ class PackagerRunner {
     let bundle = _Bundle.NamedBundle.get(internalBundle, internalBundleGraph, this.options);
     let bundleGraph = new _BundleGraph.default(internalBundleGraph, _Bundle.NamedBundle.get.bind(_Bundle.NamedBundle), this.options);
     let optimizers = await this.config.getOptimizers(bundle.name, internalBundle.pipeline);
-    if (optimizers.length === 0) {
+    if (!optimizers.length) {
       return {
         type: bundle.type,
         contents,
@@ -433,13 +453,40 @@ class PackagerRunner {
     }
     return devDepHashes;
   }
+  async readFromCache(cacheKey) {
+    let contentKey = PackagerRunner.getContentKey(cacheKey);
+    let mapKey = PackagerRunner.getMapKey(cacheKey);
+    let isLargeBlob = await this.options.cache.hasLargeBlob(contentKey);
+    let contentExists = isLargeBlob || (await this.options.cache.has(contentKey));
+    if (!contentExists) {
+      return null;
+    }
+    let mapExists = await this.options.cache.has(mapKey);
+    return {
+      contents: isLargeBlob ? this.options.cache.getStream(contentKey) : (0, _utils().blobToStream)(await this.options.cache.getBlob(contentKey)),
+      map: mapExists ? (0, _utils().blobToStream)(await this.options.cache.getBlob(mapKey)) : null
+    };
+  }
   async writeToCache(cacheKeys, type, contents, map) {
     let size = 0;
     let hash;
     let hashReferences = [];
+    let isLargeBlob = false;
 
     // TODO: don't replace hash references in binary files??
-    if (typeof contents === 'string') {
+    if (contents instanceof _stream().Readable) {
+      isLargeBlob = true;
+      let boundaryStr = '';
+      let h = new (_rust().Hash)();
+      await this.options.cache.setStream(cacheKeys.content, (0, _utils().blobToStream)(contents).pipe(new (_utils().TapStream)(buf => {
+        let str = boundaryStr + buf.toString();
+        hashReferences = hashReferences.concat(str.match(_constants.HASH_REF_REGEX) ?? []);
+        size += buf.length;
+        h.writeBuffer(buf);
+        boundaryStr = str.slice(str.length - BOUNDARY_LENGTH);
+      })));
+      hash = h.finish();
+    } else if (typeof contents === 'string') {
       let buffer = Buffer.from(contents);
       size = buffer.byteLength;
       hash = (0, _rust().hashBuffer)(buffer);
@@ -459,7 +506,8 @@ class PackagerRunner {
       size,
       hash,
       hashReferences,
-      cacheKeys
+      cacheKeys,
+      isLargeBlob
     };
     await this.options.cache.set(cacheKeys.info, info);
     return info;

package/lib/RequestTracker.js

@@ -69,7 +69,13 @@ var _constants = require("./constants");
 var _projectPath = require("./projectPath");
 var _ReporterRunner = require("./ReporterRunner");
 var _ConfigRequest = require("./requests/ConfigRequest");
-var _serializer = require("./serializer");
+function _buildCache() {
+  const data = require("@atlaspack/build-cache");
+  _buildCache = function () {
+    return data;
+  };
+  return data;
+}
 var _utils2 = require("./utils");
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
@@ -749,7 +755,7 @@ class RequestTracker {
     } else if (node.resultCacheKey != null && ifMatch == null) {
       let key = node.resultCacheKey;
       (0, _assert().default)(this.options.cache.hasLargeBlob(key));
-      let cachedResult = (0, _serializer.deserialize)(await this.options.cache.getLargeBlob(key));
+      let cachedResult = (0, _buildCache().deserialize)(await this.options.cache.getLargeBlob(key));
       node.result = cachedResult;
       return cachedResult;
     }
@@ -927,7 +933,7 @@ class RequestTracker {
       if (signal !== null && signal !== void 0 && signal.aborted) {
         throw new Error('Serialization was aborted');
       }
-      await this.options.cache.setLargeBlob(key, (0, _serializer.serialize)(contents), signal ? {
+      await this.options.cache.setLargeBlob(key, (0, _buildCache().serialize)(contents), signal ? {
         signal: signal
       } : undefined);
       total += 1;
@@ -1046,7 +1052,7 @@ async function readAndDeserializeRequestGraph(cache, requestGraphKey, cacheKey)
   const getAndDeserialize = async key => {
     let buffer = await cache.getLargeBlob(key);
     bufferLength += Buffer.byteLength(buffer);
-    return (0, _serializer.deserialize)(buffer);
+    return (0, _buildCache().deserialize)(buffer);
   };
   let serializedRequestGraph = await getAndDeserialize(requestGraphKey);
   let nodePromises = serializedRequestGraph.nodeCountsPerBlob.map(async (nodesCount, i) => {

package/lib/SymbolPropagation.js

@@ -438,15 +438,7 @@ function propagateSymbolsDown(assetGraph, changedAssets, assetGroupsWithRemovedP
       wasNodeDirty = node.usedSymbolsDownDirty;
       node.usedSymbolsDownDirty = false;
     } else if (node.type === 'asset' && node.usedSymbolsDownDirty) {
-      visit(node, assetGraph.getIncomingDependencies(node.value).map(d => {
-        let dep = assetGraph.getNodeByContentKey(d.id);
-        (0, _assert().default)(dep && dep.type === 'dependency');
-        return dep;
-      }), outgoing.map(dep => {
-        let depNode = (0, _nullthrows().default)(assetGraph.getNode(dep));
-        (0, _assert().default)(depNode.type === 'dependency');
-        return depNode;
-      }));
+      visit(node, incomingDependencyNodesFromAsset(assetGraph, node.value), dependencyNodesFromIds(assetGraph, outgoing));
       node.usedSymbolsDownDirty = false;
     }
     for (let child of outgoing) {
@@ -494,23 +486,15 @@ function propagateSymbolsUp(assetGraph, changedAssets, changedDepsUsedSymbolsUpD
     let rootNodeId = (0, _nullthrows().default)(assetGraph.rootNodeId, 'A root node is required to traverse');
     const nodeVisitor = nodeId => {
       let node = (0, _nullthrows().default)(assetGraph.getNode(nodeId));
-      let outgoing = assetGraph.getNodeIdsConnectedFrom(nodeId);
-      for (let childId of outgoing) {
-        let child = (0, _nullthrows().default)(assetGraph.getNode(childId));
-        if (node.type === 'asset') {
-          (0, _assert().default)(child.type === 'dependency');
+      if (node.type === 'asset') {
+        let outgoing = outgoingDependencyNodesFromAsset(assetGraph, nodeId);
+        for (let child of outgoing) {
           if (child.usedSymbolsUpDirtyUp) {
             node.usedSymbolsUpDirty = true;
             child.usedSymbolsUpDirtyUp = false;
           }
         }
-      }
-      if (node.type === 'asset') {
-        let incoming = assetGraph.getIncomingDependencies(node.value).map(d => {
-          let n = assetGraph.getNodeByContentKey(d.id);
-          (0, _assert().default)(n && n.type === 'dependency');
-          return n;
-        });
+        let incoming = incomingDependencyNodesFromAsset(assetGraph, node.value);
         for (let dep of incoming) {
           if (dep.usedSymbolsUpDirtyDown) {
             dep.usedSymbolsUpDirtyDown = false;
@@ -518,11 +502,7 @@ function propagateSymbolsUp(assetGraph, changedAssets, changedDepsUsedSymbolsUpD
           }
         }
         if (node.usedSymbolsUpDirty) {
-          let e = visit(node, incoming, outgoing.map(depNodeId => {
-            let depNode = (0, _nullthrows().default)(assetGraph.getNode(depNodeId));
-            (0, _assert().default)(depNode.type === 'dependency');
-            return depNode;
-          }));
+          let e = visit(node, incoming, outgoing);
           if (e.length > 0) {
             node.usedSymbolsUpDirty = true;
             errors.set(nodeId, e);
@@ -548,22 +528,14 @@ function propagateSymbolsUp(assetGraph, changedAssets, changedDepsUsedSymbolsUpD
     let queuedNodeId = setPop(queue);
     let node = (0, _nullthrows().default)(assetGraph.getNode(queuedNodeId));
     if (node.type === 'asset') {
-      let incoming = assetGraph.getIncomingDependencies(node.value).map(dep => {
-        let depNode = assetGraph.getNodeByContentKey(dep.id);
-        (0, _assert().default)(depNode && depNode.type === 'dependency');
-        return depNode;
-      });
+      let incoming = incomingDependencyNodesFromAsset(assetGraph, node.value);
       for (let dep of incoming) {
         if (dep.usedSymbolsUpDirtyDown) {
           dep.usedSymbolsUpDirtyDown = false;
           node.usedSymbolsUpDirty = true;
         }
       }
-      let outgoing = assetGraph.getNodeIdsConnectedFrom(queuedNodeId).map(depNodeId => {
-        let depNode = (0, _nullthrows().default)(assetGraph.getNode(depNodeId));
-        (0, _assert().default)(depNode.type === 'dependency');
-        return depNode;
-      });
+      let outgoing = outgoingDependencyNodesFromAsset(assetGraph, queuedNodeId);
       for (let dep of outgoing) {
         if (dep.usedSymbolsUpDirtyUp) {
           node.usedSymbolsUpDirty = true;
@@ -622,4 +594,21 @@ function setPop(set) {
   let v = (0, _nullthrows().default)(set.values().next().value);
   set.delete(v);
   return v;
+}
+function outgoingDependencyNodesFromAsset(assetGraph, assetNode) {
+  return dependencyNodesFromIds(assetGraph, assetGraph.getNodeIdsConnectedFrom(assetNode));
+}
+function dependencyNodesFromIds(assetGraph, dependencyIds) {
+  return dependencyIds.map(depNodeId => {
+    let depNode = (0, _nullthrows().default)(assetGraph.getNode(depNodeId));
+    (0, _assert().default)(depNode.type === 'dependency');
+    return depNode;
+  });
+}
+function incomingDependencyNodesFromAsset(assetGraph, assetNodeValue) {
+  return assetGraph.getIncomingDependencies(assetNodeValue).map(d => {
+    let n = assetGraph.getNodeByContentKey(d.id);
+    (0, _assert().default)(n && n.type === 'dependency');
+    return n;
+  });
 }

package/lib/UncommittedAsset.js

@@ -39,7 +39,13 @@ function _rust() {
   };
   return data;
 }
-var _serializer = require("./serializer");
+function _buildCache() {
+  const data = require("@atlaspack/build-cache");
+  _buildCache = function () {
+    return data;
+  };
+  return data;
+}
 var _Dependency = require("./Dependency");
 var _Environment = require("./Environment");
 var _constants = require("./constants");
@@ -88,7 +94,7 @@ class UncommittedAsset {
 
     // Since we can only read from the stream once, compute the content length
     // and hash while it's being written to the cache.
-    await Promise.all([contentKey != null && this.commitContent(contentKey).then(s => (size = s.size, outputHash = s.hash)), this.mapBuffer != null && mapKey != null && this.options.cache.setBlob(mapKey, this.mapBuffer), astKey != null && this.options.cache.setBlob(astKey, (0, _serializer.serializeRaw)(this.ast))]);
+    await Promise.all([contentKey != null && this.commitContent(contentKey).then(s => (size = s.size, outputHash = s.hash)), this.mapBuffer != null && mapKey != null && this.options.cache.setBlob(mapKey, this.mapBuffer), astKey != null && this.options.cache.setBlob(astKey, (0, _buildCache().serializeRaw)(this.ast))]);
     this.value.contentKey = contentKey;
     this.value.mapKey = mapKey;
     this.value.astKey = astKey;

package/lib/assetUtils.js

@@ -42,7 +42,13 @@ function _utils() {
   return data;
 }
 var _utils2 = require("./utils");
-var _buildCache = require("./buildCache");
+function _buildCache() {
+  const data = require("@atlaspack/build-cache");
+  _buildCache = function () {
+    return data;
+  };
+  return data;
+}
 var _projectPath = require("./projectPath");
 function _rust() {
   const data = require("@atlaspack/rust");
@@ -62,7 +68,7 @@ function _profiler() {
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 function createAssetIdFromOptions(options) {
   return (0, _rust().createAssetId)({
-    env: options.env,
+    environmentId: options.env.id,
     filePath: options.filePath,
     code: options.code,
     pipeline: options.pipeline,
@@ -161,7 +167,7 @@ function getInvalidationId(invalidation) {
       throw new Error('Unknown invalidation type: ' + invalidation.type);
   }
 }
-const hashCache = (0, _buildCache.createBuildCache)();
+const hashCache = (0, _buildCache().createBuildCache)();
 async function getInvalidationHash(invalidations, options) {
   if (invalidations.length === 0) {
     return '';

package/lib/atlaspack-v3/fs.js

@@ -3,35 +3,37 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.toFileSystemV3 = toFileSystemV3;
+exports.FileSystemV3 = void 0;
 var _jsCallable = require("./jsCallable");
-// Move to @atlaspack/utils or a dedicated v3 / migration package later
-function toFileSystemV3(fs) {
-  return {
-    canonicalize: (0, _jsCallable.jsCallable)(path => fs.realpathSync(path)),
-    createDirectory: (0, _jsCallable.jsCallable)(path => fs.mkdirp(path)),
-    cwd: (0, _jsCallable.jsCallable)(() => fs.cwd()),
-    readFile: (0, _jsCallable.jsCallable)((path, encoding) => {
-      if (!encoding) {
-        // $FlowFixMe
-        return [...fs.readFileSync(path)];
-      } else {
-        return fs.readFileSync(path, encoding);
-      }
-    }),
-    isFile: path => {
-      try {
-        return fs.statSync(path).isFile();
-      } catch {
-        return false;
-      }
-    },
-    isDir: path => {
-      try {
-        return fs.statSync(path).isDirectory();
-      } catch {
-        return false;
-      }
+class FileSystemV3 {
+  #fs;
+  constructor(fs) {
+    this.#fs = fs;
+  }
+  canonicalize = (0, _jsCallable.jsCallable)(path => this.#fs.realpathSync(path));
+  createDirectory = (0, _jsCallable.jsCallable)(path => this.#fs.mkdirp(path));
+  cwd = (0, _jsCallable.jsCallable)(() => this.#fs.cwd());
+  isFile = path => {
+    try {
+      return this.#fs.statSync(path).isFile();
+    } catch {
+      return false;
     }
   };
-}
+  isDir = path => {
+    try {
+      return this.#fs.statSync(path).isDirectory();
+    } catch {
+      return false;
+    }
+  };
+  readFile = (0, _jsCallable.jsCallable)((path, encoding) => {
+    if (!encoding) {
+      // $FlowFixMe
+      return [...this.#fs.readFileSync(path)];
+    } else {
+      return this.#fs.readFileSync(path, encoding);
+    }
+  });
+}
+exports.FileSystemV3 = FileSystemV3;

package/lib/atlaspack-v3/index.js

@@ -9,10 +9,10 @@ Object.defineProperty(exports, "AtlaspackV3", {
     return _AtlaspackV.AtlaspackV3;
   }
 });
-Object.defineProperty(exports, "toFileSystemV3", {
+Object.defineProperty(exports, "FileSystemV3", {
   enumerable: true,
   get: function () {
-    return _fs.toFileSystemV3;
+    return _fs.FileSystemV3;
   }
 });
 var _fs = require("./fs");

package/lib/atlaspack-v3/jsCallable.js

@@ -5,11 +5,16 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.jsCallable = jsCallable;
 function jsCallable(fn) {
-  return (...args) => {
+  return async (...args) => {
     try {
-      return fn(...args);
-    } catch (err) {
-      return err;
+      // $FlowFixMe "await" is needed to log the error from
+      // Promises and is safe to use on non-promises
+      return await fn(...args);
+      // $FlowFixMe
+    } catch (error) {
+      // eslint-disable-next-line no-console
+      console.error(error);
+      throw error;
     }
   };
 }