@atlaspack/core 2.12.1-dev.3401 → 2.12.1-dev.3443

package/src/applyRuntimes.js

@@ -32,6 +32,7 @@ import createAssetGraphRequest from './requests/AssetGraphRequest';
 import {createDevDependency, runDevDepRequest} from './requests/DevDepRequest';
 import {toProjectPath, fromProjectPathRelative} from './projectPath';
 import {tracer, PluginTracer} from '@atlaspack/profiler';
+import {DefaultMap} from '@atlaspack/utils';
 
 type RuntimeConnection = {|
   bundle: InternalBundle,
@@ -60,6 +61,14 @@ function nameRuntimeBundle(
   bundle.displayName = name.replace(hashReference, '[hash]');
 }
 
+/**
+ * The applyRuntimes function is responsible for generating all the runtimes
+ * (assets created during the build that don't actually exist on disk) and then
+ * linking them into the bundle graph.
+ *
+ * Introduction of manifest bundles: https://github.com/parcel-bundler/parcel/pull/8837
+ * Introduction of reverse topology: https://github.com/parcel-bundler/parcel/pull/8981
+ */
 export default async function applyRuntimes<TResult: RequestResult>({
   bundleGraph,
   config,
@@ -82,11 +91,20 @@ export default async function applyRuntimes<TResult: RequestResult>({
   configs: Map<string, Config>,
 |}): Promise<Map<string, Asset>> {
   let runtimes = await config.getRuntimes();
-  let connections: Array<RuntimeConnection> = [];
 
-  // As manifest bundles may be added during runtimes we process them in reverse topological
-  // sort order. This allows bundles to be added to their bundle groups before they are referenced
-  // by other bundle groups by loader runtimes
+  /**
+   * Usually, the assets returned from a runtime will go in the same bundle. It is
+   * possible though, for a runtime to return an asset with a `parallel` priority,
+   * which allows it to be moved to a separate bundle. In practice, this is
+   * usually used to generate application manifest files.
+   *
+   * When adding a manifest bundle (a whole new separate bundle) during a runtime,
+   * it needs to be added to a bundle group which will be potentially referenced
+   * by another bundle group. To avoid trying to reference a manifest entry which
+   * hasn't been created yet, we process the bundles from the bottom up (topological
+   * order), so that children will always be available when parents try to reference
+   * them.
+   */
   let bundles = [];
   bundleGraph.traverseBundles({
     exit(bundle) {
@@ -94,6 +112,8 @@ export default async function applyRuntimes<TResult: RequestResult>({
     },
   });
 
+  let connectionMap = new DefaultMap(() => []);
+
   for (let bundle of bundles) {
     for (let runtime of runtimes) {
       let measurement;
@@ -146,6 +166,11 @@ export default async function applyRuntimes<TResult: RequestResult>({
 
             let connectionBundle = bundle;
 
+            /**
+             * If a runtime asset is marked with a priority of `parallel` this
+             * means we need to create a new bundle for the asset and add it to
+             * all the same bundle groups.
+             */
             if (priority === 'parallel' && !bundle.needsStableName) {
               let bundleGroups =
                 bundleGraph.getBundleGroupsContainingBundle(bundle);
@@ -169,10 +194,12 @@ export default async function applyRuntimes<TResult: RequestResult>({
               }
               bundleGraph.createBundleReference(bundle, connectionBundle);
 
+              // Ensure we name the bundle now as all other bundles have already
+              // been named as this point.
               nameRuntimeBundle(connectionBundle, bundle);
             }
 
-            connections.push({
+            connectionMap.get(connectionBundle).push({
               bundle: connectionBundle,
               assetGroup,
               dependency,
@@ -192,8 +219,28 @@ export default async function applyRuntimes<TResult: RequestResult>({
     }
   }
 
-  // Correct connection order after generating runtimes in reverse order
-  connections.reverse();
+  /**
+   * When merging the connections in to the bundle graph, the topological
+   * order can create module not found errors in some situations, often when HMR
+   * is enabled. To fix this, we put the connections into DFS order.
+   *
+   * Note: While DFS order seems to be the most reliable order to process the
+   * connections, this is likely due to it being close to the order that the bundles were
+   * inserted into the graph. There is a known issue where runtime assets marked
+   * as `isEntry` can create scenarios where there is no correct load order that
+   * won't error, as the entry runtime assets are added to many bundles in a
+   * single bundle group but their dependencies are not.
+   *
+   * This issue is almost exclusive to HMR scenarios as the two HMR runtime
+   * plugins (@atlaspack/runtime-browser-hmr and @atlaspack/runtime-react-refresh)
+   * are the only known cases where a runtime asset is marked as `isEntry`.
+   */
+  let connections: Array<RuntimeConnection> = [];
+  bundleGraph.traverseBundles({
+    enter(bundle) {
+      connections.push(...connectionMap.get(bundle));
+    },
+  });
 
   // Add dev deps for runtime plugins AFTER running them, to account for lazy require().
   for (let runtime of runtimes) {
@@ -214,23 +261,31 @@ export default async function applyRuntimes<TResult: RequestResult>({
     await runDevDepRequest(api, devDepRequest);
   }
 
+  // Create a new AssetGraph from the generated runtime assets which also runs
+  // transforms and resolves all dependencies.
   let {assetGraph: runtimesAssetGraph, changedAssets} =
     await reconcileNewRuntimes(api, connections, optionsRef);
 
-  let runtimesGraph = InternalBundleGraph.fromAssetGraph(
+  // Convert the runtime AssetGraph into a BundleGraph, this includes assigning
+  // the assets their public ids
+  let runtimesBundleGraph = InternalBundleGraph.fromAssetGraph(
     runtimesAssetGraph,
     options.mode === 'production',
     bundleGraph._publicIdByAssetId,
     bundleGraph._assetPublicIds,
   );
 
-  // Merge the runtimes graph into the main bundle graph.
-  bundleGraph.merge(runtimesGraph);
-  for (let [assetId, publicId] of runtimesGraph._publicIdByAssetId) {
+  // Merge the runtimes bundle graph into the main bundle graph.
+  bundleGraph.merge(runtimesBundleGraph);
+
+  // Add the public id mappings from the runtumes bundlegraph to the main bundle graph
+  for (let [assetId, publicId] of runtimesBundleGraph._publicIdByAssetId) {
     bundleGraph._publicIdByAssetId.set(assetId, publicId);
     bundleGraph._assetPublicIds.add(publicId);
   }
 
+  // Connect each of the generated runtime assets to bundles in the main bundle
+  // graph. This is like a mini-bundling algorithm for runtime assets.
   for (let {bundle, assetGroup, dependency, isEntry} of connections) {
     let assetGroupNode = nodeFromAssetGroup(assetGroup);
     let assetGroupAssetNodeIds = runtimesAssetGraph.getNodeIdsConnectedFrom(
@@ -241,6 +296,8 @@ export default async function applyRuntimes<TResult: RequestResult>({
     let runtimeNode = nullthrows(runtimesAssetGraph.getNode(runtimeNodeId));
     invariant(runtimeNode.type === 'asset');
 
+    // Find the asset that the runtime asset should be connected from by resolving
+    // it's dependency.
     let resolution =
       dependency &&
       bundleGraph.getResolvedAsset(
@@ -248,20 +305,25 @@ export default async function applyRuntimes<TResult: RequestResult>({
         bundle,
       );
 
-    let runtimesGraphRuntimeNodeId = runtimesGraph._graph.getNodeIdByContentKey(
-      runtimeNode.id,
-    );
+    // Walk all the dependencies of the runtime assets and check if they are
+    // already reachable from the bundle that the runtime asset is assigned to.
+    // If so, we add them to `duplicatedContentKeys` to be skipped when assigning
+    // assets to bundles.
+    let runtimesBundleGraphRuntimeNodeId =
+      runtimesBundleGraph._graph.getNodeIdByContentKey(runtimeNode.id);
     let duplicatedContentKeys: Set<ContentKey> = new Set();
-    runtimesGraph._graph.traverse((nodeId, _, actions) => {
-      let node = nullthrows(runtimesGraph._graph.getNode(nodeId));
+    runtimesBundleGraph._graph.traverse((nodeId, _, actions) => {
+      let node = nullthrows(runtimesBundleGraph._graph.getNode(nodeId));
       if (node.type !== 'dependency') {
         return;
       }
 
-      let assets = runtimesGraph._graph
+      let assets = runtimesBundleGraph._graph
         .getNodeIdsConnectedFrom(nodeId)
         .map(assetNodeId => {
-          let assetNode = nullthrows(runtimesGraph._graph.getNode(assetNodeId));
+          let assetNode = nullthrows(
+            runtimesBundleGraph._graph.getNode(assetNodeId),
+          );
           invariant(assetNode.type === 'asset');
           return assetNode.value;
         });
@@ -275,15 +337,17 @@ export default async function applyRuntimes<TResult: RequestResult>({
           actions.skipChildren();
         }
       }
-    }, runtimesGraphRuntimeNodeId);
+    }, runtimesBundleGraphRuntimeNodeId);
 
     let bundleNodeId = bundleGraph._graph.getNodeIdByContentKey(bundle.id);
     let bundleGraphRuntimeNodeId = bundleGraph._graph.getNodeIdByContentKey(
       runtimeNode.id,
     ); // the node id is not constant between graphs
 
-    runtimesGraph._graph.traverse((nodeId, _, actions) => {
-      let node = nullthrows(runtimesGraph._graph.getNode(nodeId));
+    // Assign the runtime assets and all of it's depepdencies to the bundle unless
+    // we have detected it as already being reachable from this bundle via `duplicatedContentKeys`.
+    runtimesBundleGraph._graph.traverse((nodeId, _, actions) => {
+      let node = nullthrows(runtimesBundleGraph._graph.getNode(nodeId));
       if (node.type === 'asset' || node.type === 'dependency') {
         if (duplicatedContentKeys.has(node.id)) {
           actions.skipChildren();
@@ -299,7 +363,7 @@ export default async function applyRuntimes<TResult: RequestResult>({
           bundleGraphEdgeTypes.contains,
         );
       }
-    }, runtimesGraphRuntimeNodeId);
+    }, runtimesBundleGraphRuntimeNodeId);
 
     if (isEntry) {
       bundleGraph._graph.addEdge(bundleNodeId, bundleGraphRuntimeNodeId);

package/src/assetUtils.js

@@ -36,14 +36,14 @@ import {
   fromProjectPath,
   fromProjectPathRelative,
 } from './projectPath';
-import {hashString} from '@atlaspack/rust';
+import {hashString, createAssetId as createAssetIdRust} from '@atlaspack/rust';
 import {BundleBehavior as BundleBehaviorMap} from './types';
 import {PluginTracer} from '@atlaspack/profiler';
 
-type AssetOptions = {|
+export type AssetOptions = {|
   id?: string,
   committed?: boolean,
-  idBase?: ?string,
+  code: string | void | null,
   filePath: ProjectPath,
   query?: ?string,
   type: string,
@@ -69,22 +69,15 @@ type AssetOptions = {|
 |};
 
 export function createAssetIdFromOptions(options: AssetOptions): string {
-  let uniqueKey = options.uniqueKey ?? '';
-  let idBase =
-    options.idBase != null
-      ? options.idBase
-      : fromProjectPathRelative(options.filePath);
-
-  return hashString(
-    idBase +
-      options.type +
-      options.env.id +
-      uniqueKey +
-      ':' +
-      (options.pipeline ?? '') +
-      ':' +
-      (options.query ?? ''),
-  );
+  return createAssetIdRust({
+    env: options.env,
+    filePath: options.filePath,
+    code: options.code,
+    pipeline: options.pipeline,
+    query: options.query,
+    uniqueKey: options.uniqueKey,
+    fileType: options.type,
+  });
 }
 
 export function createAsset(

package/src/atlaspack-v3/AtlaspackV3.js

@@ -24,6 +24,14 @@ export class AtlaspackV3 {
     threads,
     ...options
   }: AtlaspackV3Options) {
+    options.logLevel = options.logLevel || 'error';
+    options.defaultTargetOptions = options.defaultTargetOptions || {};
+    // $FlowFixMe "engines" are readonly
+    options.defaultTargetOptions.engines = options.defaultTargetOptions
+      .engines || {
+      browsers: [],
+    };
+
     this._internal = new AtlaspackNapi({
       fs,
       nodeWorkers,
@@ -33,17 +41,6 @@ export class AtlaspackV3 {
     });
   }
 
-  async build(): Promise<any> {
-    const [workers, registerWorker] = this.#createWorkers();
-
-    let result = await this._internal.build({
-      registerWorker,
-    });
-
-    for (const worker of workers) worker.terminate();
-    return result;
-  }
-
   async buildAssetGraph(): Promise<any> {
     const [workers, registerWorker] = this.#createWorkers();
 

package/src/atlaspack-v3/fs.js

@@ -15,9 +15,14 @@ export function toFileSystemV3(fs: ClassicFileSystem): FileSystem {
     canonicalize: jsCallable((path: FilePath) => fs.realpathSync(path)),
     createDirectory: jsCallable((path: FilePath) => fs.mkdirp(path)),
     cwd: jsCallable(() => fs.cwd()),
-    readFile: jsCallable((path: string, encoding?: Encoding) =>
-      fs.readFileSync(path, encoding ?? 'utf8'),
-    ),
+    readFile: jsCallable((path: string, encoding?: Encoding) => {
+      if (!encoding) {
+        // $FlowFixMe
+        return [...fs.readFileSync(path)];
+      } else {
+        return fs.readFileSync(path, encoding);
+      }
+    }),
     isFile: (path: string) => {
       try {
         return fs.statSync(path).isFile();

package/src/atlaspack-v3/jsCallable.js

@@ -1,5 +1,9 @@
 // @flow strict-local
 
+export type JsCallable<Args: $ReadOnlyArray<mixed>, Return> = (
+  ...Args
+) => Return;
+
 export function jsCallable<Args: $ReadOnlyArray<mixed>, Return>(
   fn: (...Args) => Return,
 ): (...Args) => Return {

package/src/atlaspack-v3/worker/compat.js

@@ -0,0 +1,82 @@
+// @flow
+import type {PluginOptions, BundleBehavior, AST} from '@atlaspack/types';
+
+import Environment from '../../public/Environment';
+import {
+  BundleBehaviorNames,
+  BundleBehavior as BundleBehaviorMap,
+} from '../../types';
+
+export type InnerAsset = {|
+  id: string,
+  filePath: string,
+  code: Array<number>,
+  type: string,
+  bundleBehavior: number | null,
+  env: any,
+|};
+
+export class AssetCompat {
+  _inner: InnerAsset;
+  _ast: ?AST;
+  // TODO: Type properly
+  _dependencies: Array<any>;
+
+  constructor(_inner: InnerAsset, options: PluginOptions) {
+    this._inner = _inner;
+    // $FlowFixMe This isn't correct to flow but is enough to satisfy the runtime checks
+    this.env = new Environment(_inner.env, options);
+    this._dependencies = [];
+  }
+
+  get id(): string {
+    return this._inner.id;
+  }
+
+  get filePath(): string {
+    return this._inner.filePath;
+  }
+
+  get type(): string {
+    return this._inner.type;
+  }
+  set type(value: string) {
+    this._inner.type = value;
+  }
+
+  get bundleBehavior(): ?BundleBehavior {
+    let bundleBehavior = this._inner.bundleBehavior;
+    return bundleBehavior == null ? null : BundleBehaviorNames[bundleBehavior];
+  }
+  set bundleBehavior(bundleBehavior: ?BundleBehavior): void {
+    this._inner.bundleBehavior = bundleBehavior
+      ? BundleBehaviorMap[bundleBehavior]
+      : null;
+  }
+
+  getCode(): string {
+    return Buffer.from(this._inner.code).toString();
+  }
+  setCode(code: string) {
+    this._inner.code = Array.from(new TextEncoder().encode(code));
+  }
+
+  getAST(): ?AST {
+    return this._ast;
+  }
+  setAST(ast: AST) {
+    this._ast = ast;
+  }
+
+  addDependency(): string {
+    throw new Error(
+      '[V3] Unimplemented: Asset.addDependency not yet implemented',
+    );
+  }
+
+  addURLDependency(): string {
+    throw new Error(
+      '[V3] Unimplemented: Asset.addURLDependency not yet implemented',
+    );
+  }
+}

package/src/atlaspack-v3/worker/worker.js

@@ -1,14 +1,221 @@
 // @flow
+import assert from 'assert';
 import * as napi from '@atlaspack/rust';
+import type {
+  Resolver,
+  Transformer,
+  PluginOptions,
+  Dependency,
+  FilePath,
+} from '@atlaspack/types';
 import {workerData} from 'worker_threads';
-import type {ResolverNapi} from '../plugins/Resolver';
+import * as module from 'module';
+
+import {AssetCompat} from './compat';
+import type {InnerAsset} from './compat';
+import {jsCallable} from '../jsCallable';
+import type {JsCallable} from '../jsCallable';
+
+const CONFIG = Symbol.for('parcel-plugin-config');
 
 export class AtlaspackWorker {
-  #resolvers: Map<string, ResolverNapi>;
+  #resolvers: Map<string, Resolver<*>>;
+
+  constructor() {
+    this.#resolvers = new Map();
+  }
+
+  loadPlugin: JsCallable<[LoadPluginOptions], Promise<void>> = jsCallable(
+    async ({kind, specifier, resolveFrom}) => {
+      let customRequire = module.createRequire(resolveFrom);
+      let resolvedPath = customRequire.resolve(specifier);
+      // $FlowFixMe
+      let resolvedModule = await import(resolvedPath);
+
+      let instance = undefined;
+      if (resolvedModule.default && resolvedModule.default[CONFIG]) {
+        instance = resolvedModule.default[CONFIG];
+      } else if (
+        resolvedModule.default &&
+        resolvedModule.default.default[CONFIG]
+      ) {
+        instance = resolvedModule.default.default[CONFIG];
+      } else {
+        throw new Error(
+          `Plugin could not be resolved\n\t${kind}\n\t${resolveFrom}\n\t${specifier}`,
+        );
+      }
+
+      switch (kind) {
+        case 'resolver':
+          this.#resolvers.set(specifier, instance);
+          break;
+      }
+    },
+  );
+
+  runResolverResolve: JsCallable<
+    [RunResolverResolveOptions],
+    Promise<RunResolverResolveResult>,
+  > = jsCallable(async ({key, dependency, specifier}) => {
+    const resolver = this.#resolvers.get(key);
+    if (!resolver) {
+      throw new Error(`Resolver not found: ${key}`);
+    }
+
+    const result = await resolver.resolve({
+      specifier,
+      dependency,
+      get options() {
+        throw new Error('TODO: Resolver.resolve.options');
+      },
+      get logger() {
+        throw new Error('TODO: Resolver.resolve.logger');
+      },
+      get tracer() {
+        throw new Error('TODO: Resolver.resolve.tracer');
+      },
+      get pipeline() {
+        throw new Error('TODO: Resolver.resolve.pipeline');
+      },
+      get config() {
+        throw new Error('TODO: Resolver.resolve.config');
+      },
+    });
+
+    if (!result) {
+      return {
+        invalidations: [],
+        resolution: {type: 'unresolved'},
+      };
+    }
+
+    return {
+      invalidations: [],
+      resolution: {
+        type: 'resolved',
+        filePath: result.filePath || '',
+        canDefer: result.canDefer || false,
+        sideEffects: result.sideEffects || false,
+        code: result.code || undefined,
+        meta: result.meta || undefined,
+        pipeline: result.pipeline || undefined,
+        priority: result.priority && PriorityMap[result.priority],
+        query: result.query && result.query.toString(),
+      },
+    };
+  });
 
   ping() {
     // console.log('Hi');
   }
+
+  async runTransformer({
+    resolveFrom,
+    specifier,
+    options,
+    asset,
+  }: {|
+    resolveFrom: string,
+    specifier: string,
+    options: PluginOptions,
+    asset: InnerAsset,
+  |}): any {
+    const customRequire = module.createRequire(resolveFrom);
+    const resolvedPath = customRequire.resolve(specifier);
+    // $FlowFixMe
+    const transformerModule = await import(resolvedPath);
+    const transformer: Transformer<*> =
+      transformerModule.default.default[CONFIG];
+
+    let assetCompat = new AssetCompat(asset, options);
+
+    try {
+      if (transformer.parse) {
+        // $FlowFixMe
+        const ast = await transformer.parse({asset: assetCompat}); // missing "config"
+        // $FlowFixMe
+        assetCompat.setAST(ast);
+      }
+
+      // $FlowFixMe
+      const result = await transformer.transform({
+        // $FlowFixMe
+        asset: assetCompat,
+        options,
+        config: null,
+      });
+
+      if (transformer.generate) {
+        // $FlowFixMe
+        let output = await transformer.generate({
+          // $FlowFixMe
+          asset: assetCompat,
+          // $FlowFixMe
+          ast: assetCompat.getAST(),
+        });
+        // $FlowFixMe
+        assetCompat.setCode(output.content);
+      }
+
+      assert(
+        result.length === 1,
+        '[V3] Unimplemented: Multiple asset return from Node transformer',
+      );
+      assert(
+        result[0] === assetCompat,
+        '[V3] Unimplemented: New asset returned from Node transformer',
+      );
+
+      return {
+        asset,
+        dependencies: assetCompat._dependencies,
+      };
+    } catch (e) {
+      // TODO: Improve error logging from JS plugins. Without this you currently
+      // only see the error message, no stack trace.
+      // eslint-disable-next-line no-console
+      console.error(e);
+      throw e;
+    }
+  }
 }
 
 napi.registerWorker(workerData.tx_worker, new AtlaspackWorker());
+
+type LoadPluginOptions = {|
+  kind: 'resolver',
+  specifier: string,
+  resolveFrom: string,
+|};
+
+type RunResolverResolveOptions = {|
+  key: string,
+  dependency: Dependency,
+  specifier: FilePath,
+|};
+
+type RunResolverResolveResult = {|
+  invalidations: Array<*>,
+  resolution:
+    | {|type: 'unresolved'|}
+    | {|type: 'excluded'|}
+    | {|
+        type: 'resolved',
+        canDefer: boolean,
+        filePath: string,
+        sideEffects: boolean,
+        code?: string,
+        meta?: mixed,
+        pipeline?: string,
+        priority?: number,
+        query?: string,
+      |},
+|};
+
+const PriorityMap = {
+  sync: 0,
+  parallel: 1,
+  lazy: 2,
+  conditional: 3,
+};