@atlaspack/core 2.12.1-canary.3354

package/src/applyRuntimes.js

@@ -0,0 +1,341 @@
+// @flow strict-local
+
+import type {ContentKey} from '@atlaspack/graph';
+import type {Dependency, NamedBundle as INamedBundle} from '@atlaspack/types';
+import type {SharedReference} from '@atlaspack/workers';
+import type {
+  Asset,
+  AssetGroup,
+  Bundle as InternalBundle,
+  Config,
+  DevDepRequest,
+  AtlaspackOptions,
+} from './types';
+import type AtlaspackConfig from './AtlaspackConfig';
+import type PluginOptions from './public/PluginOptions';
+import type {RequestResult, RunAPI} from './RequestTracker';
+
+import path from 'path';
+import assert from 'assert';
+import invariant from 'assert';
+import nullthrows from 'nullthrows';
+import {nodeFromAssetGroup} from './AssetGraph';
+import BundleGraph from './public/BundleGraph';
+import InternalBundleGraph, {bundleGraphEdgeTypes} from './BundleGraph';
+import {NamedBundle} from './public/Bundle';
+import {PluginLogger} from '@atlaspack/logger';
+import {hashString} from '@atlaspack/rust';
+import ThrowableDiagnostic, {errorToDiagnostic} from '@atlaspack/diagnostic';
+import {dependencyToInternalDependency} from './public/Dependency';
+import {mergeEnvironments} from './Environment';
+import createAssetGraphRequest from './requests/AssetGraphRequest';
+import {createDevDependency, runDevDepRequest} from './requests/DevDepRequest';
+import {toProjectPath, fromProjectPathRelative} from './projectPath';
+import {tracer, PluginTracer} from '@atlaspack/profiler';
+
+type RuntimeConnection = {|
+  bundle: InternalBundle,
+  assetGroup: AssetGroup,
+  dependency: ?Dependency,
+  isEntry: ?boolean,
+|};
+
+function nameRuntimeBundle(
+  bundle: InternalBundle,
+  siblingBundle: InternalBundle,
+) {
+  // We don't run custom namers on runtime bundles as the runtime assumes that they are
+  // located at the same nesting level as their owning bundle. Custom naming could
+  // be added in future as long as the custom name is validated.
+  let {hashReference} = bundle;
+
+  let name = nullthrows(siblingBundle.name)
+    // Remove the existing hash from standard file patterns
+    // e.g. 'main.[hash].js' -> 'main.js' or 'main~[hash].js' -> 'main.js'
+    .replace(new RegExp(`[\\.~\\-_]?${siblingBundle.hashReference}`), '')
+    // Ensure the file ends with 'runtime.[hash].js'
+    .replace(`.${bundle.type}`, `.runtime.${hashReference}.${bundle.type}`);
+
+  bundle.name = name;
+  bundle.displayName = name.replace(hashReference, '[hash]');
+}
+
+export default async function applyRuntimes<TResult: RequestResult>({
+  bundleGraph,
+  config,
+  options,
+  pluginOptions,
+  api,
+  optionsRef,
+  previousDevDeps,
+  devDepRequests,
+  configs,
+}: {|
+  bundleGraph: InternalBundleGraph,
+  config: AtlaspackConfig,
+  options: AtlaspackOptions,
+  optionsRef: SharedReference,
+  pluginOptions: PluginOptions,
+  api: RunAPI<TResult>,
+  previousDevDeps: Map<string, string>,
+  devDepRequests: Map<string, DevDepRequest>,
+  configs: Map<string, Config>,
+|}): Promise<Map<string, Asset>> {
+  let runtimes = await config.getRuntimes();
+  let connections: Array<RuntimeConnection> = [];
+
+  // As manifest bundles may be added during runtimes we process them in reverse topological
+  // sort order. This allows bundles to be added to their bundle groups before they are referenced
+  // by other bundle groups by loader runtimes
+  let bundles = [];
+  bundleGraph.traverseBundles({
+    exit(bundle) {
+      bundles.push(bundle);
+    },
+  });
+
+  for (let bundle of bundles) {
+    for (let runtime of runtimes) {
+      let measurement;
+      try {
+        const namedBundle = NamedBundle.get(bundle, bundleGraph, options);
+        measurement = tracer.createMeasurement(
+          runtime.name,
+          'applyRuntime',
+          namedBundle.displayName,
+        );
+        let applied = await runtime.plugin.apply({
+          bundle: namedBundle,
+          bundleGraph: new BundleGraph<INamedBundle>(
+            bundleGraph,
+            NamedBundle.get.bind(NamedBundle),
+            options,
+          ),
+          config: configs.get(runtime.name)?.result,
+          options: pluginOptions,
+          logger: new PluginLogger({origin: runtime.name}),
+          tracer: new PluginTracer({
+            origin: runtime.name,
+            category: 'applyRuntime',
+          }),
+        });
+
+        if (applied) {
+          let runtimeAssets = Array.isArray(applied) ? applied : [applied];
+          for (let {
+            code,
+            dependency,
+            filePath,
+            isEntry,
+            env,
+            priority,
+          } of runtimeAssets) {
+            let sourceName = path.join(
+              path.dirname(filePath),
+              `runtime-${hashString(code)}.${bundle.type}`,
+            );
+
+            let assetGroup = {
+              code,
+              filePath: toProjectPath(options.projectRoot, sourceName),
+              env: mergeEnvironments(options.projectRoot, bundle.env, env),
+              // Runtime assets should be considered source, as they should be
+              // e.g. compiled to run in the target environment
+              isSource: true,
+            };
+
+            let connectionBundle = bundle;
+
+            if (priority === 'parallel' && !bundle.needsStableName) {
+              let bundleGroups =
+                bundleGraph.getBundleGroupsContainingBundle(bundle);
+
+              connectionBundle = nullthrows(
+                bundleGraph.createBundle({
+                  type: bundle.type,
+                  needsStableName: false,
+                  env: bundle.env,
+                  target: bundle.target,
+                  uniqueKey: 'runtime-manifest:' + bundle.id,
+                  shouldContentHash: options.shouldContentHash,
+                }),
+              );
+
+              for (let bundleGroup of bundleGroups) {
+                bundleGraph.addBundleToBundleGroup(
+                  connectionBundle,
+                  bundleGroup,
+                );
+              }
+              bundleGraph.createBundleReference(bundle, connectionBundle);
+
+              nameRuntimeBundle(connectionBundle, bundle);
+            }
+
+            connections.push({
+              bundle: connectionBundle,
+              assetGroup,
+              dependency,
+              isEntry,
+            });
+          }
+        }
+      } catch (e) {
+        throw new ThrowableDiagnostic({
+          diagnostic: errorToDiagnostic(e, {
+            origin: runtime.name,
+          }),
+        });
+      } finally {
+        measurement && measurement.end();
+      }
+    }
+  }
+
+  // Correct connection order after generating runtimes in reverse order
+  connections.reverse();
+
+  // Add dev deps for runtime plugins AFTER running them, to account for lazy require().
+  for (let runtime of runtimes) {
+    let devDepRequest = await createDevDependency(
+      {
+        specifier: runtime.name,
+        resolveFrom: runtime.resolveFrom,
+      },
+      previousDevDeps,
+      options,
+    );
+    devDepRequests.set(
+      `${devDepRequest.specifier}:${fromProjectPathRelative(
+        devDepRequest.resolveFrom,
+      )}`,
+      devDepRequest,
+    );
+    await runDevDepRequest(api, devDepRequest);
+  }
+
+  let {assetGraph: runtimesAssetGraph, changedAssets} =
+    await reconcileNewRuntimes(api, connections, optionsRef);
+
+  let runtimesGraph = InternalBundleGraph.fromAssetGraph(
+    runtimesAssetGraph,
+    options.mode === 'production',
+    bundleGraph._publicIdByAssetId,
+    bundleGraph._assetPublicIds,
+  );
+
+  // Merge the runtimes graph into the main bundle graph.
+  bundleGraph.merge(runtimesGraph);
+  for (let [assetId, publicId] of runtimesGraph._publicIdByAssetId) {
+    bundleGraph._publicIdByAssetId.set(assetId, publicId);
+    bundleGraph._assetPublicIds.add(publicId);
+  }
+
+  for (let {bundle, assetGroup, dependency, isEntry} of connections) {
+    let assetGroupNode = nodeFromAssetGroup(assetGroup);
+    let assetGroupAssetNodeIds = runtimesAssetGraph.getNodeIdsConnectedFrom(
+      runtimesAssetGraph.getNodeIdByContentKey(assetGroupNode.id),
+    );
+    invariant(assetGroupAssetNodeIds.length === 1);
+    let runtimeNodeId = assetGroupAssetNodeIds[0];
+    let runtimeNode = nullthrows(runtimesAssetGraph.getNode(runtimeNodeId));
+    invariant(runtimeNode.type === 'asset');
+
+    let resolution =
+      dependency &&
+      bundleGraph.getResolvedAsset(
+        dependencyToInternalDependency(dependency),
+        bundle,
+      );
+
+    let runtimesGraphRuntimeNodeId = runtimesGraph._graph.getNodeIdByContentKey(
+      runtimeNode.id,
+    );
+    let duplicatedContentKeys: Set<ContentKey> = new Set();
+    runtimesGraph._graph.traverse((nodeId, _, actions) => {
+      let node = nullthrows(runtimesGraph._graph.getNode(nodeId));
+      if (node.type !== 'dependency') {
+        return;
+      }
+
+      let assets = runtimesGraph._graph
+        .getNodeIdsConnectedFrom(nodeId)
+        .map(assetNodeId => {
+          let assetNode = nullthrows(runtimesGraph._graph.getNode(assetNodeId));
+          invariant(assetNode.type === 'asset');
+          return assetNode.value;
+        });
+
+      for (let asset of assets) {
+        if (
+          bundleGraph.isAssetReachableFromBundle(asset, bundle) ||
+          resolution?.id === asset.id
+        ) {
+          duplicatedContentKeys.add(asset.id);
+          actions.skipChildren();
+        }
+      }
+    }, runtimesGraphRuntimeNodeId);
+
+    let bundleNodeId = bundleGraph._graph.getNodeIdByContentKey(bundle.id);
+    let bundleGraphRuntimeNodeId = bundleGraph._graph.getNodeIdByContentKey(
+      runtimeNode.id,
+    ); // the node id is not constant between graphs
+
+    runtimesGraph._graph.traverse((nodeId, _, actions) => {
+      let node = nullthrows(runtimesGraph._graph.getNode(nodeId));
+      if (node.type === 'asset' || node.type === 'dependency') {
+        if (duplicatedContentKeys.has(node.id)) {
+          actions.skipChildren();
+          return;
+        }
+
+        const bundleGraphNodeId = bundleGraph._graph.getNodeIdByContentKey(
+          node.id,
+        ); // the node id is not constant between graphs
+        bundleGraph._graph.addEdge(
+          bundleNodeId,
+          bundleGraphNodeId,
+          bundleGraphEdgeTypes.contains,
+        );
+      }
+    }, runtimesGraphRuntimeNodeId);
+
+    if (isEntry) {
+      bundleGraph._graph.addEdge(bundleNodeId, bundleGraphRuntimeNodeId);
+      bundle.entryAssetIds.unshift(runtimeNode.id);
+    }
+
+    if (dependency == null) {
+      // Verify this asset won't become an island
+      assert(
+        bundleGraph._graph.getNodeIdsConnectedTo(bundleGraphRuntimeNodeId)
+          .length > 0,
+        'Runtime must have an inbound dependency or be an entry',
+      );
+    } else {
+      let dependencyNodeId = bundleGraph._graph.getNodeIdByContentKey(
+        dependency.id,
+      );
+      bundleGraph._graph.addEdge(dependencyNodeId, bundleGraphRuntimeNodeId);
+    }
+  }
+
+  return changedAssets;
+}
+
+function reconcileNewRuntimes<TResult: RequestResult>(
+  api: RunAPI<TResult>,
+  connections: Array<RuntimeConnection>,
+  optionsRef: SharedReference,
+) {
+  let assetGroups = connections.map(t => t.assetGroup);
+  let request = createAssetGraphRequest({
+    name: 'Runtimes',
+    assetGroups,
+    optionsRef,
+  });
+
+  // rebuild the graph
+  return api.runRequest(request, {force: true});
+}

package/src/assetUtils.js

@@ -0,0 +1,254 @@
+// @flow strict-local
+
+import type {
+  ASTGenerator,
+  BundleBehavior,
+  FilePath,
+  GenerateOutput,
+  Meta,
+  PackageName,
+  Stats,
+  Symbol,
+  SourceLocation,
+  Transformer,
+} from '@atlaspack/types';
+import type {
+  Asset,
+  RequestInvalidation,
+  Dependency,
+  Environment,
+  AtlaspackOptions,
+} from './types';
+
+import {Readable} from 'stream';
+import {PluginLogger} from '@atlaspack/logger';
+import nullthrows from 'nullthrows';
+import CommittedAsset from './CommittedAsset';
+import UncommittedAsset from './UncommittedAsset';
+import loadPlugin from './loadAtlaspackPlugin';
+import {Asset as PublicAsset} from './public/Asset';
+import PluginOptions from './public/PluginOptions';
+import {blobToStream, hashFile} from '@atlaspack/utils';
+import {hashFromOption, toInternalSourceLocation} from './utils';
+import {createBuildCache} from './buildCache';
+import {
+  type ProjectPath,
+  fromProjectPath,
+  fromProjectPathRelative,
+} from './projectPath';
+import {hashString} from '@atlaspack/rust';
+import {BundleBehavior as BundleBehaviorMap} from './types';
+import {PluginTracer} from '@atlaspack/profiler';
+
+type AssetOptions = {|
+  id?: string,
+  committed?: boolean,
+  idBase?: ?string,
+  filePath: ProjectPath,
+  query?: ?string,
+  type: string,
+  contentKey?: ?string,
+  mapKey?: ?string,
+  astKey?: ?string,
+  astGenerator?: ?ASTGenerator,
+  dependencies?: Map<string, Dependency>,
+  bundleBehavior?: ?BundleBehavior,
+  isBundleSplittable?: ?boolean,
+  isSource: boolean,
+  env: Environment,
+  meta?: Meta,
+  outputHash?: ?string,
+  pipeline?: ?string,
+  stats: Stats,
+  symbols?: ?Map<Symbol, {|local: Symbol, loc: ?SourceLocation, meta?: ?Meta|}>,
+  sideEffects?: boolean,
+  uniqueKey?: ?string,
+  plugin?: PackageName,
+  configPath?: ProjectPath,
+  configKeyPath?: string,
+|};
+
+export function createAssetIdFromOptions(options: AssetOptions): string {
+  let uniqueKey = options.uniqueKey ?? '';
+  let idBase =
+    options.idBase != null
+      ? options.idBase
+      : fromProjectPathRelative(options.filePath);
+
+  return hashString(
+    idBase +
+      options.type +
+      options.env.id +
+      uniqueKey +
+      ':' +
+      (options.pipeline ?? '') +
+      ':' +
+      (options.query ?? ''),
+  );
+}
+
+export function createAsset(
+  projectRoot: FilePath,
+  options: AssetOptions,
+): Asset {
+  return {
+    id: options.id != null ? options.id : createAssetIdFromOptions(options),
+    committed: options.committed ?? false,
+    filePath: options.filePath,
+    query: options.query,
+    bundleBehavior: options.bundleBehavior
+      ? BundleBehaviorMap[options.bundleBehavior]
+      : null,
+    isBundleSplittable: options.isBundleSplittable ?? true,
+    type: options.type,
+    contentKey: options.contentKey,
+    mapKey: options.mapKey,
+    astKey: options.astKey,
+    astGenerator: options.astGenerator,
+    dependencies: options.dependencies || new Map(),
+    isSource: options.isSource,
+    outputHash: options.outputHash,
+    pipeline: options.pipeline,
+    env: options.env,
+    meta: options.meta || {},
+    stats: options.stats,
+    symbols:
+      options.symbols &&
+      new Map(
+        [...options.symbols].map(([k, v]) => [
+          k,
+          {
+            local: v.local,
+            meta: v.meta,
+            loc: toInternalSourceLocation(projectRoot, v.loc),
+          },
+        ]),
+      ),
+    sideEffects: options.sideEffects ?? true,
+    uniqueKey: options.uniqueKey,
+    plugin: options.plugin,
+    configPath: options.configPath,
+    configKeyPath: options.configKeyPath,
+  };
+}
+
+const generateResults: WeakMap<Asset, Promise<GenerateOutput>> = new WeakMap();
+
+export function generateFromAST(
+  asset: CommittedAsset | UncommittedAsset,
+): Promise<GenerateOutput> {
+  let output = generateResults.get(asset.value);
+  if (output == null) {
+    output = _generateFromAST(asset);
+    generateResults.set(asset.value, output);
+  }
+  return output;
+}
+
+async function _generateFromAST(asset: CommittedAsset | UncommittedAsset) {
+  let ast = await asset.getAST();
+  if (ast == null) {
+    throw new Error('Asset has no AST');
+  }
+
+  let pluginName = nullthrows(asset.value.plugin);
+  let {plugin} = await loadPlugin<Transformer<mixed>>(
+    pluginName,
+    fromProjectPath(
+      asset.options.projectRoot,
+      nullthrows(asset.value.configPath),
+    ),
+    nullthrows(asset.value.configKeyPath),
+    asset.options,
+  );
+  let generate = plugin.generate?.bind(plugin);
+  if (!generate) {
+    throw new Error(`${pluginName} does not have a generate method`);
+  }
+
+  let {content, map} = await generate({
+    asset: new PublicAsset(asset),
+    ast,
+    options: new PluginOptions(asset.options),
+    logger: new PluginLogger({origin: pluginName}),
+    tracer: new PluginTracer({origin: pluginName, category: 'asset-generate'}),
+  });
+
+  let mapBuffer = map?.toBuffer();
+  // Store the results in the cache so we can avoid generating again next time
+  await Promise.all([
+    asset.options.cache.setStream(
+      nullthrows(asset.value.contentKey),
+      blobToStream(content),
+    ),
+    mapBuffer != null &&
+      asset.options.cache.setBlob(nullthrows(asset.value.mapKey), mapBuffer),
+  ]);
+
+  return {
+    content:
+      content instanceof Readable
+        ? asset.options.cache.getStream(nullthrows(asset.value.contentKey))
+        : content,
+    map,
+  };
+}
+
+export function getInvalidationId(invalidation: RequestInvalidation): string {
+  switch (invalidation.type) {
+    case 'file':
+      return 'file:' + fromProjectPathRelative(invalidation.filePath);
+    case 'env':
+      return 'env:' + invalidation.key;
+    case 'option':
+      return 'option:' + invalidation.key;
+    default:
+      throw new Error('Unknown invalidation type: ' + invalidation.type);
+  }
+}
+
+const hashCache = createBuildCache();
+
+export async function getInvalidationHash(
+  invalidations: Array<RequestInvalidation>,
+  options: AtlaspackOptions,
+): Promise<string> {
+  if (invalidations.length === 0) {
+    return '';
+  }
+
+  let sortedInvalidations = invalidations
+    .slice()
+    .sort((a, b) => (getInvalidationId(a) < getInvalidationId(b) ? -1 : 1));
+
+  let hashes = '';
+  for (let invalidation of sortedInvalidations) {
+    switch (invalidation.type) {
+      case 'file': {
+        // Only recompute the hash of this file if we haven't seen it already during this build.
+        let fileHash = hashCache.get(invalidation.filePath);
+        if (fileHash == null) {
+          fileHash = hashFile(
+            options.inputFS,
+            fromProjectPath(options.projectRoot, invalidation.filePath),
+          );
+          hashCache.set(invalidation.filePath, fileHash);
+        }
+        hashes += await fileHash;
+        break;
+      }
+      case 'env':
+        hashes +=
+          invalidation.key + ':' + (options.env[invalidation.key] || '');
+        break;
+      case 'option':
+        hashes +=
+          invalidation.key + ':' + hashFromOption(options[invalidation.key]);
+        break;
+      default:
+        throw new Error('Unknown invalidation type: ' + invalidation.type);
+    }
+  }
+
+  return hashString(hashes);
+}

package/src/atlaspack-v3/AtlaspackV3.js

@@ -0,0 +1,73 @@
+// @flow
+
+import path from 'path';
+import {Worker} from 'worker_threads';
+import {AtlaspackNapi, type AtlaspackNapiOptions} from '@atlaspack/rust';
+
+const WORKER_PATH = path.join(__dirname, 'worker', 'index.js');
+
+export type AtlaspackV3Options = {|
+  fs?: AtlaspackNapiOptions['fs'],
+  nodeWorkers?: number,
+  packageManager?: AtlaspackNapiOptions['packageManager'],
+  threads?: number,
+  ...AtlaspackNapiOptions['options'],
+|};
+
+export class AtlaspackV3 {
+  _internal: AtlaspackNapi;
+
+  constructor({
+    fs,
+    nodeWorkers,
+    packageManager,
+    threads,
+    ...options
+  }: AtlaspackV3Options) {
+    this._internal = new AtlaspackNapi({
+      fs,
+      nodeWorkers,
+      packageManager,
+      threads,
+      options,
+    });
+  }
+
+  async build(): Promise<any> {
+    const [workers, registerWorker] = this.#createWorkers();
+
+    let result = await this._internal.build({
+      registerWorker,
+    });
+
+    for (const worker of workers) worker.terminate();
+    return result;
+  }
+
+  async buildAssetGraph(): Promise<any> {
+    const [workers, registerWorker] = this.#createWorkers();
+
+    let result = await this._internal.buildAssetGraph({
+      registerWorker,
+    });
+
+    for (const worker of workers) worker.terminate();
+    return result;
+  }
+
+  #createWorkers() {
+    const workers = [];
+
+    return [
+      workers,
+      tx_worker => {
+        let worker = new Worker(WORKER_PATH, {
+          workerData: {
+            tx_worker,
+          },
+        });
+        workers.push(worker);
+      },
+    ];
+  }
+}

package/src/atlaspack-v3/fs.js

@@ -0,0 +1,36 @@
+// @flow strict-local
+
+import type {FileSystem} from '@atlaspack/rust';
+import type {
+  Encoding,
+  FilePath,
+  FileSystem as ClassicFileSystem,
+} from '@atlaspack/types-internal';
+
+import {jsCallable} from './jsCallable';
+
+// Move to @atlaspack/utils or a dedicated v3 / migration package later
+export function toFileSystemV3(fs: ClassicFileSystem): FileSystem {
+  return {
+    canonicalize: jsCallable((path: FilePath) => fs.realpathSync(path)),
+    createDirectory: jsCallable((path: FilePath) => fs.mkdirp(path)),
+    cwd: jsCallable(() => fs.cwd()),
+    readFile: jsCallable((path: string, encoding?: Encoding) =>
+      fs.readFileSync(path, encoding ?? 'utf8'),
+    ),
+    isFile: (path: string) => {
+      try {
+        return fs.statSync(path).isFile();
+      } catch {
+        return false;
+      }
+    },
+    isDir: (path: string) => {
+      try {
+        return fs.statSync(path).isDirectory();
+      } catch {
+        return false;
+      }
+    },
+  };
+}

package/src/atlaspack-v3/index.js

@@ -0,0 +1,5 @@
+// @flow
+
+export {toFileSystemV3} from './fs';
+export {AtlaspackV3} from './AtlaspackV3';
+export type * from './AtlaspackV3';