@atlaspack/core 2.16.2-canary.37 → 2.16.2-canary.371

package/src/requests/AssetGraphRequestRust.ts

@@ -0,0 +1,364 @@
+import invariant from 'assert';
+
+import ThrowableDiagnostic from '@atlaspack/diagnostic';
+import type {Async} from '@atlaspack/types';
+import {instrument} from '@atlaspack/logger';
+import {getFeatureFlag} from '@atlaspack/feature-flags';
+
+import AssetGraph from '../AssetGraph';
+import type {AtlaspackV3} from '../atlaspack-v3';
+import {requestTypes, StaticRunOpts} from '../RequestTracker';
+import {propagateSymbols} from '../SymbolPropagation';
+import type {
+  Environment,
+  Asset,
+  AssetGraphNode,
+  AssetNode,
+  DependencyNode,
+} from '../types';
+
+import type {
+  AssetGraphRequestInput,
+  AssetGraphRequestResult,
+} from './AssetGraphRequest';
+import {toEnvironmentRef} from '../EnvironmentManager';
+import {getEnvironmentHash} from '../Environment';
+import dumpGraphToGraphViz from '../dumpGraphToGraphViz';
+import assert from 'assert';
+
+type RunInput = {
+  input: AssetGraphRequestInput;
+} & StaticRunOpts<AssetGraphRequestResult>;
+
+type AssetGraphRequest = {
+  id: string;
+  readonly type: typeof requestTypes.asset_graph_request;
+  run: (arg1: RunInput) => Async<AssetGraphRequestResult>;
+  input: AssetGraphRequestInput;
+};
+
+type SerializedAssetGraphDelta = {
+  nodes: Array<any>;
+  edges: Array<string>;
+  updates: Array<any>;
+  safeToSkipBundling: boolean;
+  hadPreviousGraph: boolean;
+};
+
+export function createAssetGraphRequestRust(
+  rustAtlaspack: AtlaspackV3,
+): (input: AssetGraphRequestInput) => AssetGraphRequest {
+  return (input: AssetGraphRequestInput) => ({
+    type: requestTypes.asset_graph_request,
+    id: input.name,
+    run: async (input) => {
+      let options = input.options;
+      let {assetGraphPromise, commitPromise} =
+        await rustAtlaspack.buildAssetGraph();
+
+      let [serializedAssetGraph, assetGraphError] =
+        (await assetGraphPromise) as [SerializedAssetGraphDelta, Error | null];
+
+      if (assetGraphError) {
+        throw new ThrowableDiagnostic({
+          diagnostic: assetGraphError,
+        });
+      }
+
+      // Don't reuse a previous asset graph result if Rust didn't have one too
+      let prevResult = null;
+      if (serializedAssetGraph.hadPreviousGraph) {
+        prevResult =
+          await input.api.getPreviousResult<AssetGraphRequestResult>();
+      }
+
+      let {assetGraph, changedAssets} = instrument(
+        'atlaspack_v3_getAssetGraph',
+        () => getAssetGraph(serializedAssetGraph, prevResult?.assetGraph),
+      );
+
+      let changedAssetsPropagation = new Set(changedAssets.keys());
+      let errors = propagateSymbols({
+        options,
+        assetGraph,
+        changedAssetsPropagation,
+        assetGroupsWithRemovedParents: new Set(),
+        previousErrors: new Map(), //this.previousSymbolPropagationErrors,
+      });
+
+      if (errors.size > 0) {
+        // Just throw the first error. Since errors can bubble (e.g. reexporting a reexported symbol also fails),
+        // determining which failing export is the root cause is nontrivial (because of circular dependencies).
+        throw new ThrowableDiagnostic({
+          diagnostic: [...errors.values()][0],
+        });
+      }
+
+      await dumpGraphToGraphViz(assetGraph, 'AssetGraphV3');
+
+      let result = {
+        assetGraph,
+        assetRequests: [],
+        assetGroupsWithRemovedParents: new Set<number>(),
+        changedAssets,
+        changedAssetsPropagation,
+        previousSymbolPropagationErrors: undefined,
+      };
+
+      let [_commitResult, commitError] = await commitPromise;
+
+      if (commitError) {
+        throw new ThrowableDiagnostic({
+          diagnostic: {
+            message:
+              'Error committing asset graph in Rust: ' + commitError.message,
+          },
+        });
+      }
+
+      await input.api.storeResult(result);
+      input.api.invalidateOnBuild();
+
+      return result;
+    },
+    input,
+  });
+}
+
+export function getAssetGraph(
+  serializedGraph: any,
+  prevAssetGraph?: AssetGraph,
+): {
+  assetGraph: AssetGraph;
+  changedAssets: Map<string, Asset>;
+} {
+  let graph: AssetGraph;
+
+  let reuseEdges = false;
+
+  if (prevAssetGraph && serializedGraph.safeToSkipBundling) {
+    graph = new AssetGraph({
+      _contentKeyToNodeId: prevAssetGraph._contentKeyToNodeId,
+      _nodeIdToContentKey: prevAssetGraph._nodeIdToContentKey,
+      nodes: prevAssetGraph.nodes,
+      rootNodeId: prevAssetGraph.rootNodeId,
+      adjacencyList: prevAssetGraph.adjacencyList,
+    });
+    reuseEdges = true;
+  } else if (
+    prevAssetGraph &&
+    (serializedGraph.updates.length > 0 || serializedGraph.nodes.length > 0)
+  ) {
+    graph = new AssetGraph({
+      _contentKeyToNodeId: prevAssetGraph._contentKeyToNodeId,
+      _nodeIdToContentKey: prevAssetGraph._nodeIdToContentKey,
+      nodes: prevAssetGraph.nodes,
+      initialCapacity: serializedGraph.edges.length,
+      // Accomodate the root node
+      initialNodeCapacity: prevAssetGraph.nodes.length + 1,
+      rootNodeId: prevAssetGraph.rootNodeId,
+    });
+    graph.safeToIncrementallyBundle = false;
+  } else {
+    graph = new AssetGraph({
+      _contentKeyToNodeId: new Map(),
+      _nodeIdToContentKey: new Map(),
+      initialCapacity: serializedGraph.edges.length,
+      // Accomodate the root node
+      initialNodeCapacity: serializedGraph.nodes.length + 1,
+    });
+
+    let rootNodeId = graph.addNodeByContentKey('@@root', {
+      id: '@@root',
+      type: 'root',
+      value: null,
+    });
+
+    graph.setRootNodeId(rootNodeId);
+    graph.safeToIncrementallyBundle = false;
+  }
+
+  invariant(graph, 'Asset graph not initialized');
+  invariant(graph.rootNodeId != null, 'Asset graph has no root node');
+
+  // @ts-expect-error TS7031
+  function mapSymbols({exported, ...symbol}) {
+    let jsSymbol = {
+      local: symbol.local ?? undefined,
+      loc: symbol.loc ?? undefined,
+      isWeak: symbol.isWeak,
+      meta: {
+        isEsm: symbol.isEsmExport,
+        isStaticBindingSafe: symbol.isStaticBindingSafe,
+      },
+    };
+
+    if (symbol.exported) {
+      // @ts-expect-error TS2339
+      jsSymbol.exported = symbol.exported;
+    }
+
+    return [exported, jsSymbol];
+  }
+
+  // See crates/atlaspack_core/src/types/environment.rs
+  let changedAssets = new Map();
+  let entry = 0;
+
+  let envs = new Map();
+  let getEnvId = (env: Environment) => {
+    let envKey = [
+      env.context,
+      env.engines.atlaspack,
+      env.engines.browsers,
+      env.engines.electron,
+      env.engines.node,
+      env.includeNodeModules,
+      env.isLibrary,
+      env.outputFormat,
+      env.shouldScopeHoist,
+      env.shouldOptimize,
+      env.sourceType,
+    ].join(':');
+
+    let envId = envs.get(envKey);
+    if (envId == null) {
+      envId = getEnvironmentHash(env);
+      envs.set(envKey, envId);
+    }
+
+    return envId;
+  };
+
+  function updateNode(newNode: AssetGraphNode, isUpdateNode: boolean) {
+    if (isUpdateNode) {
+      let existingNode = graph.getNodeByContentKey(newNode.id);
+
+      assert(existingNode && existingNode.type === newNode.type);
+
+      Object.assign(existingNode, newNode);
+    } else {
+      graph.addNodeByContentKey(newNode.id, newNode);
+    }
+  }
+
+  let nodeTypeSwitchoverIndex = serializedGraph.nodes.length;
+  let nodesCount =
+    serializedGraph.nodes.length + serializedGraph.updates.length;
+
+  for (let index = 0; index < nodesCount; index++) {
+    let isUpdateNode = index >= nodeTypeSwitchoverIndex;
+    let node = isUpdateNode
+      ? serializedGraph.updates[index - nodeTypeSwitchoverIndex]
+      : serializedGraph.nodes[index];
+
+    node = JSON.parse(node);
+
+    if (node.type === 'entry') {
+      let id = 'entry:' + ++entry;
+
+      graph.addNodeByContentKey(id, {
+        id: id,
+        type: 'root',
+        value: null,
+      });
+    } else if (node.type === 'asset') {
+      let asset = node.value;
+      let id = asset.id;
+
+      asset.committed = true;
+      asset.contentKey = id;
+      asset.env.id = getFeatureFlag('environmentDeduplication')
+        ? // TODO: Rust can do this and avoid copying a significant amount of data over
+          getEnvironmentHash(asset.env)
+        : getEnvId(asset.env);
+      asset.mapKey = `map:${asset.id}`;
+
+      asset.env = toEnvironmentRef(asset.env);
+
+      // This is populated later when we map the edges between assets and dependencies
+      asset.dependencies = new Map();
+
+      // We need to add this property for source map handling, as some assets like runtimes
+      // are processed after the Rust transformation and take the v2 code path
+      asset.meta.isV3 = true;
+
+      if (asset.symbols != null) {
+        asset.symbols = new Map(asset.symbols.map(mapSymbols));
+      }
+
+      changedAssets.set(id, asset);
+
+      let assetNode: AssetNode = {
+        id,
+        type: 'asset',
+        usedSymbols: new Set(),
+        usedSymbolsDownDirty: true,
+        usedSymbolsUpDirty: true,
+        value: asset,
+      };
+      updateNode(assetNode, isUpdateNode);
+    } else if (node.type === 'dependency') {
+      let {dependency, id} = node.value;
+
+      dependency.id = id;
+      dependency.env.id = getFeatureFlag('environmentDeduplication')
+        ? // TODO: Rust can do this and avoid copying a significant amount of data over
+          getEnvironmentHash(dependency.env)
+        : getEnvId(dependency.env);
+      dependency.env = toEnvironmentRef(dependency.env);
+
+      if (dependency.symbols != null) {
+        dependency.symbols = new Map(dependency.symbols?.map(mapSymbols));
+      }
+
+      let usedSymbolsDown = new Set();
+      let usedSymbolsUp = new Map();
+      if (dependency.isEntry && dependency.isLibrary) {
+        usedSymbolsDown.add('*');
+        usedSymbolsUp.set('*', undefined);
+      }
+
+      let depNode: DependencyNode = {
+        id,
+        type: 'dependency',
+        deferred: false,
+        excluded: false,
+        hasDeferred: node.has_deferred,
+        // @ts-expect-error TS2322
+        usedSymbolsDown,
+        usedSymbolsDownDirty: true,
+        usedSymbolsUp,
+        usedSymbolsUpDirtyDown: true,
+        usedSymbolsUpDirtyUp: true,
+        value: dependency,
+      };
+
+      updateNode(depNode, isUpdateNode);
+    }
+  }
+
+  if (!reuseEdges) {
+    for (let i = 0; i < serializedGraph.edges.length; i += 2) {
+      let from = serializedGraph.edges[i];
+      let to = serializedGraph.edges[i + 1];
+      let fromNode = graph.getNode(from);
+      let toNode = graph.getNode(to);
+
+      if (fromNode?.type === 'dependency') {
+        invariant(toNode?.type === 'asset');
+      }
+
+      if (fromNode?.type === 'asset' && toNode?.type === 'dependency') {
+        fromNode.value.dependencies.set(toNode.value.id, toNode.value);
+      }
+
+      graph.addEdge(from, to);
+    }
+  }
+
+  return {
+    assetGraph: graph,
+    changedAssets,
+  };
+}

package/src/requests/{AssetRequest.js → AssetRequest.ts}

@@ -1,5 +1,3 @@
-// @flow strict-local
-
 import type {ContentKey} from '@atlaspack/graph';
 import type {Async} from '@atlaspack/types';
 import type {StaticRunOpts} from '../RequestTracker';
@@ -21,18 +19,18 @@ import {fromProjectPath, fromProjectPathRelative} from '../projectPath';
 import {report} from '../ReporterRunner';
 import {requestTypes} from '../RequestTracker';
 import type {DevDepRequestResult} from './DevDepRequest';
+import {toEnvironmentId} from '../EnvironmentManager';
 
-type RunInput<TResult> = {|
-  input: AssetRequestInput,
-  ...StaticRunOpts<TResult>,
-|};
+type RunInput<TResult> = {
+  input: AssetRequestInput;
+} & StaticRunOpts<TResult>;
 
-export type AssetRequest = {|
-  id: ContentKey,
-  +type: typeof requestTypes.asset_request,
-  run: (RunInput<AssetRequestResult>) => Async<AssetRequestResult>,
-  input: AssetRequestInput,
-|};
+export type AssetRequest = {
+  id: ContentKey;
+  readonly type: typeof requestTypes.asset_request;
+  run: (arg1: RunInput<AssetRequestResult>) => Async<AssetRequestResult>;
+  input: AssetRequestInput;
+};
 
 export default function createAssetRequest(
   input: AssetRequestInput,
@@ -51,7 +49,7 @@ function getId(input: AssetRequestInput) {
   return hashString(
     type +
       fromProjectPathRelative(input.filePath) +
-      input.env.id +
+      toEnvironmentId(input.env) +
       String(input.isSource) +
       String(input.sideEffects) +
       (input.code ?? '') +
@@ -62,6 +60,7 @@ function getId(input: AssetRequestInput) {
   );
 }
 
+// @ts-expect-error TS7031
 async function run({input, api, farm, invalidateReason, options}) {
   report({
     type: 'buildProgress',
@@ -73,6 +72,7 @@ async function run({input, api, farm, invalidateReason, options}) {
   let start = Date.now();
   let {optionsRef, ...rest} = input;
   let {cachePath} = nullthrows(
+    // @ts-expect-error TS2347
     await api.runRequest<null, ConfigAndCachePath>(
       createAtlaspackConfigRequest(),
     ),
@@ -82,9 +82,12 @@ async function run({input, api, farm, invalidateReason, options}) {
     await Promise.all(
       api
         .getSubRequests()
+        // @ts-expect-error TS7006
         .filter((req) => req.requestType === requestTypes.dev_dep_request)
+        // @ts-expect-error TS7006
         .map(async (req) => [
           req.id,
+          // @ts-expect-error TS2347
           nullthrows(await api.getRequestResult<DevDepRequestResult>(req.id)),
         ]),
     ),
@@ -95,21 +98,24 @@ async function run({input, api, farm, invalidateReason, options}) {
     invalidateReason,
     devDeps: new Map(
       [...previousDevDepRequests.entries()]
-        .filter(([id]) => api.canSkipSubrequest(id))
-        .map(([, req]: [string, DevDepRequestResult]) => [
+        // @ts-expect-error TS2769
+        .filter(([id]: [any]) => api.canSkipSubrequest(id))
+        .map(([, req]: [any, any]) => [
           `${req.specifier}:${fromProjectPathRelative(req.resolveFrom)}`,
           req.hash,
         ]),
     ),
     invalidDevDeps: await Promise.all(
       [...previousDevDepRequests.entries()]
-        .filter(([id]) => !api.canSkipSubrequest(id))
-        .flatMap(([, req]: [string, DevDepRequestResult]) => {
+        // @ts-expect-error TS2769
+        .filter(([id]: [any]) => !api.canSkipSubrequest(id))
+        .flatMap(([, req]: [any, any]) => {
           return [
             {
               specifier: req.specifier,
               resolveFrom: req.resolveFrom,
             },
+            // @ts-expect-error TS7006
             ...(req.additionalInvalidations ?? []).map((i) => ({
               specifier: i.specifier,
               resolveFrom: i.resolveFrom,
@@ -127,7 +133,7 @@ async function run({input, api, farm, invalidateReason, options}) {
       configCachePath: cachePath,
       optionsRef,
       request,
-    }): TransformationResult);
+    })) as TransformationResult;
 
   let time = Date.now() - start;
   if (assets) {

package/src/requests/AtlaspackBuildRequest.ts

@@ -0,0 +1,163 @@
+import type {ContentKey} from '@atlaspack/graph';
+import type {Async} from '@atlaspack/types';
+import type {SharedReference} from '@atlaspack/workers';
+
+import type {StaticRunOpts} from '../RequestTracker';
+import type {
+  Asset,
+  AssetGroup,
+  BundleGraphNode,
+  PackagedBundleInfo,
+} from '../types';
+import type BundleGraph from '../BundleGraph';
+
+import createBundleGraphRequest, {
+  BundleGraphResult,
+} from './BundleGraphRequest';
+import createWriteBundlesRequest from './WriteBundlesRequest';
+import {assertSignalNotAborted} from '../utils';
+import dumpGraphToGraphViz from '../dumpGraphToGraphViz';
+import {BundleGraphEdgeType, bundleGraphEdgeTypes} from '../BundleGraph';
+import {report} from '../ReporterRunner';
+import IBundleGraph from '../public/BundleGraph';
+import {NamedBundle} from '../public/Bundle';
+import {assetFromValue} from '../public/Asset';
+
+import {tracer} from '@atlaspack/profiler';
+import {requestTypes} from '../RequestTracker';
+import {getFeatureFlag} from '@atlaspack/feature-flags';
+import {fromEnvironmentId} from '../EnvironmentManager';
+import invariant from 'assert';
+
+type AtlaspackBuildRequestInput = {
+  optionsRef: SharedReference;
+  requestedAssetIds: Set<string>;
+  signal?: AbortSignal;
+};
+
+export type AtlaspackBuildRequestResult = {
+  bundleGraph: BundleGraph;
+  bundleInfo: Map<string, PackagedBundleInfo>;
+  changedAssets: Map<string, Asset>;
+  assetRequests: Array<AssetGroup>;
+  scopeHoistingStats?: {
+    totalAssets: number;
+    wrappedAssets: number;
+  };
+};
+
+type RunInput<TResult> = {
+  input: AtlaspackBuildRequestInput;
+} & StaticRunOpts<TResult>;
+
+export type AtlaspackBuildRequest = {
+  id: ContentKey;
+  readonly type: typeof requestTypes.atlaspack_build_request;
+  run: (
+    arg1: RunInput<AtlaspackBuildRequestResult>,
+  ) => Async<AtlaspackBuildRequestResult>;
+  input: AtlaspackBuildRequestInput;
+};
+
+export default function createAtlaspackBuildRequest(
+  input: AtlaspackBuildRequestInput,
+): AtlaspackBuildRequest {
+  return {
+    type: requestTypes.atlaspack_build_request,
+    id: 'atlaspack_build_request',
+    run,
+    input,
+  };
+}
+
+async function run({
+  input,
+  api,
+  options,
+  rustAtlaspack,
+}: RunInput<AtlaspackBuildRequestResult>) {
+  let {optionsRef, requestedAssetIds, signal} = input;
+
+  let bundleGraphRequest = createBundleGraphRequest({
+    optionsRef,
+    requestedAssetIds,
+    signal,
+  });
+
+  let {bundleGraph, changedAssets, assetRequests}: BundleGraphResult =
+    await api.runRequest(bundleGraphRequest, {
+      force:
+        Boolean(rustAtlaspack) ||
+        (options.shouldBuildLazily && requestedAssetIds.size > 0),
+    });
+
+  if (
+    getFeatureFlag('nativePackager') &&
+    getFeatureFlag('nativePackagerSSRDev') &&
+    rustAtlaspack
+  ) {
+    let hasSupportedTarget = false;
+    bundleGraph.traverseBundles((bundle, ctx, actions) => {
+      if (
+        fromEnvironmentId(bundle.env).context === 'tesseract' &&
+        bundle.type === 'js'
+      ) {
+        hasSupportedTarget = true;
+        actions.stop();
+      }
+    });
+    if (hasSupportedTarget) {
+      // In theory this could be a (somehow) null[] - but we know if we've gotten this far it probably isn't..
+      let nodes = bundleGraph._graph.nodes as BundleGraphNode[];
+      let rawEdges: [number, number, BundleGraphEdgeType][] = [];
+      const gen = bundleGraph._graph.getAllEdges();
+      let next = gen.next();
+      while (!next.done) {
+        let edge = next.value;
+        rawEdges.push([edge.from, edge.to, edge.type]);
+        next = gen.next();
+      }
+      await rustAtlaspack.loadBundleGraph({nodes, edges: rawEdges});
+    }
+  }
+
+  // @ts-expect-error TS2345
+  dumpGraphToGraphViz(bundleGraph._graph, 'BundleGraph', bundleGraphEdgeTypes);
+
+  await report({
+    type: 'buildProgress',
+    phase: 'bundled',
+    bundleGraph: new IBundleGraph(
+      bundleGraph,
+      // @ts-expect-error TS2304
+      (bundle: Bundle, bundleGraph: BundleGraph, options: AtlaspackOptions) =>
+        NamedBundle.get(bundle, bundleGraph, options),
+      options,
+    ),
+    changedAssets: new Map(
+      Array.from(changedAssets).map(([id, asset]: [any, any]) => [
+        id,
+        assetFromValue(asset, options),
+      ]),
+    ),
+  });
+
+  let packagingMeasurement = tracer.createMeasurement('packaging');
+  let writeBundlesRequest = createWriteBundlesRequest({
+    bundleGraph,
+    optionsRef,
+  });
+
+  let {bundleInfo, scopeHoistingStats} =
+    await api.runRequest(writeBundlesRequest);
+  packagingMeasurement && packagingMeasurement.end();
+  assertSignalNotAborted(signal);
+
+  return {
+    bundleGraph,
+    bundleInfo,
+    changedAssets,
+    assetRequests,
+    scopeHoistingStats,
+  };
+}