@atlaspack/core 2.16.2-canary.48 → 2.16.2-canary.481

package/src/requests/AssetGraphRequestRust.ts

@@ -0,0 +1,557 @@
+import invariant from 'assert';
+
+import ThrowableDiagnostic from '@atlaspack/diagnostic';
+import type {Async} from '@atlaspack/types';
+import logger, {instrument} from '@atlaspack/logger';
+import {getFeatureFlag} from '@atlaspack/feature-flags';
+
+import AssetGraph from '../AssetGraph';
+import type {AtlaspackV3} from '../atlaspack-v3';
+import {report} from '../ReporterRunner';
+import {requestTypes, StaticRunOpts} from '../RequestTracker';
+import {propagateSymbols} from '../SymbolPropagation';
+import type {
+  Environment,
+  Asset,
+  AssetGraphNode,
+  AssetNode,
+  DependencyNode,
+} from '../types';
+
+import type {
+  AssetGraphRequestInput,
+  AssetGraphRequestResult,
+} from './AssetGraphRequest';
+import {toEnvironmentRef} from '../EnvironmentManager';
+import {getEnvironmentHash} from '../Environment';
+import dumpGraphToGraphViz from '../dumpGraphToGraphViz';
+import assert from 'assert';
+
+type RunInput = {
+  input: AssetGraphRequestInput;
+} & StaticRunOpts<AssetGraphRequestResult>;
+
+type AssetGraphRequest = {
+  id: string;
+  readonly type: typeof requestTypes.asset_graph_request;
+  run: (arg1: RunInput) => Async<AssetGraphRequestResult>;
+  input: AssetGraphRequestInput;
+};
+
+type SerializedAssetGraphDelta = {
+  nodes: Array<any>;
+  edges: Array<string>;
+  updates: Array<any>;
+  safeToSkipBundling: boolean;
+  hadPreviousGraph: boolean;
+};
+
+export function createAssetGraphRequestRust(
+  rustAtlaspack: AtlaspackV3,
+): (input: AssetGraphRequestInput) => AssetGraphRequest {
+  return (input: AssetGraphRequestInput) => ({
+    type: requestTypes.asset_graph_request,
+    id: input.name,
+    run: async (runInput) => {
+      let options = runInput.options;
+      let {assetGraphPromise, commitPromise} =
+        await rustAtlaspack.buildAssetGraph((eventJson: string) => {
+          let event = JSON.parse(eventJson);
+          report(event);
+        });
+
+      let [serializedAssetGraph, assetGraphError] =
+        (await assetGraphPromise) as [SerializedAssetGraphDelta, Error | null];
+
+      if (assetGraphError) {
+        throw new ThrowableDiagnostic({
+          diagnostic: assetGraphError,
+        });
+      }
+
+      // Don't reuse a previous asset graph result if Rust didn't have one too
+      let prevResult = null;
+      if (serializedAssetGraph.hadPreviousGraph) {
+        prevResult =
+          await runInput.api.getPreviousResult<AssetGraphRequestResult>();
+      }
+
+      let {assetGraph, changedAssets} = instrument(
+        'atlaspack_v3_getAssetGraph',
+        () => getAssetGraph(serializedAssetGraph, prevResult?.assetGraph),
+      );
+
+      let changedAssetsPropagation = new Set(changedAssets.keys());
+
+      let result = {
+        assetGraph,
+        assetRequests: [],
+        assetGroupsWithRemovedParents: new Set<number>(),
+        changedAssets,
+        changedAssetsPropagation,
+        previousSymbolPropagationErrors: undefined,
+      };
+
+      // When v3AssetGraphSyncImprovements is enabled, wrap downstream steps
+      // in try/catch and store the result even on error. This prevents Rust/JS
+      // divergence when symbol propagation or commit_assets fails.
+      let storeResultOnError = getFeatureFlag('v3AssetGraphSyncImprovements');
+
+      try {
+        // Skip symbol propagation for runtime assets - they have pre-computed symbol data
+        if (input.skipSymbolProp) {
+          logger.verbose({
+            origin: '@atlaspack/core',
+            message: 'Skipping symbol propagation for runtime asset graph',
+          });
+        } else {
+          let errors = propagateSymbols({
+            options,
+            assetGraph,
+            changedAssetsPropagation,
+            assetGroupsWithRemovedParents: new Set(),
+            previousErrors: new Map(), //this.previousSymbolPropagationErrors,
+          });
+
+          if (errors.size > 0) {
+            // Just throw the first error. Since errors can bubble (e.g. reexporting a reexported symbol also fails),
+            // determining which failing export is the root cause is nontrivial (because of circular dependencies).
+            throw new ThrowableDiagnostic({
+              diagnostic: [...errors.values()][0],
+            });
+          }
+        }
+
+        await dumpGraphToGraphViz(assetGraph, 'AssetGraphV3');
+
+        let [_commitResult, commitError] = await commitPromise;
+
+        if (commitError) {
+          throw new ThrowableDiagnostic({
+            diagnostic: {
+              message:
+                'Error committing asset graph in Rust: ' + commitError.message,
+            },
+          });
+        }
+
+        await runInput.api.storeResult(result);
+        runInput.api.invalidateOnBuild();
+      } catch (e) {
+        if (storeResultOnError) {
+          // Store the graph even on error to prevent Rust/JS divergence.
+          // The graph from getAssetGraph is structurally correct — only
+          // downstream processing (symbols, commit) failed.
+          await runInput.api.storeResult(result);
+          runInput.api.invalidateOnBuild();
+        }
+        throw e;
+      }
+
+      return result;
+    },
+    input,
+  });
+}
+
+export function getAssetGraph(
+  serializedGraph: any,
+  prevAssetGraph?: AssetGraph,
+): {
+  assetGraph: AssetGraph;
+  changedAssets: Map<string, Asset>;
+} {
+  let graph: AssetGraph;
+
+  let reuseEdges = false;
+
+  let clonePrevGraph = getFeatureFlag('v3AssetGraphSyncImprovements');
+
+  if (prevAssetGraph && serializedGraph.safeToSkipBundling) {
+    if (clonePrevGraph) {
+      graph = new AssetGraph({
+        _contentKeyToNodeId: new Map(prevAssetGraph._contentKeyToNodeId),
+        _nodeIdToContentKey: new Map(prevAssetGraph._nodeIdToContentKey),
+        nodes: [...prevAssetGraph.nodes],
+        rootNodeId: prevAssetGraph.rootNodeId,
+        adjacencyList: prevAssetGraph.adjacencyList,
+      });
+    } else {
+      graph = new AssetGraph({
+        _contentKeyToNodeId: prevAssetGraph._contentKeyToNodeId,
+        _nodeIdToContentKey: prevAssetGraph._nodeIdToContentKey,
+        nodes: prevAssetGraph.nodes,
+        rootNodeId: prevAssetGraph.rootNodeId,
+        adjacencyList: prevAssetGraph.adjacencyList,
+      });
+    }
+    reuseEdges = true;
+  } else if (
+    prevAssetGraph &&
+    (serializedGraph.updates.length > 0 || serializedGraph.nodes.length > 0)
+  ) {
+    if (clonePrevGraph) {
+      graph = new AssetGraph({
+        _contentKeyToNodeId: new Map(prevAssetGraph._contentKeyToNodeId),
+        _nodeIdToContentKey: new Map(prevAssetGraph._nodeIdToContentKey),
+        nodes: [...prevAssetGraph.nodes],
+        initialCapacity: serializedGraph.edges.length,
+        // Accomodate the root node
+        initialNodeCapacity: prevAssetGraph.nodes.length + 1,
+        rootNodeId: prevAssetGraph.rootNodeId,
+      });
+    } else {
+      graph = new AssetGraph({
+        _contentKeyToNodeId: prevAssetGraph._contentKeyToNodeId,
+        _nodeIdToContentKey: prevAssetGraph._nodeIdToContentKey,
+        nodes: prevAssetGraph.nodes,
+        initialCapacity: serializedGraph.edges.length,
+        // Accomodate the root node
+        initialNodeCapacity: prevAssetGraph.nodes.length + 1,
+        rootNodeId: prevAssetGraph.rootNodeId,
+      });
+    }
+    graph.safeToIncrementallyBundle = false;
+  } else {
+    graph = new AssetGraph({
+      _contentKeyToNodeId: new Map(),
+      _nodeIdToContentKey: new Map(),
+      initialCapacity: serializedGraph.edges.length,
+      // Accomodate the root node
+      initialNodeCapacity: serializedGraph.nodes.length + 1,
+    });
+
+    let rootNodeId = graph.addNodeByContentKey('@@root', {
+      id: '@@root',
+      type: 'root',
+      value: null,
+    });
+
+    graph.setRootNodeId(rootNodeId);
+    graph.safeToIncrementallyBundle = false;
+  }
+
+  invariant(graph, 'Asset graph not initialized');
+  invariant(graph.rootNodeId != null, 'Asset graph has no root node');
+
+  // @ts-expect-error TS7031
+  function mapSymbols({exported, ...symbol}) {
+    let jsSymbol = {
+      local: symbol.local ?? undefined,
+      loc: symbol.loc ?? undefined,
+      isWeak: symbol.isWeak,
+      meta: {
+        isEsm: symbol.isEsmExport,
+        isStaticBindingSafe: symbol.isStaticBindingSafe,
+      },
+    };
+
+    if (symbol.exported) {
+      // @ts-expect-error TS2339
+      jsSymbol.exported = symbol.exported;
+    }
+
+    return [exported, jsSymbol];
+  }
+
+  // See crates/atlaspack_core/src/types/environment.rs
+  let changedAssets = new Map();
+  let entry = 0;
+
+  let envs = new Map();
+  let getEnvId = (env: Environment) => {
+    let envKey = [
+      env.context,
+      env.engines.atlaspack,
+      env.engines.browsers,
+      env.engines.electron,
+      env.engines.node,
+      env.includeNodeModules,
+      env.isLibrary,
+      env.outputFormat,
+      env.shouldScopeHoist,
+      env.shouldOptimize,
+      env.sourceType,
+    ].join(':');
+
+    let envId = envs.get(envKey);
+    if (envId == null) {
+      envId = getEnvironmentHash(env);
+      envs.set(envKey, envId);
+    }
+
+    return envId;
+  };
+
+  function describeNode(node: AssetGraphNode): Record<string, unknown> {
+    const base = {type: node.type, id: node.id};
+    if (node.type === 'asset') {
+      return {
+        ...base,
+        filePath: node.value.filePath,
+        fileType: node.value.type,
+        pipeline: node.value.pipeline,
+      };
+    } else if (node.type === 'dependency') {
+      return {
+        ...base,
+        specifier: node.value.specifier,
+        specifierType: node.value.specifierType,
+        sourceAssetId: node.value.sourceAssetId,
+        sourcePath: node.value.sourcePath,
+      };
+    }
+    return base;
+  }
+
+  function buildDivergenceDiagnostics(
+    divergenceType: string,
+    newNode: AssetGraphNode,
+    existingNode: AssetGraphNode | null | undefined,
+    index: number,
+  ) {
+    return {
+      contentKey: newNode.id,
+      divergenceType,
+      newNode: describeNode(newNode),
+      existingNode: existingNode ? describeNode(existingNode) : null,
+      iterationIndex: index,
+      totalSerializedNodes: nodesCount,
+      newNodesCount: serializedGraph.nodes.length,
+      updatesCount: serializedGraph.updates.length,
+      edgesCount: serializedGraph.edges.length,
+      hadPreviousGraph: !!prevAssetGraph,
+      safeToSkipBundling: serializedGraph.safeToSkipBundling,
+      graphNodeCount: graph._contentKeyToNodeId.size,
+    };
+  }
+
+  function updateNode(
+    newNode: AssetGraphNode,
+    isUpdateNode: boolean,
+    index: number,
+  ) {
+    let healDivergence = getFeatureFlag('v3AssetGraphSyncImprovements');
+
+    if (isUpdateNode) {
+      let existingNode = graph.getNodeByContentKey(newNode.id);
+
+      if (healDivergence) {
+        if (existingNode) {
+          assert(existingNode.type === newNode.type);
+          Object.assign(existingNode, newNode);
+        } else {
+          // Rust sent an update for a node JS doesn't have.
+          // This means JS's graph is stale. Add the node as new instead.
+          // This handles the "undefined == true" error class.
+          let diagnostics = buildDivergenceDiagnostics(
+            'update_node_not_found',
+            newNode,
+            existingNode,
+            index,
+          );
+          logger.warn({
+            origin: '@atlaspack/core',
+            message:
+              `Rust/JS asset graph divergence healed: update node not found, ` +
+              `adding as new. contentKey=${newNode.id} type=${newNode.type} iterationIndex=${index}`,
+            meta: {
+              trackableEvent: 'asset_graph_divergence_healed',
+              ...diagnostics,
+            },
+          });
+          graph.addNodeByContentKey(newNode.id, newNode);
+        }
+      } else {
+        assert(existingNode && existingNode.type === newNode.type);
+        Object.assign(existingNode, newNode);
+      }
+    } else {
+      if (healDivergence) {
+        if (graph.hasContentKey(newNode.id)) {
+          // Rust sent a "new" node that JS already has.
+          // This means JS's graph has nodes that Rust considers new.
+          // Treat as an update instead of throwing.
+          // This handles the "Graph already has content key" error class.
+          let existingNode = graph.getNodeByContentKey(newNode.id);
+          let diagnostics = buildDivergenceDiagnostics(
+            'new_node_already_exists',
+            newNode,
+            existingNode,
+            index,
+          );
+          logger.warn({
+            origin: '@atlaspack/core',
+            message:
+              `Rust/JS asset graph divergence healed: new node already exists, ` +
+              `treating as update. contentKey=${newNode.id} type=${newNode.type} iterationIndex=${index}`,
+            meta: {
+              trackableEvent: 'asset_graph_divergence_healed',
+              ...diagnostics,
+            },
+          });
+          if (existingNode) {
+            Object.assign(existingNode, newNode);
+          }
+        } else {
+          graph.addNodeByContentKey(newNode.id, newNode);
+        }
+      } else {
+        try {
+          graph.addNodeByContentKey(newNode.id, newNode);
+        } catch (e) {
+          if (
+            e instanceof Error &&
+            e.message.includes('already has content key')
+          ) {
+            let existingNode = graph.getNodeByContentKey(newNode.id);
+            let diagnostics = buildDivergenceDiagnostics(
+              'new_node_already_exists',
+              newNode,
+              existingNode,
+              index,
+            );
+
+            throw new Error(
+              `Graph already has content key '${newNode.id}'. Diagnostics: ${JSON.stringify(diagnostics, null, 2)}`,
+            );
+          }
+          throw e;
+        }
+      }
+    }
+  }
+
+  let nodeTypeSwitchoverIndex = serializedGraph.nodes.length;
+  let nodesCount =
+    serializedGraph.nodes.length + serializedGraph.updates.length;
+
+  for (let index = 0; index < nodesCount; index++) {
+    let isUpdateNode = index >= nodeTypeSwitchoverIndex;
+    let node = isUpdateNode
+      ? serializedGraph.updates[index - nodeTypeSwitchoverIndex]
+      : serializedGraph.nodes[index];
+
+    node = JSON.parse(node);
+
+    if (node.type === 'entry') {
+      let id = 'entry:' + ++entry;
+
+      graph.addNodeByContentKey(id, {
+        id: id,
+        type: 'root',
+        value: null,
+      });
+    } else if (node.type === 'asset') {
+      let asset = node.value;
+      let id = asset.id;
+
+      asset.committed = true;
+      asset.contentKey = id;
+      asset.env.id = getFeatureFlag('environmentDeduplication')
+        ? // TODO: Rust can do this and avoid copying a significant amount of data over
+          getEnvironmentHash(asset.env)
+        : getEnvId(asset.env);
+      asset.mapKey = `map:${asset.id}`;
+
+      asset.env = toEnvironmentRef(asset.env);
+
+      // This is populated later when we map the edges between assets and dependencies
+      asset.dependencies = new Map();
+
+      // We need to add this property for source map handling, as some assets like runtimes
+      // are processed after the Rust transformation and take the v2 code path
+      asset.meta.isV3 = true;
+
+      if (asset.symbols != null) {
+        asset.symbols = new Map(asset.symbols.map(mapSymbols));
+      }
+
+      changedAssets.set(id, asset);
+
+      let assetNode: AssetNode = {
+        id,
+        type: 'asset',
+        usedSymbols: new Set(),
+        usedSymbolsDownDirty: true,
+        usedSymbolsUpDirty: true,
+        value: asset,
+      };
+      updateNode(assetNode, isUpdateNode, index);
+    } else if (node.type === 'dependency') {
+      let {dependency, id} = node.value;
+
+      dependency.id = id;
+      dependency.env.id = getFeatureFlag('environmentDeduplication')
+        ? // TODO: Rust can do this and avoid copying a significant amount of data over
+          getEnvironmentHash(dependency.env)
+        : getEnvId(dependency.env);
+      dependency.env = toEnvironmentRef(dependency.env);
+
+      if (dependency.symbols != null) {
+        dependency.symbols = new Map(dependency.symbols?.map(mapSymbols));
+      }
+
+      let usedSymbolsDown = new Set();
+      let usedSymbolsUp = new Map();
+
+      if (node.used_symbols_up) {
+        for (let usedSymbol of node.used_symbols_up) {
+          // Transform Rust UsedSymbol { symbol: Symbol, asset: string, resolved_symbol: string }
+          // to JS format { symbol: string, asset: string } where symbol is the resolved name
+          const exportedName = usedSymbol.symbol.exported;
+          usedSymbolsUp.set(exportedName, {
+            asset: usedSymbol.asset,
+            symbol: usedSymbol.resolved_symbol ?? exportedName,
+          });
+        }
+      }
+
+      if (dependency.isEntry && dependency.isLibrary) {
+        usedSymbolsDown.add('*');
+        usedSymbolsUp.set('*', undefined);
+      }
+
+      let depNode: DependencyNode = {
+        id,
+        type: 'dependency',
+        deferred: false,
+        excluded: false,
+        hasDeferred: node.has_deferred,
+        // @ts-expect-error TS2322
+        usedSymbolsDown,
+        usedSymbolsDownDirty: true,
+        usedSymbolsUp,
+        usedSymbolsUpDirtyDown: true,
+        usedSymbolsUpDirtyUp: true,
+        value: dependency,
+      };
+
+      updateNode(depNode, isUpdateNode, index);
+    }
+  }
+
+  if (!reuseEdges) {
+    for (let i = 0; i < serializedGraph.edges.length; i += 2) {
+      let from = serializedGraph.edges[i];
+      let to = serializedGraph.edges[i + 1];
+      let fromNode = graph.getNode(from);
+      let toNode = graph.getNode(to);
+
+      if (fromNode?.type === 'dependency') {
+        invariant(toNode?.type === 'asset');
+      }
+
+      if (fromNode?.type === 'asset' && toNode?.type === 'dependency') {
+        fromNode.value.dependencies.set(toNode.value.id, toNode.value);
+      }
+
+      graph.addEdge(from, to);
+    }
+  }
+
+  return {
+    assetGraph: graph,
+    changedAssets,
+  };
+}

package/src/requests/{AssetRequest.js → AssetRequest.ts}

@@ -1,5 +1,3 @@
-// @flow strict-local
-
 import type {ContentKey} from '@atlaspack/graph';
 import type {Async} from '@atlaspack/types';
 import type {StaticRunOpts} from '../RequestTracker';
@@ -21,18 +19,18 @@ import {fromProjectPath, fromProjectPathRelative} from '../projectPath';
 import {report} from '../ReporterRunner';
 import {requestTypes} from '../RequestTracker';
 import type {DevDepRequestResult} from './DevDepRequest';
+import {toEnvironmentId} from '../EnvironmentManager';
 
-type RunInput<TResult> = {|
-  input: AssetRequestInput,
-  ...StaticRunOpts<TResult>,
-|};
+type RunInput<TResult> = {
+  input: AssetRequestInput;
+} & StaticRunOpts<TResult>;
 
-export type AssetRequest = {|
-  id: ContentKey,
-  +type: typeof requestTypes.asset_request,
-  run: (RunInput<AssetRequestResult>) => Async<AssetRequestResult>,
-  input: AssetRequestInput,
-|};
+export type AssetRequest = {
+  id: ContentKey;
+  readonly type: typeof requestTypes.asset_request;
+  run: (arg1: RunInput<AssetRequestResult>) => Async<AssetRequestResult>;
+  input: AssetRequestInput;
+};
 
 export default function createAssetRequest(
   input: AssetRequestInput,
@@ -51,7 +49,7 @@ function getId(input: AssetRequestInput) {
   return hashString(
     type +
       fromProjectPathRelative(input.filePath) +
-      input.env.id +
+      toEnvironmentId(input.env) +
       String(input.isSource) +
       String(input.sideEffects) +
       (input.code ?? '') +
@@ -62,6 +60,7 @@ function getId(input: AssetRequestInput) {
   );
 }
 
+// @ts-expect-error TS7031
 async function run({input, api, farm, invalidateReason, options}) {
   report({
     type: 'buildProgress',
@@ -73,6 +72,7 @@ async function run({input, api, farm, invalidateReason, options}) {
   let start = Date.now();
   let {optionsRef, ...rest} = input;
   let {cachePath} = nullthrows(
+    // @ts-expect-error TS2347
     await api.runRequest<null, ConfigAndCachePath>(
       createAtlaspackConfigRequest(),
     ),
@@ -82,9 +82,12 @@ async function run({input, api, farm, invalidateReason, options}) {
     await Promise.all(
       api
         .getSubRequests()
+        // @ts-expect-error TS7006
         .filter((req) => req.requestType === requestTypes.dev_dep_request)
+        // @ts-expect-error TS7006
         .map(async (req) => [
           req.id,
+          // @ts-expect-error TS2347
           nullthrows(await api.getRequestResult<DevDepRequestResult>(req.id)),
         ]),
     ),
@@ -95,21 +98,24 @@ async function run({input, api, farm, invalidateReason, options}) {
     invalidateReason,
     devDeps: new Map(
       [...previousDevDepRequests.entries()]
-        .filter(([id]) => api.canSkipSubrequest(id))
-        .map(([, req]: [string, DevDepRequestResult]) => [
+        // @ts-expect-error TS2769
+        .filter(([id]: [any]) => api.canSkipSubrequest(id))
+        .map(([, req]: [any, any]) => [
           `${req.specifier}:${fromProjectPathRelative(req.resolveFrom)}`,
           req.hash,
         ]),
     ),
     invalidDevDeps: await Promise.all(
       [...previousDevDepRequests.entries()]
-        .filter(([id]) => !api.canSkipSubrequest(id))
-        .flatMap(([, req]: [string, DevDepRequestResult]) => {
+        // @ts-expect-error TS2769
+        .filter(([id]: [any]) => !api.canSkipSubrequest(id))
+        .flatMap(([, req]: [any, any]) => {
           return [
             {
               specifier: req.specifier,
               resolveFrom: req.resolveFrom,
             },
+            // @ts-expect-error TS7006
             ...(req.additionalInvalidations ?? []).map((i) => ({
               specifier: i.specifier,
               resolveFrom: i.resolveFrom,
@@ -127,7 +133,7 @@ async function run({input, api, farm, invalidateReason, options}) {
       configCachePath: cachePath,
       optionsRef,
       request,
-    }): TransformationResult);
+    })) as TransformationResult;
 
   let time = Date.now() - start;
   if (assets) {