@atlaspack/core 2.12.1-canary.3354

package/src/requests/AssetGraphRequestRust.js

@@ -0,0 +1,222 @@
+// @flow strict-local
+
+import invariant from 'assert';
+
+import ThrowableDiagnostic from '@atlaspack/diagnostic';
+import type {Async} from '@atlaspack/types';
+
+import AssetGraph, {nodeFromAssetGroup} from '../AssetGraph';
+import type {AtlaspackV3} from '../atlaspack-v3';
+import {toProjectPath} from '../projectPath';
+import {requestTypes, type StaticRunOpts} from '../RequestTracker';
+import {propagateSymbols} from '../SymbolPropagation';
+
+import type {
+  AssetGraphRequestInput,
+  AssetGraphRequestResult,
+} from './AssetGraphRequest';
+
+type RunInput = {|
+  input: AssetGraphRequestInput,
+  ...StaticRunOpts<AssetGraphRequestResult>,
+|};
+
+type AssetGraphRequest = {|
+  id: string,
+  +type: typeof requestTypes.asset_graph_request,
+  run: RunInput => Async<AssetGraphRequestResult>,
+  input: AssetGraphRequestInput,
+|};
+
+export function createAssetGraphRequestRust(
+  rustAtlaspack: AtlaspackV3,
+): (input: AssetGraphRequestInput) => AssetGraphRequest {
+  return input => ({
+    type: requestTypes.asset_graph_request,
+    id: input.name,
+    run: async input => {
+      let options = input.options;
+      let serializedAssetGraph;
+      try {
+        serializedAssetGraph = await rustAtlaspack.buildAssetGraph();
+      } catch (err) {
+        throw new ThrowableDiagnostic({
+          diagnostic: err,
+        });
+      }
+
+      let {assetGraph, cachedAssets, changedAssets} = getAssetGraph(
+        serializedAssetGraph,
+        options,
+      );
+
+      // TODO: Make it a bulk transaction
+      await Promise.all(
+        Array.from(cachedAssets.entries(), ([id, code]) =>
+          options.cache.setBlob(id, Buffer.from(code)),
+        ),
+      );
+
+      let changedAssetsPropagation = new Set(changedAssets.keys());
+      let errors = propagateSymbols({
+        options,
+        assetGraph,
+        changedAssetsPropagation,
+        assetGroupsWithRemovedParents: new Set(),
+        previousErrors: new Map(), //this.previousSymbolPropagationErrors,
+      });
+
+      if (errors.size > 0) {
+        // Just throw the first error. Since errors can bubble (e.g. reexporting a reexported symbol also fails),
+        // determining which failing export is the root cause is nontrivial (because of circular dependencies).
+        throw new ThrowableDiagnostic({
+          diagnostic: [...errors.values()][0],
+        });
+      }
+
+      return {
+        assetGraph,
+        assetRequests: [],
+        assetGroupsWithRemovedParents: new Set(),
+        changedAssets,
+        changedAssetsPropagation,
+        previousSymbolPropagationErrors: undefined,
+      };
+    },
+    input,
+  });
+}
+
+function getAssetGraph(serializedGraph, options) {
+  let graph = new AssetGraph({
+    _contentKeyToNodeId: new Map(),
+    _nodeIdToContentKey: new Map(),
+  });
+
+  graph.safeToIncrementallyBundle = false;
+
+  let cachedAssets = new Map();
+  let changedAssets = new Map();
+  let entry = 0;
+  for (let node of serializedGraph.nodes) {
+    if (node.type === 'root') {
+      let index = graph.addNodeByContentKey('@@root', {
+        id: '@@root',
+        type: 'root',
+        value: null,
+      });
+
+      graph.setRootNodeId(index);
+    } else if (node.type === 'entry') {
+      let id = 'entry:' + ++entry;
+
+      graph.addNodeByContentKey(id, {
+        id: id,
+        type: 'root',
+        value: null,
+      });
+    } else if (node.type === 'asset') {
+      let id = node.value.id;
+      let asset = node.value.asset;
+
+      asset = {
+        ...asset,
+        id,
+        committed: true,
+        contentKey: id,
+        filePath: toProjectPath(options.projectRoot, asset.filePath),
+        symbols: asset.hasSymbols
+          ? new Map(
+              asset.symbols.map(({exported, ...symbol}) => [exported, symbol]),
+            )
+          : null,
+      };
+
+      cachedAssets.set(id, asset.code);
+      changedAssets.set(id, asset);
+
+      graph.addNodeByContentKey(id, {
+        id,
+        type: 'asset',
+        usedSymbols: new Set(),
+        usedSymbolsDownDirty: true,
+        usedSymbolsUpDirty: true,
+        value: asset,
+      });
+    } else if (node.type === 'dependency') {
+      let id = node.value.id;
+      let dependency = node.value.dependency;
+
+      dependency = {
+        ...dependency,
+        id,
+        contentKey: id,
+        sourcePath: dependency.sourcePath
+          ? toProjectPath(options.projectRoot, dependency.sourcePath)
+          : null,
+        symbols: dependency.hasSymbols
+          ? new Map(
+              dependency.symbols.map(({exported, ...symbol}) => [
+                exported,
+                symbol,
+              ]),
+            )
+          : null,
+      };
+      let usedSymbolsDown = new Set();
+      let usedSymbolsUp = new Map();
+      if (dependency.isEntry && dependency.isLibrary) {
+        usedSymbolsDown.add('*');
+        usedSymbolsUp.set('*', undefined);
+      }
+
+      graph.addNodeByContentKey(id, {
+        id,
+        type: 'dependency',
+        deferred: false,
+        excluded: false,
+        hasDeferred: node.has_deferred,
+        usedSymbolsDown,
+        usedSymbolsDownDirty: true,
+        usedSymbolsUp,
+        usedSymbolsUpDirtyDown: true,
+        usedSymbolsUpDirtyUp: true,
+        value: dependency,
+      });
+    }
+  }
+
+  for (let i = 0; i < serializedGraph.edges.length; i += 2) {
+    let from = serializedGraph.edges[i];
+    let to = serializedGraph.edges[i + 1];
+    let fromNode = graph.getNode(from);
+    if (fromNode?.type === 'dependency') {
+      let toNode = graph.getNode(to);
+      invariant(toNode?.type === 'asset');
+
+      // For backwards compatibility, create asset group node if needed.
+      let assetGroupNode = nodeFromAssetGroup({
+        filePath: toNode.value.filePath,
+        env: fromNode.value.env,
+        pipeline: toNode.value.pipeline,
+        sideEffects: Boolean(toNode.value.sideEffects),
+      });
+
+      let index = graph.addNodeByContentKeyIfNeeded(
+        assetGroupNode.id,
+        assetGroupNode,
+      );
+
+      graph.addEdge(from, index);
+      graph.addEdge(index, to);
+    } else {
+      graph.addEdge(from, to);
+    }
+  }
+
+  return {
+    assetGraph: graph,
+    cachedAssets,
+    changedAssets,
+  };
+}

package/src/requests/AssetRequest.js

@@ -0,0 +1,179 @@
+// @flow strict-local
+
+import type {ContentKey} from '@atlaspack/graph';
+import type {Async} from '@atlaspack/types';
+import type {StaticRunOpts} from '../RequestTracker';
+import type {
+  AssetRequestInput,
+  AssetRequestResult,
+  TransformationRequest,
+} from '../types';
+import type {ConfigAndCachePath} from './AtlaspackConfigRequest';
+import type {TransformationResult} from '../Transformation';
+
+import nullthrows from 'nullthrows';
+import ThrowableDiagnostic from '@atlaspack/diagnostic';
+import {hashString} from '@atlaspack/rust';
+import createAtlaspackConfigRequest from './AtlaspackConfigRequest';
+import {runDevDepRequest} from './DevDepRequest';
+import {runConfigRequest} from './ConfigRequest';
+import {fromProjectPath, fromProjectPathRelative} from '../projectPath';
+import {report} from '../ReporterRunner';
+import {requestTypes} from '../RequestTracker';
+import type {DevDepRequestResult} from './DevDepRequest';
+
+type RunInput<TResult> = {|
+  input: AssetRequestInput,
+  ...StaticRunOpts<TResult>,
+|};
+
+export type AssetRequest = {|
+  id: ContentKey,
+  +type: typeof requestTypes.asset_request,
+  run: (RunInput<AssetRequestResult>) => Async<AssetRequestResult>,
+  input: AssetRequestInput,
+|};
+
+export default function createAssetRequest(
+  input: AssetRequestInput,
+): AssetRequest {
+  return {
+    type: requestTypes.asset_request,
+    id: getId(input),
+    run,
+    input,
+  };
+}
+
+const type = 'asset_request';
+
+function getId(input: AssetRequestInput) {
+  // eslint-disable-next-line no-unused-vars
+  let {optionsRef, ...hashInput} = input;
+  return hashString(
+    type +
+      fromProjectPathRelative(input.filePath) +
+      input.env.id +
+      String(input.isSource) +
+      String(input.sideEffects) +
+      (input.code ?? '') +
+      ':' +
+      (input.pipeline ?? '') +
+      ':' +
+      (input.query ?? ''),
+  );
+}
+
+async function run({input, api, farm, invalidateReason, options}) {
+  report({
+    type: 'buildProgress',
+    phase: 'transforming',
+    filePath: fromProjectPath(options.projectRoot, input.filePath),
+  });
+
+  api.invalidateOnFileUpdate(input.filePath);
+  let start = Date.now();
+  let {optionsRef, ...rest} = input;
+  let {cachePath} = nullthrows(
+    await api.runRequest<null, ConfigAndCachePath>(
+      createAtlaspackConfigRequest(),
+    ),
+  );
+
+  let previousDevDepRequests: Map<string, DevDepRequestResult> = new Map(
+    await Promise.all(
+      api
+        .getSubRequests()
+        .filter(req => req.requestType === requestTypes.dev_dep_request)
+        .map(async req => [
+          req.id,
+          nullthrows(await api.getRequestResult<DevDepRequestResult>(req.id)),
+        ]),
+    ),
+  );
+
+  let request: TransformationRequest = {
+    ...rest,
+    invalidateReason,
+    devDeps: new Map(
+      [...previousDevDepRequests.entries()]
+        .filter(([id]) => api.canSkipSubrequest(id))
+        .map(([, req]: [string, DevDepRequestResult]) => [
+          `${req.specifier}:${fromProjectPathRelative(req.resolveFrom)}`,
+          req.hash,
+        ]),
+    ),
+    invalidDevDeps: await Promise.all(
+      [...previousDevDepRequests.entries()]
+        .filter(([id]) => !api.canSkipSubrequest(id))
+        .flatMap(([, req]: [string, DevDepRequestResult]) => {
+          return [
+            {
+              specifier: req.specifier,
+              resolveFrom: req.resolveFrom,
+            },
+            ...(req.additionalInvalidations ?? []).map(i => ({
+              specifier: i.specifier,
+              resolveFrom: i.resolveFrom,
+            })),
+          ];
+        }),
+    ),
+  };
+
+  let {assets, configRequests, error, invalidations, devDepRequests} =
+    (await farm.createHandle(
+      'runTransform',
+      input.isSingleChangeRebuild,
+    )({
+      configCachePath: cachePath,
+      optionsRef,
+      request,
+    }): TransformationResult);
+
+  let time = Date.now() - start;
+  if (assets) {
+    for (let asset of assets) {
+      asset.stats.time = time;
+    }
+  }
+
+  for (let filePath of invalidations.invalidateOnFileChange) {
+    api.invalidateOnFileUpdate(filePath);
+    api.invalidateOnFileDelete(filePath);
+  }
+
+  for (let invalidation of invalidations.invalidateOnFileCreate) {
+    api.invalidateOnFileCreate(invalidation);
+  }
+
+  for (let env of invalidations.invalidateOnEnvChange) {
+    api.invalidateOnEnvChange(env);
+  }
+
+  for (let option of invalidations.invalidateOnOptionChange) {
+    api.invalidateOnOptionChange(option);
+  }
+
+  if (invalidations.invalidateOnStartup) {
+    api.invalidateOnStartup();
+  }
+
+  if (invalidations.invalidateOnBuild) {
+    api.invalidateOnBuild();
+  }
+
+  for (let devDepRequest of devDepRequests) {
+    await runDevDepRequest(api, devDepRequest);
+  }
+
+  for (let configRequest of configRequests) {
+    await runConfigRequest(api, configRequest);
+  }
+
+  if (error != null) {
+    throw new ThrowableDiagnostic({diagnostic: error});
+  } else {
+    return nullthrows(assets);
+  }
+}

package/src/requests/AtlaspackBuildRequest.js

@@ -0,0 +1,109 @@
+// @flow strict-local
+
+import type {ContentKey} from '@atlaspack/graph';
+import type {Async} from '@atlaspack/types';
+import type {SharedReference} from '@atlaspack/workers';
+
+import type {StaticRunOpts} from '../RequestTracker';
+import type {Asset, AssetGroup, PackagedBundleInfo} from '../types';
+import type BundleGraph from '../BundleGraph';
+
+import createBundleGraphRequest, {
+  type BundleGraphResult,
+} from './BundleGraphRequest';
+import createWriteBundlesRequest from './WriteBundlesRequest';
+import {assertSignalNotAborted} from '../utils';
+import dumpGraphToGraphViz from '../dumpGraphToGraphViz';
+import {bundleGraphEdgeTypes} from '../BundleGraph';
+import {report} from '../ReporterRunner';
+import IBundleGraph from '../public/BundleGraph';
+import {NamedBundle} from '../public/Bundle';
+import {assetFromValue} from '../public/Asset';
+
+import {tracer} from '@atlaspack/profiler';
+import {requestTypes} from '../RequestTracker';
+
+type AtlaspackBuildRequestInput = {|
+  optionsRef: SharedReference,
+  requestedAssetIds: Set<string>,
+  signal?: AbortSignal,
+|};
+
+export type AtlaspackBuildRequestResult = {|
+  bundleGraph: BundleGraph,
+  bundleInfo: Map<string, PackagedBundleInfo>,
+  changedAssets: Map<string, Asset>,
+  assetRequests: Array<AssetGroup>,
+|};
+
+type RunInput<TResult> = {|
+  input: AtlaspackBuildRequestInput,
+  ...StaticRunOpts<TResult>,
+|};
+
+export type AtlaspackBuildRequest = {|
+  id: ContentKey,
+  +type: typeof requestTypes.atlaspack_build_request,
+  run: (
+    RunInput<AtlaspackBuildRequestResult>,
+  ) => Async<AtlaspackBuildRequestResult>,
+  input: AtlaspackBuildRequestInput,
+|};
+
+export default function createAtlaspackBuildRequest(
+  input: AtlaspackBuildRequestInput,
+): AtlaspackBuildRequest {
+  return {
+    type: requestTypes.atlaspack_build_request,
+    id: 'atlaspack_build_request',
+    run,
+    input,
+  };
+}
+
+async function run({input, api, options}) {
+  let {optionsRef, requestedAssetIds, signal} = input;
+
+  let bundleGraphRequest = createBundleGraphRequest({
+    optionsRef,
+    requestedAssetIds,
+    signal,
+  });
+
+  let {bundleGraph, changedAssets, assetRequests}: BundleGraphResult =
+    await api.runRequest(bundleGraphRequest, {
+      force: options.shouldBuildLazily && requestedAssetIds.size > 0,
+    });
+
+  // $FlowFixMe Added in Flow 0.121.0 upgrade in #4381 (Windows only)
+  dumpGraphToGraphViz(bundleGraph._graph, 'BundleGraph', bundleGraphEdgeTypes);
+
+  await report({
+    type: 'buildProgress',
+    phase: 'bundled',
+    bundleGraph: new IBundleGraph(
+      bundleGraph,
+      (bundle, bundleGraph, options) =>
+        NamedBundle.get(bundle, bundleGraph, options),
+      options,
+    ),
+    changedAssets: new Map(
+      Array.from(changedAssets).map(([id, asset]) => [
+        id,
+        assetFromValue(asset, options),
+      ]),
+    ),
+  });
+
+  let packagingMeasurement = tracer.createMeasurement('packaging');
+  let writeBundlesRequest = createWriteBundlesRequest({
+    bundleGraph,
+    optionsRef,
+  });
+
+  let bundleInfo = await api.runRequest(writeBundlesRequest);
+  packagingMeasurement && packagingMeasurement.end();
+  assertSignalNotAborted(signal);
+
+  return {bundleGraph, bundleInfo, changedAssets, assetRequests};
+}