@atlaspack/core 2.34.0 → 2.35.0

package/src/atlaspack-v3/AtlaspackV3.ts

@@ -6,10 +6,12 @@ import {
   atlaspackNapiCompleteSession,
   atlaspackNapiLoadBundleGraph,
   atlaspackNapiPackage,
+  atlaspackNapiUpdateBundleGraph,
   AtlaspackNapi,
   Lmdb,
   AtlaspackNapiOptions,
   CacheStats,
+  PackageOptions,
 } from '@atlaspack/rust';
 import {NapiWorkerPool} from './NapiWorkerPool';
 import ThrowableDiagnostic, {Diagnostic} from '@atlaspack/diagnostic';
@@ -119,12 +121,26 @@ export class AtlaspackV3 {
     ) as Promise<void>;
   }
 
+  updateBundleGraph(
+    bundleGraph: BundleGraph,
+    changedAssetIds: string[],
+  ): Promise<void> {
+    const nodesJson = bundleGraph.serializeAssetNodesForNative(changedAssetIds);
+    return atlaspackNapiUpdateBundleGraph(
+      this._atlaspack_napi,
+      nodesJson,
+    ) as Promise<void>;
+  }
+
   package(
     bundleId: string,
+    options?: PackageOptions,
   ): Promise<[RunPackagerRunnerResult, Diagnostic | null]> {
-    return atlaspackNapiPackage(this._atlaspack_napi, bundleId) as Promise<
-      [RunPackagerRunnerResult, Diagnostic | null]
-    >;
+    return atlaspackNapiPackage(
+      this._atlaspack_napi,
+      bundleId,
+      options,
+    ) as Promise<[RunPackagerRunnerResult, Diagnostic | null]>;
   }
 
   async respondToFsEvents(events: Array<Event>): Promise<boolean> {

package/src/requests/AssetGraphRequest.ts

@@ -18,7 +18,6 @@ import logger from '@atlaspack/logger';
 
 import invariant from 'assert';
 import nullthrows from 'nullthrows';
-import {getFeatureFlag} from '@atlaspack/feature-flags';
 import {PromiseQueue, setEqual} from '@atlaspack/utils';
 import {hashString} from '@atlaspack/rust';
 import ThrowableDiagnostic from '@atlaspack/diagnostic';
@@ -85,10 +84,12 @@ export default function createAssetGraphRequest(
       let assetGraphRequest = await builder.build();
 
       // early break for incremental bundling if production or flag is off;
-      assetGraphRequest.assetGraph.setDisableIncrementalBundling(
+      if (
         !input.options.shouldBundleIncrementally ||
-          input.options.mode === 'production',
-      );
+        input.options.mode === 'production'
+      ) {
+        assetGraphRequest.assetGraph.safeToIncrementallyBundle = false;
+      }
 
       if (
         !input.options.shouldBundleIncrementally ||
@@ -171,21 +172,12 @@ export class AssetGraphBuilder {
     this.lazyExcludes = lazyExcludes ?? [];
     this.skipSymbolProp = input.skipSymbolProp ?? false;
 
-    if (getFeatureFlag('cachePerformanceImprovements')) {
-      const key = hashString(
+    this.cacheKey =
+      hashString(
         `${ATLASPACK_VERSION}${name}${JSON.stringify(entries) ?? ''}${
           options.mode
         }${options.shouldBuildLazily ? 'lazy' : 'eager'}`,
-      );
-      this.cacheKey = `AssetGraph/${ATLASPACK_VERSION}/${options.mode}/${key}`;
-    } else {
-      this.cacheKey =
-        hashString(
-          `${ATLASPACK_VERSION}${name}${JSON.stringify(entries) ?? ''}${
-            options.mode
-          }${options.shouldBuildLazily ? 'lazy' : 'eager'}`,
-        ) + '-AssetGraph';
-    }
+      ) + '-AssetGraph';
 
     this.isSingleChangeRebuild =
       api
@@ -535,7 +527,6 @@ export class AssetGraphBuilder {
 
       if (didEntriesChange) {
         this.assetGraph.safeToIncrementallyBundle = false;
-        this.assetGraph.setNeedsBundling();
       }
     }
   }
@@ -590,12 +581,10 @@ export class AssetGraphBuilder {
             invariant(otherAsset.type === 'asset');
             if (!this._areDependenciesEqualForAssets(asset, otherAsset.value)) {
               this.assetGraph.safeToIncrementallyBundle = false;
-              this.assetGraph.setNeedsBundling();
             }
           } else {
             // adding a new entry or dependency
             this.assetGraph.safeToIncrementallyBundle = false;
-            this.assetGraph.setNeedsBundling();
           }
         }
         this.changedAssets.set(asset.id, asset);
@@ -604,7 +593,6 @@ export class AssetGraphBuilder {
       this.assetGraph.resolveAssetGroup(input, assets, request.id);
     } else {
       this.assetGraph.safeToIncrementallyBundle = false;
-      this.assetGraph.setNeedsBundling();
     }
 
     this.isSingleChangeRebuild = false;

package/src/requests/AssetGraphRequestRust.ts

@@ -238,7 +238,32 @@ export function getAssetGraph(
     return envId;
   };
 
-  function updateNode(newNode: AssetGraphNode, isUpdateNode: boolean) {
+  function describeNode(node: AssetGraphNode): Record<string, unknown> {
+    const base = {type: node.type, id: node.id};
+    if (node.type === 'asset') {
+      return {
+        ...base,
+        filePath: node.value.filePath,
+        fileType: node.value.type,
+        pipeline: node.value.pipeline,
+      };
+    } else if (node.type === 'dependency') {
+      return {
+        ...base,
+        specifier: node.value.specifier,
+        specifierType: node.value.specifierType,
+        sourceAssetId: node.value.sourceAssetId,
+        sourcePath: node.value.sourcePath,
+      };
+    }
+    return base;
+  }
+
+  function updateNode(
+    newNode: AssetGraphNode,
+    isUpdateNode: boolean,
+    index: number,
+  ) {
     if (isUpdateNode) {
       let existingNode = graph.getNodeByContentKey(newNode.id);
 
@@ -246,7 +271,34 @@ export function getAssetGraph(
 
       Object.assign(existingNode, newNode);
     } else {
-      graph.addNodeByContentKey(newNode.id, newNode);
+      try {
+        graph.addNodeByContentKey(newNode.id, newNode);
+      } catch (e) {
+        if (
+          e instanceof Error &&
+          e.message.includes('already has content key')
+        ) {
+          let existingNode = graph.getNodeByContentKey(newNode.id);
+          let diagnostics = {
+            contentKey: newNode.id,
+            newNode: describeNode(newNode),
+            existingNode: existingNode ? describeNode(existingNode) : null,
+            iterationIndex: index,
+            totalSerializedNodes: nodesCount,
+            newNodesCount: serializedGraph.nodes.length,
+            updatesCount: serializedGraph.updates.length,
+            edgesCount: serializedGraph.edges.length,
+            hadPreviousGraph: !!prevAssetGraph,
+            safeToSkipBundling: serializedGraph.safeToSkipBundling,
+            graphNodeCount: graph._contentKeyToNodeId.size,
+          };
+
+          throw new Error(
+            `Graph already has content key '${newNode.id}'. Diagnostics: ${JSON.stringify(diagnostics, null, 2)}`,
+          );
+        }
+        throw e;
+      }
     }
   }
 
@@ -305,7 +357,7 @@ export function getAssetGraph(
         usedSymbolsUpDirty: true,
         value: asset,
       };
-      updateNode(assetNode, isUpdateNode);
+      updateNode(assetNode, isUpdateNode, index);
     } else if (node.type === 'dependency') {
       let {dependency, id} = node.value;
 
@@ -325,12 +377,12 @@ export function getAssetGraph(
 
       if (node.used_symbols_up) {
         for (let usedSymbol of node.used_symbols_up) {
-          // Transform Rust UsedSymbol { symbol: Symbol, asset: string }
-          // to JS format { symbol: string, asset: string } where symbol is the exported name
+          // Transform Rust UsedSymbol { symbol: Symbol, asset: string, resolved_symbol: string }
+          // to JS format { symbol: string, asset: string } where symbol is the resolved name
           const exportedName = usedSymbol.symbol.exported;
           usedSymbolsUp.set(exportedName, {
             asset: usedSymbol.asset,
-            symbol: exportedName,
+            symbol: usedSymbol.resolved_symbol ?? exportedName,
           });
         }
       }
@@ -355,7 +407,7 @@ export function getAssetGraph(
         value: dependency,
       };
 
-      updateNode(depNode, isUpdateNode);
+      updateNode(depNode, isUpdateNode, index);
     }
   }
 

package/src/requests/AtlaspackBuildRequest.ts

@@ -23,7 +23,6 @@ import {tracer} from '@atlaspack/profiler';
 import {requestTypes} from '../RequestTracker';
 import {getFeatureFlag} from '@atlaspack/feature-flags';
 import {fromEnvironmentId} from '../EnvironmentManager';
-import invariant from 'assert';
 
 type AtlaspackBuildRequestInput = {
   optionsRef: SharedReference;
@@ -87,12 +86,16 @@ async function run({
           signal,
         });
 
-  let {bundleGraph, changedAssets, assetRequests}: BundleGraphResult =
-    await api.runRequest(bundleGraphRequest, {
-      force:
-        Boolean(rustAtlaspack) ||
-        (options.shouldBuildLazily && requestedAssetIds.size > 0),
-    });
+  let {
+    bundleGraph,
+    changedAssets,
+    assetRequests,
+    didIncrementallyBundle,
+  }: BundleGraphResult = await api.runRequest(bundleGraphRequest, {
+    force:
+      Boolean(rustAtlaspack) ||
+      (options.shouldBuildLazily && requestedAssetIds.size > 0),
+  });
 
   if (
     getFeatureFlag('nativePackager') &&
@@ -110,7 +113,12 @@ async function run({
       }
     });
     if (hasSupportedTarget) {
-      await rustAtlaspack.loadBundleGraph(bundleGraph);
+      if (didIncrementallyBundle) {
+        const changedAssetIds = Array.from(changedAssets.keys());
+        await rustAtlaspack.updateBundleGraph(bundleGraph, changedAssetIds);
+      } else {
+        await rustAtlaspack.loadBundleGraph(bundleGraph);
+      }
     }
   }
 

package/src/requests/BundleGraphRequest.ts

@@ -14,7 +14,6 @@ import type {ConfigAndCachePath} from './AtlaspackConfigRequest';
 
 import fs from 'fs';
 import invariant from 'assert';
-import assert from 'assert';
 import nullthrows from 'nullthrows';
 import {instrumentAsync, PluginLogger} from '@atlaspack/logger';
 import {getFeatureFlag} from '@atlaspack/feature-flags';
@@ -36,13 +35,8 @@ import {
   createDevDependency,
   getDevDepRequests,
   invalidateDevDeps,
-  runDevDepRequest,
 } from './DevDepRequest';
-import {
-  loadPluginConfig,
-  runConfigRequest,
-  PluginWithLoadConfig,
-} from './ConfigRequest';
+import {PluginWithLoadConfig} from './ConfigRequest';
 import {fromProjectPathRelative} from '../projectPath';
 import {
   validateBundles,
@@ -74,9 +68,9 @@ type RunInput = {
 // TODO: Rename to BundleGraphRequestResult
 export type BundleGraphResult = {
   bundleGraph: InternalBundleGraph;
-  assetGraphBundlingVersion: number;
   changedAssets: Map<string, Asset>;
   assetRequests: Array<AssetGroup>;
+  didIncrementallyBundle: boolean;
 };
 
 type BundleGraphRequest = {
@@ -214,7 +208,6 @@ export default function createBundleGraphRequest(
 
       if (subRequestsInvalid) {
         assetGraph.safeToIncrementallyBundle = false;
-        assetGraph.setNeedsBundling();
       }
 
       let configResult = nullthrows(
@@ -283,21 +276,12 @@ class BundlerRunner {
     this.pluginOptions = new PluginOptions(
       optionsProxy(this.options, api.invalidateOnOptionChange),
     );
-    if (getFeatureFlag('cachePerformanceImprovements')) {
-      const key = hashString(
+    this.cacheKey =
+      hashString(
         `${ATLASPACK_VERSION}:BundleGraph:${
           JSON.stringify(options.entries) ?? ''
         }${options.mode}${options.shouldBuildLazily ? 'lazy' : 'eager'}`,
-      );
-      this.cacheKey = `BundleGraph/${ATLASPACK_VERSION}/${options.mode}/${key}`;
-    } else {
-      this.cacheKey =
-        hashString(
-          `${ATLASPACK_VERSION}:BundleGraph:${
-            JSON.stringify(options.entries) ?? ''
-          }${options.mode}${options.shouldBuildLazily ? 'lazy' : 'eager'}`,
-        ) + '-BundleGraph';
-    }
+      ) + '-BundleGraph';
   }
 
   async loadConfigs() {
@@ -344,7 +328,7 @@ class BundlerRunner {
       type: 'buildProgress',
       phase: 'bundling',
     });
-
+    let didIncrementallyBundle = false;
     await this.loadConfigs();
 
     let plugin = await this.config.getBundler();
@@ -354,14 +338,10 @@ class BundlerRunner {
     const previousBundleGraphResult: BundleGraphResult | null | undefined =
       await this.api.getPreviousResult();
     const canIncrementallyBundle =
-      previousBundleGraphResult?.assetGraphBundlingVersion != null &&
-      graph.canIncrementallyBundle(
-        previousBundleGraphResult.assetGraphBundlingVersion,
-      );
+      previousBundleGraphResult != null && graph.canIncrementallyBundle();
 
     if (graph.safeToIncrementallyBundle && previousBundleGraphResult == null) {
       graph.safeToIncrementallyBundle = false;
-      graph.setNeedsBundling();
     }
 
     let internalBundleGraph;
@@ -382,6 +362,7 @@ class BundlerRunner {
           invariant(changedAssetNode.type === 'asset');
           internalBundleGraph.updateAsset(changedAssetNode);
         }
+        didIncrementallyBundle = true;
       } else {
         internalBundleGraph = InternalBundleGraph.fromAssetGraph(
           graph,
@@ -474,9 +455,9 @@ class BundlerRunner {
         this.api.storeResult(
           {
             bundleGraph: internalBundleGraph,
-            assetGraphBundlingVersion: graph.getBundlingVersion(),
             changedAssets: new Map(),
             assetRequests: [],
+            didIncrementallyBundle,
           },
           this.cacheKey,
         );
@@ -560,18 +541,18 @@ class BundlerRunner {
     this.api.storeResult(
       {
         bundleGraph: internalBundleGraph,
-        assetGraphBundlingVersion: graph.getBundlingVersion(),
         changedAssets: new Map(),
         assetRequests: [],
+        didIncrementallyBundle,
       },
       this.cacheKey,
     );
 
     return {
       bundleGraph: internalBundleGraph,
-      assetGraphBundlingVersion: graph.getBundlingVersion(),
       changedAssets: changedRuntimes,
       assetRequests,
+      didIncrementallyBundle,
     };
   }
 }

package/src/requests/BundleGraphRequestRust.ts

@@ -181,10 +181,9 @@ export default function createBundleGraphRequestRust(
 
       return {
         bundleGraph,
-        // Not accurate yet — ok for now.
-        assetGraphBundlingVersion: 0,
         changedAssets: changedRuntimes,
         assetRequests: [],
+        didIncrementallyBundle: false,
       };
     },
     input,

package/src/requests/PackageRequest.ts

@@ -72,7 +72,7 @@ async function run({input, api, farm, rustAtlaspack}: RunInput<BundleInfo>) {
     bundle.type === 'js'
   ) {
     // Once this actually does something, the code below will be in an `else` block (i.e. we'll only run one or the other)
-    let result = await rustAtlaspack.package(bundle.id);
+    let result = await rustAtlaspack.package(bundle.id, {inlineRequires: true});
     let error: Diagnostic | null = null;
     [packagingResult, error] = result;
     if (error) {

package/src/requests/TargetRequest.ts

@@ -332,6 +332,7 @@ export class TargetResolver {
               publicUrl:
                 descriptor.publicUrl ??
                 this.options.defaultTargetOptions.publicUrl,
+              inlineRequires: descriptor.inlineRequires ?? false,
               env: createEnvironment({
                 engines: descriptor.engines,
                 context: descriptor.context,
@@ -426,6 +427,7 @@ export class TargetResolver {
               this.options.serveOptions.distDir,
             ),
             publicUrl: this.options.defaultTargetOptions.publicUrl ?? '/',
+            inlineRequires: false,
             env: createEnvironment({
               context: 'browser',
               engines: {
@@ -930,6 +932,7 @@ export class TargetResolver {
           distEntry,
           publicUrl:
             descriptor.publicUrl ?? this.options.defaultTargetOptions.publicUrl,
+          inlineRequires: descriptor.inlineRequires ?? false,
           env: createEnvironment({
             engines: descriptor.engines ?? pkgEngines,
             // @ts-expect-error TS2322
@@ -1131,6 +1134,7 @@ export class TargetResolver {
           distEntry,
           publicUrl:
             descriptor.publicUrl ?? this.options.defaultTargetOptions.publicUrl,
+          inlineRequires: descriptor.inlineRequires ?? false,
           env: createEnvironment({
             engines: descriptor.engines ?? pkgEngines,
             context: descriptor.context,
@@ -1205,6 +1209,7 @@ export class TargetResolver {
             path.join(pkgDir, DEFAULT_DIST_DIRNAME),
           ),
         publicUrl: this.options.defaultTargetOptions.publicUrl,
+        inlineRequires: false,
         env: createEnvironment({
           engines: pkgEngines,
           // @ts-expect-error TS2322

package/src/requests/WriteBundleRequest.ts

@@ -185,9 +185,7 @@ async function run({input, options, api}) {
     bundleReplacements,
   );
 
-  const hasSourceMap = getFeatureFlag('cachePerformanceImprovements')
-    ? await options.cache.hasLargeBlob(mapKey)
-    : await options.cache.has(mapKey);
+  const hasSourceMap = await options.cache.has(mapKey);
   if (mapKey && env.sourceMap && !env.sourceMap.inline && hasSourceMap) {
     let mapStream: Readable;
     if (
@@ -195,9 +193,7 @@ async function run({input, options, api}) {
       bundleReplacements &&
       bundleReplacements.length > 0
     ) {
-      const mapEntry = getFeatureFlag('cachePerformanceImprovements')
-        ? await options.cache.getLargeBlob(mapKey)
-        : await options.cache.getBlob(mapKey);
+      const mapEntry = await options.cache.getBlob(mapKey);
       const mapBuffer = Buffer.isBuffer(mapEntry)
         ? mapEntry
         : Buffer.from(mapEntry);
@@ -219,9 +215,7 @@ async function run({input, options, api}) {
         ),
       );
     } else {
-      const mapEntry = getFeatureFlag('cachePerformanceImprovements')
-        ? await options.cache.getLargeBlob(mapKey)
-        : await options.cache.getBlob(mapKey);
+      const mapEntry = await options.cache.getBlob(mapKey);
       mapStream = blobToStream(mapEntry);
     }
     await writeFiles(

package/src/resolveOptions.ts

@@ -184,10 +184,8 @@ export default async function resolveOptions(
     const needsRustLmdbCache =
       getFeatureFlag('atlaspackV3') || getFeatureFlag('nativePackager');
 
-    if (!getFeatureFlag('cachePerformanceImprovements')) {
-      if (!needsRustLmdbCache && !(outputFS instanceof NodeFS)) {
-        return new FSCache(outputFS, cacheDir);
-      }
+    if (!needsRustLmdbCache && !(outputFS instanceof NodeFS)) {
+      return new FSCache(outputFS, cacheDir);
     }
 
     return new LMDBLiteCache(cacheDir);

package/src/types.ts

@@ -112,6 +112,7 @@ export type Target = {
   loc?: InternalSourceLocation | null | undefined;
   pipeline?: string;
   source?: FilePath | Array<FilePath>;
+  inlineRequires?: boolean;
 };
 
 export const SpecifierType = {

package/test/AssetGraph.test.ts

@@ -687,36 +687,4 @@ describe('AssetGraph', () => {
     invariant(node.type === 'asset_group');
     assert(!node.hasDeferred);
   });
-
-  it('should serialize the bundling version and incremental bundling flag', () => {
-    const graph = new AssetGraph();
-    graph.setDisableIncrementalBundling(true);
-    graph.setNeedsBundling();
-    const serialized = serialize(graph);
-    const deserialized = deserialize(serialized);
-
-    assert.equal(deserialized.getBundlingVersion(), 1);
-    assert.equal(deserialized.testing_getDisableIncrementalBundling(), true);
-  });
-
-  describe('setNeedsBundling', () => {
-    it('should increment the bundling version', () => {
-      const graph = new AssetGraph();
-      assert.equal(graph.getBundlingVersion(), 0);
-      graph.setNeedsBundling();
-      assert.equal(graph.getBundlingVersion(), 1);
-      graph.setNeedsBundling();
-      assert.equal(graph.getBundlingVersion(), 2);
-    });
-  });
-
-  describe('canIncrementallyBundle', () => {
-    it('should return true if the bundling version has changed', () => {
-      const graph = new AssetGraph();
-      const lastVersion = graph.getBundlingVersion();
-      assert.equal(graph.canIncrementallyBundle(lastVersion), true);
-      graph.setNeedsBundling();
-      assert.equal(graph.canIncrementallyBundle(lastVersion), false);
-    });
-  });
 });