@atlaspack/core 2.33.1 → 2.35.0

package/src/requests/WriteBundleRequest.ts

@@ -13,6 +13,7 @@ import type {ProjectPath} from '../projectPath';
 import {HASH_REF_HASH_LEN, HASH_REF_PREFIX} from '../constants';
 import nullthrows from 'nullthrows';
 import path from 'path';
+import url from 'url';
 import {NamedBundle} from '../public/Bundle';
 import {blobToStream, TapStream} from '@atlaspack/utils';
 import {Readable, Transform, pipeline} from 'stream';
@@ -40,9 +41,18 @@ import {PluginTracer, tracer} from '@atlaspack/profiler';
 import {requestTypes} from '../RequestTracker';
 import {getFeatureFlag} from '@atlaspack/feature-flags';
 import {fromEnvironmentId} from '../EnvironmentManager';
+import SourceMap from '@atlaspack/source-map';
 
 const HASH_REF_PREFIX_LEN = HASH_REF_PREFIX.length;
 const BOUNDARY_LENGTH = HASH_REF_PREFIX.length + 32 - 1;
+const HASH_REF_PLACEHOLDER_LEN = HASH_REF_PREFIX_LEN + HASH_REF_HASH_LEN;
+
+export type HashRefReplacement = {
+  line: number;
+  column: number;
+  originalLength: number;
+  newLength: number;
+};
 
 type WriteBundleRequestInput = {
   bundleGraph: BundleGraph;
@@ -158,6 +168,9 @@ async function run({input, options, api}) {
   let {devDeps, invalidDevDeps} = await getDevDepRequests(api);
   invalidateDevDeps(invalidDevDeps, options, config);
 
+  const bundleReplacements = getFeatureFlag('fixSourceMapHashRefs')
+    ? []
+    : undefined;
   await writeFiles(
     contentStream,
     info,
@@ -169,17 +182,44 @@ async function run({input, options, api}) {
     writeOptions,
     devDeps,
     api,
+    bundleReplacements,
   );
 
-  const hasSourceMap = getFeatureFlag('cachePerformanceImprovements')
-    ? await options.cache.hasLargeBlob(mapKey)
-    : await options.cache.has(mapKey);
+  const hasSourceMap = await options.cache.has(mapKey);
   if (mapKey && env.sourceMap && !env.sourceMap.inline && hasSourceMap) {
-    const mapEntry = getFeatureFlag('cachePerformanceImprovements')
-      ? await options.cache.getLargeBlob(mapKey)
-      : await options.cache.getBlob(mapKey);
+    let mapStream: Readable;
+    if (
+      getFeatureFlag('fixSourceMapHashRefs') &&
+      bundleReplacements &&
+      bundleReplacements.length > 0
+    ) {
+      const mapEntry = await options.cache.getBlob(mapKey);
+      const mapBuffer = Buffer.isBuffer(mapEntry)
+        ? mapEntry
+        : Buffer.from(mapEntry);
+      const projectRoot =
+        typeof options.projectRoot === 'string'
+          ? options.projectRoot
+          : String(options.projectRoot);
+      const sourceMap = new SourceMap(projectRoot, mapBuffer);
+      applyReplacementsToSourceMap(sourceMap, bundleReplacements);
+      const mapJson = await sourceMap.stringify({
+        format: 'string',
+        file: name,
+        sourceRoot: computeSourceMapRoot(bundle, options),
+      });
+      mapStream = blobToStream(
+        Buffer.from(
+          typeof mapJson === 'string' ? mapJson : JSON.stringify(mapJson),
+          'utf8',
+        ),
+      );
+    } else {
+      const mapEntry = await options.cache.getBlob(mapKey);
+      mapStream = blobToStream(mapEntry);
+    }
     await writeFiles(
-      blobToStream(mapEntry),
+      mapStream,
       info,
       hashRefToNameHash,
       options,
@@ -206,6 +246,88 @@ async function run({input, options, api}) {
   return res;
 }
 
+export function applyReplacementsToSourceMap(
+  sourceMap: SourceMap,
+  replacements: HashRefReplacement[],
+): void {
+  if (replacements.length === 0) return;
+  const sorted = [...replacements].sort(
+    (a, b) => a.line - b.line || a.column - b.column,
+  );
+  for (const r of sorted) {
+    const delta = r.newLength - r.originalLength;
+    if (delta !== 0) {
+      // r.column is in post-replacement coordinates (matching the already-shifted
+      // source map state after previous offsetColumns calls). The end of the
+      // placeholder in these coordinates is simply r.column + r.originalLength.
+      const offsetStartColumn = r.column + r.originalLength;
+      const line1Based = r.line + 1;
+      if (line1Based >= 1 && offsetStartColumn + delta >= 0) {
+        sourceMap.offsetColumns(line1Based, offsetStartColumn, delta);
+      }
+    }
+  }
+}
+
+/**
+ * Computes the sourceRoot for a source map file. This is the relative path from
+ * the output directory back to the project root, so that source paths (stored
+ * relative to projectRoot) resolve correctly from the .map file location.
+ *
+ * Returns undefined when sources are inlined (inlineSources), since the browser
+ * doesn't need to fetch them and sourceRoot would be unnecessary.
+ *
+ * This logic must stay in sync with PackagerRunner.generateSourceMap.
+ */
+export function computeSourceMapRoot(
+  bundle: Bundle,
+  options: AtlaspackOptions,
+): string | undefined {
+  let name = nullthrows(bundle.name);
+  let filePath = joinProjectPath(bundle.target.distDir, name);
+  let fullPath = fromProjectPath(options.projectRoot, filePath);
+  let sourceRoot: string = path.relative(
+    path.dirname(fullPath),
+    options.projectRoot,
+  );
+
+  let inlineSources = false;
+
+  const bundleEnv = fromEnvironmentId(bundle.env);
+  if (bundle.target) {
+    const bundleTargetEnv = fromEnvironmentId(bundle.target.env);
+
+    if (bundleEnv.sourceMap && bundleEnv.sourceMap.sourceRoot !== undefined) {
+      sourceRoot = bundleEnv.sourceMap.sourceRoot;
+    } else if (options.serveOptions && bundleTargetEnv.context === 'browser') {
+      sourceRoot = '/__parcel_source_root';
+    }
+
+    if (
+      bundleEnv.sourceMap &&
+      bundleEnv.sourceMap.inlineSources !== undefined
+    ) {
+      inlineSources = bundleEnv.sourceMap.inlineSources;
+    } else if (bundleTargetEnv.context !== 'node') {
+      inlineSources = options.mode === 'production';
+    }
+  }
+
+  let isInlineMap = bundleEnv.sourceMap && bundleEnv.sourceMap.inline;
+
+  if (getFeatureFlag('omitSourcesContentInMemory') && !isInlineMap) {
+    if (!(bundleEnv.sourceMap && bundleEnv.sourceMap.inlineSources === false)) {
+      inlineSources = true;
+    }
+  }
+
+  if (inlineSources) {
+    return undefined;
+  }
+
+  return url.format(url.parse(sourceRoot + '/'));
+}
+
 async function writeFiles(
   // @ts-expect-error TS2503
   inputStream: stream.Readable,
@@ -218,6 +340,7 @@ async function writeFiles(
   writeOptions: FileOptions | null | undefined,
   devDeps: Map<string, string>,
   api: RunAPI<PackagedBundleInfo>,
+  bundleReplacements?: HashRefReplacement[],
 ) {
   let compressors = await config.getCompressors(
     fromProjectPathRelative(filePath),
@@ -225,7 +348,7 @@ async function writeFiles(
   let fullPath = fromProjectPath(options.projectRoot, filePath);
 
   let stream = info.hashReferences.length
-    ? inputStream.pipe(replaceStream(hashRefToNameHash))
+    ? inputStream.pipe(replaceStream(hashRefToNameHash, bundleReplacements))
     : inputStream;
 
   let promises: Array<Promise<undefined>> = [];
@@ -314,9 +437,30 @@ async function runCompressor(
   }
 }
 
-function replaceStream(hashRefToNameHash: Map<string, string>) {
+function advanceLineColumn(
+  line: number,
+  column: number,
+  buf: Buffer,
+): {line: number; column: number} {
+  for (let i = 0; i < buf.length; i++) {
+    if (buf[i] === 0x0a) {
+      line++;
+      column = 0;
+    } else {
+      column++;
+    }
+  }
+  return {line, column};
+}
+
+function replaceStream(
+  hashRefToNameHash: Map<string, string>,
+  replacements?: HashRefReplacement[],
+) {
   let boundaryStr = Buffer.alloc(0);
   let replaced = Buffer.alloc(0);
+  let outputLine = 0;
+  let outputColumn = 0;
   return new Transform({
     transform(
       chunk: Buffer | string,
@@ -347,22 +491,42 @@ function replaceStream(hashRefToNameHash: Map<string, string>) {
             .subarray(matchI, matchI + HASH_REF_PREFIX_LEN + HASH_REF_HASH_LEN)
             .toString();
           let replacement = Buffer.from(hashRefToNameHash.get(match) ?? match);
+          // Copy pre-match content FIRST so position calculation includes it
           replaced.set(str.subarray(lastMatchI, matchI), replacedLength);
           replacedLength += matchI - lastMatchI;
+          if (replacements) {
+            const pos = advanceLineColumn(
+              outputLine,
+              outputColumn,
+              replaced.subarray(0, replacedLength),
+            );
+            replacements.push({
+              line: pos.line,
+              column: pos.column,
+              originalLength: HASH_REF_PLACEHOLDER_LEN,
+              newLength: replacement.byteLength,
+            });
+          }
           replaced.set(replacement, replacedLength);
           replacedLength += replacement.byteLength;
           lastMatchI = matchI + HASH_REF_PREFIX_LEN + HASH_REF_HASH_LEN;
         }
       }
 
+      const pushLen = replacedLength - BOUNDARY_LENGTH;
+      const pushed = advanceLineColumn(
+        outputLine,
+        outputColumn,
+        replaced.subarray(0, pushLen),
+      );
+      outputLine = pushed.line;
+      outputColumn = pushed.column;
+
       boundaryStr = replaced.subarray(
         replacedLength - BOUNDARY_LENGTH,
         replacedLength,
       );
-      let strUpToBoundary = replaced.subarray(
-        0,
-        replacedLength - BOUNDARY_LENGTH,
-      );
+      let strUpToBoundary = replaced.subarray(0, pushLen);
       cb(null, strUpToBoundary);
     },
 

package/src/requests/WriteBundlesRequest.ts

@@ -22,6 +22,16 @@ import {createPackageRequest} from './PackageRequest';
 import createWriteBundleRequest from './WriteBundleRequest';
 import {debugTools} from '@atlaspack/utils';
 
+/** Length of the hash suffix in output filenames (e.g. .runtime.13dc01ac.js). */
+const NAME_HASH_DISPLAY_LEN = 8;
+
+/** Use at most NAME_HASH_DISPLAY_LEN chars for the name hash so filenames stay short. */
+function nameHashForFilename(hash: string): string {
+  return hash.length <= NAME_HASH_DISPLAY_LEN
+    ? hash
+    : hash.slice(-NAME_HASH_DISPLAY_LEN);
+}
+
 type WriteBundlesRequestInput = {
   bundleGraph: BundleGraph;
   optionsRef: SharedReference;
@@ -111,12 +121,12 @@ async function run({
       // Do not package and write placeholder bundles to disk. We just
       // need to update the name so other bundles can reference it.
       if (bundle.isPlaceholder) {
-        let hash = bundle.id.slice(-8);
-        hashRefToNameHash.set(bundle.hashReference, hash);
+        const nameHash = nameHashForFilename(bundle.id);
+        hashRefToNameHash.set(bundle.hashReference, nameHash);
         let name = nullthrows(
           bundle.name,
           `Expected ${bundle.type} bundle to have a name`,
-        ).replace(bundle.hashReference, hash);
+        ).replace(bundle.hashReference, nameHash);
         res.set(bundle.id, {
           filePath: joinProjectPath(bundle.target.distDir, name),
           bundleId: bundle.id,
@@ -178,9 +188,9 @@ async function run({
         if (!info.hashReferences.length) {
           hashRefToNameHash.set(
             bundle.hashReference,
-            options.shouldContentHash
-              ? info.hash.slice(-8)
-              : bundle.id.slice(-8),
+            nameHashForFilename(
+              options.shouldContentHash ? info.hash : bundle.id,
+            ),
           );
           let writeBundleRequest = createWriteBundleRequest({
             bundle,
@@ -256,13 +266,15 @@ function assignComplexNameHashes(
     }
     hashRefToNameHash.set(
       bundle.hashReference,
-      options.shouldContentHash
-        ? hashString(
-            [...getBundlesIncludedInHash(bundle.id, bundleInfoMap)]
-              .map((bundleId) => bundleInfoMap[bundleId].hash)
-              .join(':'),
-          ).slice(-8)
-        : bundle.id.slice(-8),
+      nameHashForFilename(
+        options.shouldContentHash
+          ? hashString(
+              [...getBundlesIncludedInHash(bundle.id, bundleInfoMap)]
+                .map((bundleId) => bundleInfoMap[bundleId].hash)
+                .join(':'),
+            )
+          : bundle.id,
+      ),
     );
   }
 }

package/src/resolveOptions.ts

@@ -184,10 +184,8 @@ export default async function resolveOptions(
     const needsRustLmdbCache =
       getFeatureFlag('atlaspackV3') || getFeatureFlag('nativePackager');
 
-    if (!getFeatureFlag('cachePerformanceImprovements')) {
-      if (!needsRustLmdbCache && !(outputFS instanceof NodeFS)) {
-        return new FSCache(outputFS, cacheDir);
-      }
+    if (!needsRustLmdbCache && !(outputFS instanceof NodeFS)) {
+      return new FSCache(outputFS, cacheDir);
     }
 
     return new LMDBLiteCache(cacheDir);

package/src/types.ts

@@ -112,6 +112,7 @@ export type Target = {
   loc?: InternalSourceLocation | null | undefined;
   pipeline?: string;
   source?: FilePath | Array<FilePath>;
+  inlineRequires?: boolean;
 };
 
 export const SpecifierType = {

package/test/AssetGraph.test.ts

@@ -687,36 +687,4 @@ describe('AssetGraph', () => {
     invariant(node.type === 'asset_group');
     assert(!node.hasDeferred);
   });
-
-  it('should serialize the bundling version and incremental bundling flag', () => {
-    const graph = new AssetGraph();
-    graph.setDisableIncrementalBundling(true);
-    graph.setNeedsBundling();
-    const serialized = serialize(graph);
-    const deserialized = deserialize(serialized);
-
-    assert.equal(deserialized.getBundlingVersion(), 1);
-    assert.equal(deserialized.testing_getDisableIncrementalBundling(), true);
-  });
-
-  describe('setNeedsBundling', () => {
-    it('should increment the bundling version', () => {
-      const graph = new AssetGraph();
-      assert.equal(graph.getBundlingVersion(), 0);
-      graph.setNeedsBundling();
-      assert.equal(graph.getBundlingVersion(), 1);
-      graph.setNeedsBundling();
-      assert.equal(graph.getBundlingVersion(), 2);
-    });
-  });
-
-  describe('canIncrementallyBundle', () => {
-    it('should return true if the bundling version has changed', () => {
-      const graph = new AssetGraph();
-      const lastVersion = graph.getBundlingVersion();
-      assert.equal(graph.canIncrementallyBundle(lastVersion), true);
-      graph.setNeedsBundling();
-      assert.equal(graph.canIncrementallyBundle(lastVersion), false);
-    });
-  });
 });

package/test/RequestTracker.test.ts

@@ -538,171 +538,6 @@ describe('RequestTracker', () => {
       });
     });
   });
-
-  describe('incremental bundling', () => {
-    async function runIncrementalBundlingScenario(
-      incrementalBundlingVersioning: boolean,
-    ) {
-      const fs = new OverlayFS(new MemoryFS(farm), new NodeFS());
-      const appRoot = __dirname;
-      await fs.mkdirp(path.join(appRoot, 'app'));
-      await fs.writeFile(path.join(appRoot, 'app', 'package.json'), '{}');
-      await fs.writeFile(path.join(appRoot, 'app', '.git'), '');
-      await fs.writeFile(
-        path.join(appRoot, 'app', '.parcelrc'),
-        '{"extends":"@atlaspack/config-default"}',
-      );
-      await fs.writeFile(
-        path.join(appRoot, 'app', 'target.js'),
-        'console.log("hello")',
-      );
-
-      const atlaspack = new Atlaspack({
-        featureFlags: {
-          incrementalBundlingVersioning,
-        },
-        workerFarm: farm,
-        entries: [path.join(appRoot, 'app', 'target.js')],
-        cache: new LMDBLiteCache(DEFAULT_OPTIONS.cacheDir),
-        inputFS: fs,
-        outputFS: fs,
-      });
-      await atlaspack._init();
-      const options = atlaspack._getResolvedAtlaspackOptions();
-      const tracker = new RequestTracker({farm, options});
-      let {ref: optionsRef} = await farm.createSharedReference(options, false);
-
-      const getAssetRequests = () =>
-        runRequestSpy
-          .getCalls()
-          .map((call: any) => call.args[0])
-          .filter(
-            (request: any) => request.type === requestTypes.asset_request,
-          );
-
-      // Running the build once builds one asset
-      const runRequestSpy = sinon.spy(tracker, 'runRequest');
-      await tracker.runRequest(
-        createAtlaspackBuildRequest({
-          optionsRef,
-          requestedAssetIds: new Set(),
-        }),
-      );
-      assert.equal(getAssetRequests().length, 1);
-      runRequestSpy.resetHistory();
-
-      // Running the build again with no invalidations does not build any assets
-      await tracker.runRequest(
-        createAtlaspackBuildRequest({
-          optionsRef,
-          requestedAssetIds: new Set(),
-        }),
-      );
-      assert.equal(getAssetRequests().length, 0);
-      runRequestSpy.resetHistory();
-
-      // Running the build again with a file change builds the asset again
-      tracker.respondToFSEvents(
-        [
-          {
-            type: 'update',
-            path: path.join(appRoot, 'app', 'target.js'),
-          },
-        ],
-        Number.MAX_VALUE,
-      );
-      await tracker.runRequest(
-        createAtlaspackBuildRequest({
-          optionsRef,
-          requestedAssetIds: new Set(),
-        }),
-      );
-      assert.equal(getAssetRequests().length, 1);
-      runRequestSpy.resetHistory();
-
-      // Run the asset graph request, but not bundling
-      await fs.writeFile(
-        path.join(appRoot, 'app', 'target.js'),
-        'require("./dep.js")',
-      );
-      await fs.writeFile(
-        path.join(appRoot, 'app', 'dep.js'),
-        'console.log("dep")',
-      );
-      tracker.respondToFSEvents(
-        [
-          {
-            type: 'update',
-            path: path.join(appRoot, 'app', 'target.js'),
-          },
-        ],
-        Number.MAX_VALUE,
-      );
-      const assetGraphRequestResult = await tracker.runRequest(
-        createAssetGraphRequest({
-          name: 'Main',
-          entries: [
-            toProjectPath(
-              path.join(appRoot, 'app'),
-              path.join(appRoot, 'app', 'target.js'),
-            ),
-          ],
-          optionsRef,
-          shouldBuildLazily: false,
-          lazyIncludes: [],
-          lazyExcludes: [],
-          requestedAssetIds: new Set(),
-        }),
-      );
-      assert.equal(getAssetRequests().length, 2);
-      assert.equal(
-        assetGraphRequestResult.assetGraph.safeToIncrementallyBundle,
-        false,
-      );
-
-      // Now make another change
-      tracker.respondToFSEvents(
-        [
-          {
-            type: 'update',
-            path: path.join(appRoot, 'app', 'target.js'),
-          },
-          {
-            type: 'update',
-            path: path.join(appRoot, 'app', 'dep.js'),
-          },
-        ],
-        Number.MAX_VALUE,
-      );
-      // And run the build again
-
-      if (!incrementalBundlingVersioning) {
-        await assert.rejects(async () => {
-          await tracker.runRequest(
-            createAtlaspackBuildRequest({
-              optionsRef,
-              requestedAssetIds: new Set(),
-            }),
-          );
-        });
-      } else {
-        await tracker.runRequest(
-          createAtlaspackBuildRequest({
-            optionsRef,
-            requestedAssetIds: new Set(),
-          }),
-        );
-      }
-    }
-
-    it('throws a content key not found exception without bundling versioning', async () => {
-      await runIncrementalBundlingScenario(false);
-    });
-
-    it('works fine with bundling versioning', async () => {
-      await runIncrementalBundlingScenario(true);
-    });
-  });
 });
 
 describe('cleanUpOrphans', () => {