@atlaspack/core 2.17.4 → 2.18.0

package/src/Environment.js

@@ -10,6 +10,8 @@ import {toInternalSourceLocation} from './utils';
 import PublicEnvironment from './public/Environment';
 import {environmentToInternalEnvironment} from './public/Environment';
 import {identifierRegistry} from './IdentifierRegistry';
+import {toEnvironmentRef} from './EnvironmentManager';
+import type {EnvironmentRef} from './EnvironmentManager';
 
 const DEFAULT_ENGINES = {
   browsers: ['> 0.25%'],
@@ -35,7 +37,7 @@ export function createEnvironment({
   loc,
 }: EnvironmentOpts = {
   /*::...null*/
-}): Environment {
+}): EnvironmentRef {
   if (context == null) {
     if (engines?.node) {
       context = 'node';
@@ -112,21 +114,22 @@ export function createEnvironment({
   };
 
   res.id = getEnvironmentHash(res);
-  return Object.freeze(res);
+
+  return toEnvironmentRef(Object.freeze(res));
 }
 
 export function mergeEnvironments(
   projectRoot: FilePath,
   a: Environment,
   b: ?(EnvironmentOptions | IEnvironment),
-): Environment {
+): EnvironmentRef {
   // If merging the same object, avoid copying.
   if (a === b || !b) {
-    return a;
+    return toEnvironmentRef(a);
   }
 
   if (b instanceof PublicEnvironment) {
-    return environmentToInternalEnvironment(b);
+    return toEnvironmentRef(environmentToInternalEnvironment(b));
   }
 
   // $FlowFixMe - ignore the `id` that is already on a

package/src/EnvironmentManager.js

@@ -0,0 +1,145 @@
+// @flow strict-local
+/*!
+ * At the moment we're doing this change for `CoreEnvironment`,
+ * but the same change must be made for `TypesEnvironment` in @atlaspack/types.
+ */
+import type {Environment as CoreEnvironment} from './types';
+import {type Cache} from '@atlaspack/cache';
+import {
+  addEnvironment,
+  getEnvironment,
+  getAllEnvironments,
+  setAllEnvironments,
+} from '@atlaspack/rust';
+import {getFeatureFlag} from '@atlaspack/feature-flags';
+import {instrument} from '@atlaspack/logger';
+import {ATLASPACK_VERSION} from './constants';
+
+const localEnvironmentCache = new Map<string, CoreEnvironment>();
+
+export opaque type EnvironmentId = string;
+/**
+ * When deduplication is cleaned-up this will always be a string.
+ */
+export type EnvironmentRef = EnvironmentId | CoreEnvironment;
+
+/**
+ * Convert environment to a ref.
+ * This is what we should be using to store environments.
+ */
+export function toEnvironmentRef(env: CoreEnvironment): EnvironmentRef {
+  if (!getFeatureFlag('environmentDeduplication')) {
+    return env;
+  }
+
+  const id = toEnvironmentId(env);
+  return id;
+}
+
+/**
+ * Convert environment to a string ID
+ */
+export function toEnvironmentId(
+  /**
+   * Redundant type during roll-out
+   */
+  env: CoreEnvironment | EnvironmentRef,
+): string {
+  if (!getFeatureFlag('environmentDeduplication')) {
+    return typeof env === 'string' ? env : env.id;
+  }
+
+  if (typeof env === 'string') {
+    return env;
+  }
+
+  addEnvironment(env);
+  return env.id;
+}
+
+export function fromEnvironmentId(id: EnvironmentRef): CoreEnvironment {
+  if (!getFeatureFlag('environmentDeduplication')) {
+    if (typeof id === 'string') {
+      throw new Error(
+        'This should never happen when environmentDeduplication feature-flag is off',
+      );
+    } else {
+      return id;
+    }
+  }
+
+  if (typeof id !== 'string') {
+    return id;
+  }
+
+  const localEnv = localEnvironmentCache.get(id);
+
+  if (localEnv) {
+    return localEnv;
+  }
+
+  const env = Object.freeze(getEnvironment(id));
+  localEnvironmentCache.set(id, env);
+  return env;
+}
+
+/**
+ * Writes all environments and their IDs to the cache
+ * @param {Cache} cache
+ * @returns {Promise<void>}
+ */
+export async function writeEnvironmentsToCache(cache: Cache): Promise<void> {
+  const environments = getAllEnvironments();
+  const environmentIds = new Set<string>();
+
+  // Store each environment individually
+  for (const env of environments) {
+    environmentIds.add(env.id);
+    const envKey = `Environment/${ATLASPACK_VERSION}/${env.id}`;
+
+    await instrument(
+      `RequestTracker::writeToCache::cache.put(${envKey})`,
+      async () => {
+        await cache.set(envKey, env);
+      },
+    );
+  }
+
+  // Store the list of environment IDs
+  await instrument(
+    `RequestTracker::writeToCache::cache.put(${`EnvironmentManager/${ATLASPACK_VERSION}`})`,
+    async () => {
+      await cache.set(
+        `EnvironmentManager/${ATLASPACK_VERSION}`,
+        Array.from(environmentIds),
+      );
+    },
+  );
+}
+
+/**
+ * Loads all environments and their IDs from the cache
+ * @param {Cache} cache
+ * @returns {Promise<void>}
+ */
+export async function loadEnvironmentsFromCache(cache: Cache): Promise<void> {
+  const cachedEnvIds = await cache.get(
+    `EnvironmentManager/${ATLASPACK_VERSION}`,
+  );
+
+  if (cachedEnvIds == null) {
+    return;
+  }
+
+  const environments = [];
+  for (const envId of cachedEnvIds) {
+    const envKey = `Environment/${ATLASPACK_VERSION}/${envId}`;
+    const cachedEnv = await cache.get(envKey);
+    if (cachedEnv != null) {
+      environments.push(cachedEnv);
+    }
+  }
+  if (environments.length > 0) {
+    setAllEnvironments(environments);
+  }
+}

package/src/InternalConfig.js

@@ -3,7 +3,6 @@
 import type {PackageName, ConfigResult} from '@atlaspack/types';
 import type {
   Config,
-  Environment,
   InternalFileCreateInvalidation,
   InternalDevDepOptions,
 } from './types';
@@ -13,17 +12,19 @@ import {fromProjectPathRelative} from './projectPath';
 import {createEnvironment} from './Environment';
 import {hashString} from '@atlaspack/rust';
 import {identifierRegistry} from './IdentifierRegistry';
+import type {EnvironmentRef} from './EnvironmentManager';
+import {toEnvironmentId} from './EnvironmentManager';
 
 type ConfigOpts = {|
   plugin: PackageName,
   searchPath: ProjectPath,
   isSource?: boolean,
-  env?: Environment,
+  env?: EnvironmentRef,
   result?: ConfigResult,
   invalidateOnFileChange?: Set<ProjectPath>,
   invalidateOnConfigKeyChange?: Array<{|
     filePath: ProjectPath,
-    configKey: string,
+    configKey: string[],
   |}>,
   invalidateOnFileCreate?: Array<InternalFileCreateInvalidation>,
   invalidateOnEnvChange?: Set<string>,
@@ -52,13 +53,13 @@ export function createConfig({
   const configId = hashString(
     plugin +
       fromProjectPathRelative(searchPath) +
-      environment.id +
+      toEnvironmentId(environment) +
       String(isSource),
   );
   identifierRegistry.addIdentifier('config_request', configId, {
     plugin,
     searchPath,
-    environmentId: environment.id,
+    environmentId: toEnvironmentId(environment),
     isSource,
   });
   return {

package/src/PackagerRunner.js

@@ -63,6 +63,7 @@ import {getInvalidationId, getInvalidationHash} from './assetUtils';
 import {optionsProxy} from './utils';
 import {invalidateDevDeps} from './requests/DevDepRequest';
 import {tracer, PluginTracer} from '@atlaspack/profiler';
+import {fromEnvironmentId} from './EnvironmentManager';
 import {getFeatureFlag} from '@atlaspack/feature-flags';
 
 type Opts = {|
@@ -578,32 +579,32 @@ export default class PackagerRunner {
     );
     let inlineSources = false;
 
+    const bundleEnv = fromEnvironmentId(bundle.env);
     if (bundle.target) {
-      if (
-        bundle.env.sourceMap &&
-        bundle.env.sourceMap.sourceRoot !== undefined
-      ) {
-        sourceRoot = bundle.env.sourceMap.sourceRoot;
+      const bundleTargetEnv = fromEnvironmentId(bundle.target.env);
+
+      if (bundleEnv.sourceMap && bundleEnv.sourceMap.sourceRoot !== undefined) {
+        sourceRoot = bundleEnv.sourceMap.sourceRoot;
       } else if (
         this.options.serveOptions &&
-        bundle.target.env.context === 'browser'
+        bundleTargetEnv.context === 'browser'
       ) {
         sourceRoot = '/__parcel_source_root';
       }
 
       if (
-        bundle.env.sourceMap &&
-        bundle.env.sourceMap.inlineSources !== undefined
+        bundleEnv.sourceMap &&
+        bundleEnv.sourceMap.inlineSources !== undefined
       ) {
-        inlineSources = bundle.env.sourceMap.inlineSources;
-      } else if (bundle.target.env.context !== 'node') {
+        inlineSources = bundleEnv.sourceMap.inlineSources;
+      } else if (bundleTargetEnv.context !== 'node') {
         // inlining should only happen in production for browser targets by default
         inlineSources = this.options.mode === 'production';
       }
     }
 
     let mapFilename = fullPath + '.map';
-    let isInlineMap = bundle.env.sourceMap && bundle.env.sourceMap.inline;
+    let isInlineMap = bundleEnv.sourceMap && bundleEnv.sourceMap.inline;
 
     let stringified = await map.stringify({
       file: path.basename(mapFilename),

package/src/RequestTracker.js

@@ -30,15 +30,15 @@ import nullthrows from 'nullthrows';
 
 import {
   ATLASPACK_VERSION,
-  VALID,
-  INITIAL_BUILD,
   FILE_CREATE,
-  FILE_UPDATE,
   FILE_DELETE,
+  FILE_UPDATE,
   ENV_CHANGE,
+  ERROR,
+  INITIAL_BUILD,
   OPTION_CHANGE,
   STARTUP,
-  ERROR,
+  VALID,
 } from './constants';
 import type {AtlaspackV3} from './atlaspack-v3/AtlaspackV3';
 import {
@@ -71,6 +71,11 @@ import type {
 import {BuildAbortError, assertSignalNotAborted, hashFromOption} from './utils';
 import {performance} from 'perf_hooks';
 
+import {
+  loadEnvironmentsFromCache,
+  writeEnvironmentsToCache,
+} from './EnvironmentManager';
+
 export const requestGraphEdgeTypes = {
   subrequest: 2,
   invalidated_by_update: 3,
@@ -144,7 +149,7 @@ type OptionNode = {|
 type ConfigKeyNode = {|
   id: ContentKey,
   +type: typeof CONFIG_KEY,
-  configKey: string,
+  configKey: string[],
   contentHash: string,
 |};
 
@@ -216,7 +221,7 @@ export type RunAPI<TResult: RequestResult> = {|
   invalidateOnFileUpdate: (ProjectPath) => void,
   invalidateOnConfigKeyChange: (
     filePath: ProjectPath,
-    configKey: string,
+    configKey: string[],
     contentHash: string,
   ) => void,
   invalidateOnStartup: () => void,
@@ -283,10 +288,12 @@ const nodeFromOption = (option: string, value: mixed): RequestGraphNode => ({
 
 const nodeFromConfigKey = (
   fileName: ProjectPath,
-  configKey: string,
+  configKey: string[],
   contentHash: string,
 ): RequestGraphNode => ({
-  id: `config_key:${fromProjectPathRelative(fileName)}:${configKey}`,
+  id: `config_key:${fromProjectPathRelative(fileName)}:${JSON.stringify(
+    configKey,
+  )}`,
   type: CONFIG_KEY,
   configKey,
   contentHash,
@@ -527,7 +534,7 @@ export class RequestGraph extends ContentGraph<
   invalidateOnConfigKeyChange(
     requestNodeId: NodeId,
     filePath: ProjectPath,
-    configKey: string,
+    configKey: string[],
     contentHash: string,
   ) {
     let configKeyNodeId = this.addNode(
@@ -714,8 +721,8 @@ export class RequestGraph extends ContentGraph<
     env: string,
     value: string | void,
   ) {
-    let envNode = nodeFromEnv(env, value);
-    let envNodeId = this.addNode(envNode);
+    const envNode = nodeFromEnv(env, value);
+    const envNodeId = this.addNode(envNode);
 
     if (
       !this.hasEdge(
@@ -1109,11 +1116,22 @@ export class RequestGraph extends ContentGraph<
       }
 
       let configKeyNodes = this.configKeyNodes.get(_filePath);
-      if (configKeyNodes && (type === 'delete' || type === 'update')) {
+
+      // With granular invalidations we will always run this block,
+      // so even if we get a create event (for whatever reason), we will still
+      // try to limit invalidations from config key changes through hashing.
+      //
+      // Currently create events can invalidate a large number of nodes due to
+      // "create above" invalidations.
+      const isConfigKeyChange =
+        getFeatureFlag('granularTsConfigInvalidation') ||
+        type === 'delete' ||
+        type === 'update';
+      if (configKeyNodes && isConfigKeyChange) {
         for (let nodeId of configKeyNodes) {
           let isInvalid = type === 'delete';
 
-          if (type === 'update') {
+          if (type !== 'delete') {
             let node = this.getNode(nodeId);
             invariant(node && node.type === CONFIG_KEY);
 
@@ -1559,6 +1577,10 @@ export default class RequestTracker {
         size: this.graph.nodes.length,
       });
 
+      if (getFeatureFlag('environmentDeduplication')) {
+        await writeEnvironmentsToCache(options.cache);
+      }
+
       let serialisedGraph = this.graph.serialize();
 
       // Delete an existing request graph cache, to prevent invalid states
@@ -1843,6 +1865,10 @@ async function loadRequestGraph(options): Async<RequestGraph> {
     },
   });
 
+  if (getFeatureFlag('environmentDeduplication')) {
+    await loadEnvironmentsFromCache(options.cache);
+  }
+
   const hasRequestGraphInCache = getFeatureFlag('cachePerformanceImprovements')
     ? await options.cache.has(requestGraphKey)
     : await options.cache.hasLargeBlob(requestGraphKey);

package/src/UncommittedAsset.js

@@ -36,6 +36,7 @@ import {
   fromProjectPathRelative,
 } from './projectPath';
 import {getFeatureFlag} from '@atlaspack/feature-flags';
+import {fromEnvironmentId} from './EnvironmentManager';
 
 type UncommittedAssetOptions = {|
   value: Asset,
@@ -321,7 +322,11 @@ export default class UncommittedAsset {
       ...rest,
       // $FlowFixMe "convert" the $ReadOnlyMaps to the interal mutable one
       symbols,
-      env: mergeEnvironments(this.options.projectRoot, this.value.env, env),
+      env: mergeEnvironments(
+        this.options.projectRoot,
+        fromEnvironmentId(this.value.env),
+        env,
+      ),
       sourceAssetId: this.value.id,
       sourcePath: fromProjectPath(
         this.options.projectRoot,
@@ -386,7 +391,7 @@ export default class UncommittedAsset {
         isSource: this.value.isSource,
         env: mergeEnvironments(
           this.options.projectRoot,
-          this.value.env,
+          fromEnvironmentId(this.value.env),
           result.env,
         ),
         dependencies:

package/src/applyRuntimes.js

@@ -34,6 +34,7 @@ import {createDevDependency, runDevDepRequest} from './requests/DevDepRequest';
 import {toProjectPath, fromProjectPathRelative} from './projectPath';
 import {tracer, PluginTracer} from '@atlaspack/profiler';
 import {DefaultMap} from '@atlaspack/utils';
+import {fromEnvironmentId} from './EnvironmentManager';
 
 type RuntimeConnection = {|
   bundle: InternalBundle,
@@ -159,7 +160,11 @@ export default async function applyRuntimes<TResult: RequestResult>({
             let assetGroup = {
               code,
               filePath: toProjectPath(options.projectRoot, sourceName),
-              env: mergeEnvironments(options.projectRoot, bundle.env, env),
+              env: mergeEnvironments(
+                options.projectRoot,
+                fromEnvironmentId(bundle.env),
+                env,
+              ),
               // Runtime assets should be considered source, as they should be
               // e.g. compiled to run in the target environment
               isSource: true,

package/src/assetUtils.js

@@ -16,7 +16,6 @@ import type {
   Asset,
   RequestInvalidation,
   Dependency,
-  Environment,
   AtlaspackOptions,
 } from './types';
 
@@ -40,6 +39,8 @@ import {hashString, createAssetId as createAssetIdRust} from '@atlaspack/rust';
 import {BundleBehavior as BundleBehaviorMap} from './types';
 import {PluginTracer} from '@atlaspack/profiler';
 import {identifierRegistry} from './IdentifierRegistry';
+import type {EnvironmentRef} from './EnvironmentManager';
+import {toEnvironmentId} from './EnvironmentManager';
 
 export type AssetOptions = {|
   id?: string,
@@ -56,7 +57,7 @@ export type AssetOptions = {|
   bundleBehavior?: ?BundleBehavior,
   isBundleSplittable?: ?boolean,
   isSource: boolean,
-  env: Environment,
+  env: EnvironmentRef,
   meta?: Meta,
   outputHash?: ?string,
   pipeline?: ?string,
@@ -71,7 +72,7 @@ export type AssetOptions = {|
 
 export function createAssetIdFromOptions(options: AssetOptions): string {
   const data = {
-    environmentId: options.env.id,
+    environmentId: toEnvironmentId(options.env),
     filePath: options.filePath,
     code: options.code,
     pipeline: options.pipeline,

package/src/atlaspack-v3/worker/compat/plugin-config.js

@@ -107,7 +107,7 @@ export class PluginConfig implements IPluginConfig {
           exclude?: boolean,
         |}
       | {|
-          configKey?: string,
+          readTracking?: boolean,
         |},
   ): Promise<?ConfigResultWithFilePath<T>> {
     return this.#inner.getConfigFrom(searchPath, filePaths, options);

package/src/public/Asset.js

@@ -36,6 +36,7 @@ import {
   BundleBehaviorNames,
 } from '../types';
 import {toInternalSourceLocation} from '../utils';
+import {fromEnvironmentId} from '../EnvironmentManager';
 
 const inspect = Symbol.for('nodejs.util.inspect.custom');
 
@@ -101,7 +102,10 @@ class BaseAsset {
   }
 
   get env(): IEnvironment {
-    return new Environment(this.#asset.value.env, this.#asset.options);
+    return new Environment(
+      fromEnvironmentId(this.#asset.value.env),
+      this.#asset.options,
+    );
   }
 
   get fs(): FileSystem {
@@ -210,7 +214,10 @@ export class Asset extends BaseAsset implements IAsset {
   }
 
   get env(): IEnvironment {
-    this.#env ??= new Environment(this.#asset.value.env, this.#asset.options);
+    this.#env ??= new Environment(
+      fromEnvironmentId(this.#asset.value.env),
+      this.#asset.options,
+    );
     return this.#env;
   }
 

package/src/public/Bundle.js

@@ -34,6 +34,7 @@ import {
 import Target from './Target';
 import {BundleBehaviorNames} from '../types';
 import {fromProjectPath} from '../projectPath';
+import {fromEnvironmentId} from '../EnvironmentManager';
 
 const inspect = Symbol.for('nodejs.util.inspect.custom');
 
@@ -123,7 +124,7 @@ export class Bundle implements IBundle {
   }
 
   get env(): IEnvironment {
-    return new Environment(this.#bundle.env, this.#options);
+    return new Environment(fromEnvironmentId(this.#bundle.env), this.#options);
   }
 
   get needsStableName(): ?boolean {