@atlaspack/core 2.16.2-dev.55 → 2.16.2-dev.69

package/src/RequestTracker.js

@@ -30,15 +30,15 @@ import nullthrows from 'nullthrows';
 
 import {
   ATLASPACK_VERSION,
-  VALID,
-  INITIAL_BUILD,
   FILE_CREATE,
-  FILE_UPDATE,
   FILE_DELETE,
+  FILE_UPDATE,
   ENV_CHANGE,
+  ERROR,
+  INITIAL_BUILD,
   OPTION_CHANGE,
   STARTUP,
-  ERROR,
+  VALID,
 } from './constants';
 import type {AtlaspackV3} from './atlaspack-v3/AtlaspackV3';
 import {
@@ -71,6 +71,11 @@ import type {
 import {BuildAbortError, assertSignalNotAborted, hashFromOption} from './utils';
 import {performance} from 'perf_hooks';
 
+import {
+  loadEnvironmentsFromCache,
+  writeEnvironmentsToCache,
+} from './EnvironmentManager';
+
 export const requestGraphEdgeTypes = {
   subrequest: 2,
   invalidated_by_update: 3,
@@ -144,7 +149,7 @@ type OptionNode = {|
 type ConfigKeyNode = {|
   id: ContentKey,
   +type: typeof CONFIG_KEY,
-  configKey: string,
+  configKey: string[],
   contentHash: string,
 |};
 
@@ -216,7 +221,7 @@ export type RunAPI<TResult: RequestResult> = {|
   invalidateOnFileUpdate: (ProjectPath) => void,
   invalidateOnConfigKeyChange: (
     filePath: ProjectPath,
-    configKey: string,
+    configKey: string[],
     contentHash: string,
   ) => void,
   invalidateOnStartup: () => void,
@@ -283,10 +288,12 @@ const nodeFromOption = (option: string, value: mixed): RequestGraphNode => ({
 
 const nodeFromConfigKey = (
   fileName: ProjectPath,
-  configKey: string,
+  configKey: string[],
   contentHash: string,
 ): RequestGraphNode => ({
-  id: `config_key:${fromProjectPathRelative(fileName)}:${configKey}`,
+  id: `config_key:${fromProjectPathRelative(fileName)}:${JSON.stringify(
+    configKey,
+  )}`,
   type: CONFIG_KEY,
   configKey,
   contentHash,
@@ -527,7 +534,7 @@ export class RequestGraph extends ContentGraph<
   invalidateOnConfigKeyChange(
     requestNodeId: NodeId,
     filePath: ProjectPath,
-    configKey: string,
+    configKey: string[],
     contentHash: string,
   ) {
     let configKeyNodeId = this.addNode(
@@ -714,8 +721,8 @@ export class RequestGraph extends ContentGraph<
     env: string,
     value: string | void,
   ) {
-    let envNode = nodeFromEnv(env, value);
-    let envNodeId = this.addNode(envNode);
+    const envNode = nodeFromEnv(env, value);
+    const envNodeId = this.addNode(envNode);
 
     if (
       !this.hasEdge(
@@ -1109,11 +1116,22 @@ export class RequestGraph extends ContentGraph<
       }
 
       let configKeyNodes = this.configKeyNodes.get(_filePath);
-      if (configKeyNodes && (type === 'delete' || type === 'update')) {
+
+      // With granular invalidations we will always run this block,
+      // so even if we get a create event (for whatever reason), we will still
+      // try to limit invalidations from config key changes through hashing.
+      //
+      // Currently create events can invalidate a large number of nodes due to
+      // "create above" invalidations.
+      const isConfigKeyChange =
+        getFeatureFlag('granularTsConfigInvalidation') ||
+        type === 'delete' ||
+        type === 'update';
+      if (configKeyNodes && isConfigKeyChange) {
         for (let nodeId of configKeyNodes) {
           let isInvalid = type === 'delete';
 
-          if (type === 'update') {
+          if (type !== 'delete') {
             let node = this.getNode(nodeId);
             invariant(node && node.type === CONFIG_KEY);
 
@@ -1559,6 +1577,10 @@ export default class RequestTracker {
         size: this.graph.nodes.length,
       });
 
+      if (getFeatureFlag('environmentDeduplication')) {
+        await writeEnvironmentsToCache(options.cache);
+      }
+
       let serialisedGraph = this.graph.serialize();
 
       // Delete an existing request graph cache, to prevent invalid states
@@ -1843,6 +1865,10 @@ async function loadRequestGraph(options): Async<RequestGraph> {
     },
   });
 
+  if (getFeatureFlag('environmentDeduplication')) {
+    await loadEnvironmentsFromCache(options.cache);
+  }
+
   const hasRequestGraphInCache = getFeatureFlag('cachePerformanceImprovements')
     ? await options.cache.has(requestGraphKey)
     : await options.cache.hasLargeBlob(requestGraphKey);

package/src/UncommittedAsset.js

@@ -36,6 +36,7 @@ import {
   fromProjectPathRelative,
 } from './projectPath';
 import {getFeatureFlag} from '@atlaspack/feature-flags';
+import {fromEnvironmentId} from './EnvironmentManager';
 
 type UncommittedAssetOptions = {|
   value: Asset,
@@ -321,7 +322,11 @@ export default class UncommittedAsset {
       ...rest,
       // $FlowFixMe "convert" the $ReadOnlyMaps to the interal mutable one
       symbols,
-      env: mergeEnvironments(this.options.projectRoot, this.value.env, env),
+      env: mergeEnvironments(
+        this.options.projectRoot,
+        fromEnvironmentId(this.value.env),
+        env,
+      ),
       sourceAssetId: this.value.id,
       sourcePath: fromProjectPath(
         this.options.projectRoot,
@@ -386,7 +391,7 @@ export default class UncommittedAsset {
         isSource: this.value.isSource,
         env: mergeEnvironments(
           this.options.projectRoot,
-          this.value.env,
+          fromEnvironmentId(this.value.env),
           result.env,
         ),
         dependencies:

package/src/applyRuntimes.js

@@ -34,6 +34,7 @@ import {createDevDependency, runDevDepRequest} from './requests/DevDepRequest';
 import {toProjectPath, fromProjectPathRelative} from './projectPath';
 import {tracer, PluginTracer} from '@atlaspack/profiler';
 import {DefaultMap} from '@atlaspack/utils';
+import {fromEnvironmentId} from './EnvironmentManager';
 
 type RuntimeConnection = {|
   bundle: InternalBundle,
@@ -159,7 +160,11 @@ export default async function applyRuntimes<TResult: RequestResult>({
             let assetGroup = {
               code,
               filePath: toProjectPath(options.projectRoot, sourceName),
-              env: mergeEnvironments(options.projectRoot, bundle.env, env),
+              env: mergeEnvironments(
+                options.projectRoot,
+                fromEnvironmentId(bundle.env),
+                env,
+              ),
               // Runtime assets should be considered source, as they should be
               // e.g. compiled to run in the target environment
               isSource: true,

package/src/assetUtils.js

@@ -16,7 +16,6 @@ import type {
   Asset,
   RequestInvalidation,
   Dependency,
-  Environment,
   AtlaspackOptions,
 } from './types';
 
@@ -40,6 +39,8 @@ import {hashString, createAssetId as createAssetIdRust} from '@atlaspack/rust';
 import {BundleBehavior as BundleBehaviorMap} from './types';
 import {PluginTracer} from '@atlaspack/profiler';
 import {identifierRegistry} from './IdentifierRegistry';
+import type {EnvironmentRef} from './EnvironmentManager';
+import {toEnvironmentId} from './EnvironmentManager';
 
 export type AssetOptions = {|
   id?: string,
@@ -56,7 +57,7 @@ export type AssetOptions = {|
   bundleBehavior?: ?BundleBehavior,
   isBundleSplittable?: ?boolean,
   isSource: boolean,
-  env: Environment,
+  env: EnvironmentRef,
   meta?: Meta,
   outputHash?: ?string,
   pipeline?: ?string,
@@ -71,7 +72,7 @@ export type AssetOptions = {|
 
 export function createAssetIdFromOptions(options: AssetOptions): string {
   const data = {
-    environmentId: options.env.id,
+    environmentId: toEnvironmentId(options.env),
     filePath: options.filePath,
     code: options.code,
     pipeline: options.pipeline,

package/src/atlaspack-v3/AtlaspackV3.js

@@ -12,6 +12,8 @@ import {NapiWorkerPool} from './NapiWorkerPool';
 import ThrowableDiagnostic from '@atlaspack/diagnostic';
 import type {Event} from '@parcel/watcher';
 import type {NapiWorkerPool as INapiWorkerPool} from '@atlaspack/types';
+import {isSuperPackage} from '../isSuperPackage';
+import path from 'path';
 
 export type AtlaspackV3Options = {|
   fs?: AtlaspackNapiOptions['fs'],
@@ -23,9 +25,54 @@ export type AtlaspackV3Options = {|
   lmdb: Lmdb,
   featureFlags?: {[string]: string | boolean},
   napiWorkerPool?: INapiWorkerPool,
-  ...AtlaspackNapiOptions['options'],
+  ...$Diff<
+    AtlaspackNapiOptions['options'],
+    {|
+      jsPaths: AtlaspackNapiOptions['options']['jsPaths'],
+      useBuiltinConfigs: AtlaspackNapiOptions['options']['useBuiltinConfigs'],
+    |},
+  >,
 |};
 
+function getJsPaths(): AtlaspackNapiOptions['options']['jsPaths'] {
+  const dirname = /*#__ATLASPACK_IGNORE__*/ __dirname;
+
+  if (isSuperPackage()) {
+    // dirname: atlaspack/lib/core/core/atlaspack-v3
+    // core: atlaspack/lib/core/core/index.js
+    const corePath = path.join(dirname, '..');
+    // esmodule helpers: atlaspack/lib/transformers/js/esmodule-helpers.js
+    const esmoduleHelpersPath = path.join(
+      dirname,
+      '../../../transformers/js/esmodule-helpers.js',
+    );
+
+    // empty file: atlaspack/lib/core/core/_empty.js
+    const emptyFile = path.join(dirname, '_empty.js');
+
+    return {
+      corePath,
+      esmoduleHelpersSpecifier: path.relative(corePath, esmoduleHelpersPath),
+      esmoduleHelpersIncludeNodeModules: 'atlaspack',
+      emptyFile,
+    };
+  }
+
+  // dirname: @atlaspack/core/lib/atlaspack-v3
+  // core: @atlaspack/core
+  const corePath = path.join(dirname, '../..');
+  // empty file: atlaspack/lib/core/core/_empty.js
+  const emptyFile = path.join(dirname, '_empty.js');
+
+  return {
+    corePath,
+    esmoduleHelpersSpecifier:
+      '@atlaspack/transformer-js/src/esmodule-helpers.js',
+    esmoduleHelpersIncludeNodeModules: '@atlaspack/transformer-js',
+    emptyFile,
+  };
+}
+
 export class AtlaspackV3 {
   _atlaspack_napi: AtlaspackNapi;
 
@@ -52,7 +99,11 @@ export class AtlaspackV3 {
         fs,
         packageManager,
         threads,
-        options,
+        options: {
+          ...options,
+          jsPaths: getJsPaths(),
+          useBuiltinConfigs: isSuperPackage(),
+        },
         napiWorkerPool,
       },
       lmdb,

package/src/atlaspack-v3/NapiWorkerPool.js

@@ -6,7 +6,11 @@ import process from 'process';
 import type {Transferable} from '@atlaspack/rust';
 import {getAvailableThreads} from '@atlaspack/rust';
 
-const WORKER_PATH = path.join(__dirname, 'worker', 'index.js');
+const WORKER_PATH = path.join(
+  /*#__ATLASPACK_IGNORE__*/ __dirname,
+  'worker',
+  'napi-worker.js',
+);
 const ATLASPACK_NAPI_WORKERS =
   process.env.ATLASPACK_NAPI_WORKERS &&
   parseInt(process.env.ATLASPACK_NAPI_WORKERS, 10);

package/src/atlaspack-v3/worker/compat/plugin-config.js

@@ -107,7 +107,7 @@ export class PluginConfig implements IPluginConfig {
           exclude?: boolean,
         |}
       | {|
-          configKey?: string,
+          readTracking?: boolean,
         |},
   ): Promise<?ConfigResultWithFilePath<T>> {
     return this.#inner.getConfigFrom(searchPath, filePaths, options);

package/src/atlaspack-v3/worker/worker.js

@@ -25,6 +25,7 @@ import {
   bundleBehaviorMap,
   dependencyPriorityMap,
 } from './compat';
+import atlaspackInternalPlugins from '../../internal-plugins';
 
 const CONFIG = Symbol.for('parcel-plugin-config');
 
@@ -41,10 +42,16 @@ export class AtlaspackWorker {
 
   loadPlugin: JsCallable<[LoadPluginOptions], Promise<void>> = jsCallable(
     async ({kind, specifier, resolveFrom}) => {
-      let customRequire = module.createRequire(resolveFrom);
-      let resolvedPath = customRequire.resolve(specifier);
-      // $FlowFixMe
-      let resolvedModule = await import(resolvedPath);
+      let resolvedModule;
+      if (atlaspackInternalPlugins && atlaspackInternalPlugins[specifier]) {
+        // If the plugin is available inside the package then use it
+        resolvedModule = atlaspackInternalPlugins[specifier]();
+      } else {
+        let customRequire = module.createRequire(resolveFrom);
+        let resolvedPath = customRequire.resolve(specifier);
+        // $FlowFixMe
+        resolvedModule = await import(resolvedPath);
+      }
 
       let instance = undefined;
       if (resolvedModule.default && resolvedModule.default[CONFIG]) {

package/src/index.js

@@ -10,4 +10,6 @@ export {
   INTERNAL_TRANSFORM,
 } from './Atlaspack';
 
+export {isSuperPackage} from './isSuperPackage';
+
 export * from './atlaspack-v3';

package/src/internal-plugins.js

@@ -0,0 +1,4 @@
+// @flow
+// This file is overridden in super package builds
+// $FlowFixMe
+export default {};

package/src/isSuperPackage.js

@@ -0,0 +1,28 @@
+// @flow strict-local
+let fs = require('fs');
+let {findAncestorFile} = require('@atlaspack/rust');
+
+let dirname = /*#__ATLASPACK_IGNORE__*/ __dirname;
+
+function isSuperPackage(): boolean {
+  if (!dirname) {
+    return false;
+  }
+
+  let packageJson = JSON.parse(
+    // $FlowFixMe
+    fs.readFileSync(findAncestorFile(['package.json'], dirname, '/'), 'utf8'),
+  );
+
+  return packageJson.name === 'atlaspack';
+}
+
+let result;
+
+module.exports.isSuperPackage = (): boolean => {
+  if (result == null) {
+    result = isSuperPackage();
+  }
+
+  return result;
+};

package/src/loadAtlaspackPlugin.js

@@ -8,8 +8,14 @@ import ThrowableDiagnostic, {
   generateJSONCodeHighlights,
   md,
 } from '@atlaspack/diagnostic';
+import {version as ATLASPACK_VERSION} from '../package.json';
+import atlaspackInternalPlugins from './internal-plugins';
 import {findAlternativeNodeModules} from '@atlaspack/utils';
-import {type ProjectPath, toProjectPath} from './projectPath';
+import {
+  type ProjectPath,
+  toProjectPath,
+  toProjectPathUnsafe,
+} from './projectPath';
 
 const NODE_MODULES = `${path.sep}node_modules${path.sep}`;
 const CONFIG = Symbol.for('parcel-plugin-config');
@@ -24,6 +30,22 @@ export default async function loadPlugin<T>(
   version: Semver,
   resolveFrom: ProjectPath,
 |}> {
+  if (atlaspackInternalPlugins && atlaspackInternalPlugins[pluginName]) {
+    let plugin = atlaspackInternalPlugins[pluginName]();
+    plugin = plugin.default || plugin;
+    plugin = plugin[CONFIG];
+    if (!plugin) {
+      throw new Error(
+        `Plugin ${pluginName} is not a valid Atlaspack plugin, should export an instance of a Atlaspack plugin ex. "export default new Reporter({ ... })".`,
+      );
+    }
+    return {
+      plugin,
+      version: ATLASPACK_VERSION,
+      resolveFrom: toProjectPathUnsafe(options.projectRoot),
+    };
+  }
+
   let resolveFrom = configPath;
 
   // Config packages can reference plugins, but cannot contain other plugins within them.

package/src/public/Asset.js

@@ -36,6 +36,7 @@ import {
   BundleBehaviorNames,
 } from '../types';
 import {toInternalSourceLocation} from '../utils';
+import {fromEnvironmentId} from '../EnvironmentManager';
 
 const inspect = Symbol.for('nodejs.util.inspect.custom');
 
@@ -101,7 +102,10 @@ class BaseAsset {
   }
 
   get env(): IEnvironment {
-    return new Environment(this.#asset.value.env, this.#asset.options);
+    return new Environment(
+      fromEnvironmentId(this.#asset.value.env),
+      this.#asset.options,
+    );
   }
 
   get fs(): FileSystem {
@@ -210,7 +214,10 @@ export class Asset extends BaseAsset implements IAsset {
   }
 
   get env(): IEnvironment {
-    this.#env ??= new Environment(this.#asset.value.env, this.#asset.options);
+    this.#env ??= new Environment(
+      fromEnvironmentId(this.#asset.value.env),
+      this.#asset.options,
+    );
     return this.#env;
   }
 

package/src/public/Bundle.js

@@ -34,6 +34,7 @@ import {
 import Target from './Target';
 import {BundleBehaviorNames} from '../types';
 import {fromProjectPath} from '../projectPath';
+import {fromEnvironmentId} from '../EnvironmentManager';
 
 const inspect = Symbol.for('nodejs.util.inspect.custom');
 
@@ -123,7 +124,7 @@ export class Bundle implements IBundle {
   }
 
   get env(): IEnvironment {
-    return new Environment(this.#bundle.env, this.#options);
+    return new Environment(fromEnvironmentId(this.#bundle.env), this.#options);
   }
 
   get needsStableName(): ?boolean {

package/src/public/Config.js

@@ -21,12 +21,89 @@ import {
 import Environment from './Environment';
 import {fromProjectPath, toProjectPath} from '../projectPath';
 import {getFeatureFlag} from '@atlaspack/feature-flags';
+import {fromEnvironmentId} from '../EnvironmentManager';
 
 const internalConfigToConfig: DefaultWeakMap<
   AtlaspackOptions,
   WeakMap<Config, PublicConfig>,
 > = new DefaultWeakMap(() => new WeakMap());
 
+/**
+ * Implements read tracking over an object.
+ *
+ * Calling this function with a non-trivial object like a class instance will fail to work.
+ *
+ * We track reads to fields that resolve to:
+ *
+ * - primitive values
+ * - arrays
+ *
+ * That is, reading a nested field `a.b.c` will make a single call to `onRead` with the path
+ * `['a', 'b', 'c']`.
+ *
+ * In case the value is null or an array, we will track the read as well.
+ *
+ * Iterating over `Object.keys(obj.field)` will register a read for the `['field']` path.
+ * Other reads work normally.
+ *
+ * @example
+ *
+ *     const usedPaths = new Set();
+ *     const onRead = (path) => {
+ *        usedPaths.add(path);
+ *     };
+ *
+ *     const config = makeConfigProxy(onRead, {a: {b: {c: 'd'}}})
+ *     console.log(config.a.b.c);
+ *     console.log(Array.from(usedPaths));
+ *     // We get a single read for the path
+ *     // ['a', 'b', 'c']
+ *
+ */
+export function makeConfigProxy<T>(
+  onRead: (path: string[]) => void,
+  config: T,
+): T {
+  const reportedPaths = new Set();
+  const reportPath = (path) => {
+    if (reportedPaths.has(path)) {
+      return;
+    }
+    reportedPaths.add(path);
+    onRead(path);
+  };
+
+  const makeProxy = (target, path) => {
+    return new Proxy(target, {
+      ownKeys(target) {
+        reportPath(path);
+
+        // $FlowFixMe
+        return Object.getOwnPropertyNames(target);
+      },
+      get(target, prop) {
+        // $FlowFixMe
+        const value = target[prop];
+
+        if (
+          typeof value === 'object' &&
+          value != null &&
+          !Array.isArray(value)
+        ) {
+          return makeProxy(value, [...path, prop]);
+        }
+
+        reportPath([...path, prop]);
+
+        return value;
+      },
+    });
+  };
+
+  // $FlowFixMe
+  return makeProxy(config, []);
+}
+
 export default class PublicConfig implements IConfig {
   #config /*: Config */;
   #pkg /*: ?PackageJSON */;
@@ -46,7 +123,7 @@ export default class PublicConfig implements IConfig {
   }
 
   get env(): Environment {
-    return new Environment(this.#config.env, this.#options);
+    return new Environment(fromEnvironmentId(this.#config.env), this.#options);
   }
 
   get searchPath(): FilePath {
@@ -76,7 +153,7 @@ export default class PublicConfig implements IConfig {
     );
   }
 
-  invalidateOnConfigKeyChange(filePath: FilePath, configKey: string) {
+  invalidateOnConfigKeyChange(filePath: FilePath, configKey: string[]) {
     this.#config.invalidateOnConfigKeyChange.push({
       filePath: toProjectPath(this.#options.projectRoot, filePath),
       configKey,
@@ -144,9 +221,10 @@ export default class PublicConfig implements IConfig {
         |}
       | ?{|
           /**
-           * If specified, only invalidate when this config key changes.
+           * If specified, this function will return a proxy object that will track reads to
+           * config fields and only register invalidations for when those keys change.
            */
-          configKey?: string,
+          readTracking?: boolean,
         |},
   ): Promise<?ConfigResultWithFilePath<T>> {
     let packageKey = options?.packageKey;
@@ -157,7 +235,7 @@ export default class PublicConfig implements IConfig {
 
       if (pkg && pkg.contents[packageKey]) {
         // Invalidate only when the package key changes
-        this.invalidateOnConfigKeyChange(pkg.filePath, packageKey);
+        this.invalidateOnConfigKeyChange(pkg.filePath, [packageKey]);
 
         return {
           contents: pkg.contents[packageKey],
@@ -166,27 +244,24 @@ export default class PublicConfig implements IConfig {
       }
     }
 
-    if (getFeatureFlag('granularTsConfigInvalidation')) {
-      const configKey = options?.configKey;
-      if (configKey != null) {
-        for (let fileName of fileNames) {
-          let config = await this.getConfigFrom(searchPath, [fileName], {
-            exclude: true,
+    const readTracking = options?.readTracking;
+    if (readTracking === true) {
+      for (let fileName of fileNames) {
+        const config = await this.getConfigFrom(searchPath, [fileName], {
+          exclude: true,
+        });
+
+        if (config != null) {
+          return Promise.resolve({
+            contents: makeConfigProxy((keyPath) => {
+              this.invalidateOnConfigKeyChange(config.filePath, keyPath);
+            }, config.contents),
+            filePath: config.filePath,
           });
-
-          if (config && config.contents[configKey]) {
-            // Invalidate only when the package key changes
-            this.invalidateOnConfigKeyChange(config.filePath, configKey);
-
-            return {
-              contents: config.contents[configKey],
-              filePath: config.filePath,
-            };
-          }
         }
-
-        // fall through so that file above invalidations are registered
       }
+
+      // fall through so that file above invalidations are registered
     }
 
     if (fileNames.length === 0) {
@@ -268,11 +343,15 @@ export default class PublicConfig implements IConfig {
 
   getConfig<T>(
     filePaths: Array<FilePath>,
-    options: ?{|
-      packageKey?: string,
-      parse?: boolean,
-      exclude?: boolean,
-    |},
+    options:
+      | ?{|
+          packageKey?: string,
+          parse?: boolean,
+          exclude?: boolean,
+        |}
+      | {|
+          readTracking?: boolean,
+        |},
   ): Promise<?ConfigResultWithFilePath<T>> {
     return this.getConfigFrom(this.searchPath, filePaths, options);
   }
@@ -282,7 +361,9 @@ export default class PublicConfig implements IConfig {
       return this.#pkg;
     }
 
-    let pkgConfig = await this.getConfig<PackageJSON>(['package.json']);
+    let pkgConfig = await this.getConfig<PackageJSON>(['package.json'], {
+      readTracking: getFeatureFlag('granularTsConfigInvalidation'),
+    });
     if (!pkgConfig) {
       return null;
     }