@atlaspack/core 2.16.2-canary.57 → 2.16.2-canary.59

package/src/public/Config.js

@@ -28,6 +28,82 @@ const internalConfigToConfig: DefaultWeakMap<
   WeakMap<Config, PublicConfig>,
 > = new DefaultWeakMap(() => new WeakMap());
 
+/**
+ * Implements read tracking over an object.
+ *
+ * Calling this function with a non-trivial object like a class instance will fail to work.
+ *
+ * We track reads to fields that resolve to:
+ *
+ * - primitive values
+ * - arrays
+ *
+ * That is, reading a nested field `a.b.c` will make a single call to `onRead` with the path
+ * `['a', 'b', 'c']`.
+ *
+ * In case the value is null or an array, we will track the read as well.
+ *
+ * Iterating over `Object.keys(obj.field)` will register a read for the `['field']` path.
+ * Other reads work normally.
+ *
+ * @example
+ *
+ *     const usedPaths = new Set();
+ *     const onRead = (path) => {
+ *        usedPaths.add(path);
+ *     };
+ *
+ *     const config = makeConfigProxy(onRead, {a: {b: {c: 'd'}}})
+ *     console.log(config.a.b.c);
+ *     console.log(Array.from(usedPaths));
+ *     // We get a single read for the path
+ *     // ['a', 'b', 'c']
+ *
+ */
+export function makeConfigProxy<T>(
+  onRead: (path: string[]) => void,
+  config: T,
+): T {
+  const reportedPaths = new Set();
+  const reportPath = (path) => {
+    if (reportedPaths.has(path)) {
+      return;
+    }
+    reportedPaths.add(path);
+    onRead(path);
+  };
+
+  const makeProxy = (target, path) => {
+    return new Proxy(target, {
+      ownKeys(target) {
+        reportPath(path);
+
+        // $FlowFixMe
+        return Object.getOwnPropertyNames(target);
+      },
+      get(target, prop) {
+        // $FlowFixMe
+        const value = target[prop];
+
+        if (
+          typeof value === 'object' &&
+          value != null &&
+          !Array.isArray(value)
+        ) {
+          return makeProxy(value, [...path, prop]);
+        }
+
+        reportPath([...path, prop]);
+
+        return value;
+      },
+    });
+  };
+
+  // $FlowFixMe
+  return makeProxy(config, []);
+}
+
 export default class PublicConfig implements IConfig {
   #config /*: Config */;
   #pkg /*: ?PackageJSON */;
@@ -77,7 +153,7 @@ export default class PublicConfig implements IConfig {
     );
   }
 
-  invalidateOnConfigKeyChange(filePath: FilePath, configKey: string) {
+  invalidateOnConfigKeyChange(filePath: FilePath, configKey: string[]) {
     this.#config.invalidateOnConfigKeyChange.push({
       filePath: toProjectPath(this.#options.projectRoot, filePath),
       configKey,
@@ -145,9 +221,10 @@ export default class PublicConfig implements IConfig {
         |}
       | ?{|
           /**
-           * If specified, only invalidate when this config key changes.
+           * If specified, this function will return a proxy object that will track reads to
+           * config fields and only register invalidations for when those keys change.
            */
-          configKey?: string,
+          readTracking?: boolean,
         |},
   ): Promise<?ConfigResultWithFilePath<T>> {
     let packageKey = options?.packageKey;
@@ -158,7 +235,7 @@ export default class PublicConfig implements IConfig {
 
       if (pkg && pkg.contents[packageKey]) {
         // Invalidate only when the package key changes
-        this.invalidateOnConfigKeyChange(pkg.filePath, packageKey);
+        this.invalidateOnConfigKeyChange(pkg.filePath, [packageKey]);
 
         return {
           contents: pkg.contents[packageKey],
@@ -167,27 +244,24 @@ export default class PublicConfig implements IConfig {
       }
     }
 
-    if (getFeatureFlag('granularTsConfigInvalidation')) {
-      const configKey = options?.configKey;
-      if (configKey != null) {
-        for (let fileName of fileNames) {
-          let config = await this.getConfigFrom(searchPath, [fileName], {
-            exclude: true,
+    const readTracking = options?.readTracking;
+    if (readTracking === true) {
+      for (let fileName of fileNames) {
+        const config = await this.getConfigFrom(searchPath, [fileName], {
+          exclude: true,
+        });
+
+        if (config != null) {
+          return Promise.resolve({
+            contents: makeConfigProxy((keyPath) => {
+              this.invalidateOnConfigKeyChange(config.filePath, keyPath);
+            }, config.contents),
+            filePath: config.filePath,
           });
-
-          if (config && config.contents[configKey]) {
-            // Invalidate only when the package key changes
-            this.invalidateOnConfigKeyChange(config.filePath, configKey);
-
-            return {
-              contents: config.contents[configKey],
-              filePath: config.filePath,
-            };
-          }
         }
-
-        // fall through so that file above invalidations are registered
       }
+
+      // fall through so that file above invalidations are registered
     }
 
     if (fileNames.length === 0) {
@@ -269,11 +343,15 @@ export default class PublicConfig implements IConfig {
 
   getConfig<T>(
     filePaths: Array<FilePath>,
-    options: ?{|
-      packageKey?: string,
-      parse?: boolean,
-      exclude?: boolean,
-    |},
+    options:
+      | ?{|
+          packageKey?: string,
+          parse?: boolean,
+          exclude?: boolean,
+        |}
+      | {|
+          readTracking?: boolean,
+        |},
   ): Promise<?ConfigResultWithFilePath<T>> {
     return this.getConfigFrom(this.searchPath, filePaths, options);
   }
@@ -283,7 +361,9 @@ export default class PublicConfig implements IConfig {
       return this.#pkg;
     }
 
-    let pkgConfig = await this.getConfig<PackageJSON>(['package.json']);
+    let pkgConfig = await this.getConfig<PackageJSON>(['package.json'], {
+      readTracking: getFeatureFlag('granularTsConfigInvalidation'),
+    });
     if (!pkgConfig) {
       return null;
     }

package/src/requests/ConfigRequest.js

@@ -63,7 +63,7 @@ export type ConfigRequest = {
   invalidateOnFileChange: Set<ProjectPath>,
   invalidateOnConfigKeyChange: Array<{|
     filePath: ProjectPath,
-    configKey: string,
+    configKey: string[],
   |}>,
   invalidateOnFileCreate: Array<InternalFileCreateInvalidation>,
   invalidateOnEnvChange: Set<string>,
@@ -108,34 +108,58 @@ export async function loadPluginConfig<T: PluginWithLoadConfig>(
   }
 }
 
+/**
+ * Return value at a given key path within an object.
+ *
+ * @example
+ *     const obj = { a: { b: { c: 'd' } } };
+ *     getValueAtPath(obj, ['a', 'b', 'c']);        // 'd'
+ *     getValueAtPath(obj, ['a', 'b', 'd']);        // undefined
+ *     getValueAtPath(obj, ['a', 'b']);             // { c: 'd' }
+ *     getValueAtPath(obj, ['a', 'b', 'c', 'd']);   // undefined
+ */
+export function getValueAtPath(obj: Object, key: string[]): any {
+  let current = obj;
+  for (let part of key) {
+    if (current == null) {
+      return undefined;
+    }
+    current = current[part];
+  }
+  return current;
+}
+
 const configKeyCache = createBuildCache();
 
 export async function getConfigKeyContentHash(
   filePath: ProjectPath,
-  configKey: string,
+  configKey: string[],
   options: AtlaspackOptions,
 ): Async<string> {
-  let cacheKey = `${fromProjectPathRelative(filePath)}:${configKey}`;
+  let cacheKey = `${fromProjectPathRelative(filePath)}:${JSON.stringify(
+    configKey,
+  )}`;
   let cachedValue = configKeyCache.get(cacheKey);
 
   if (cachedValue) {
     return cachedValue;
   }
 
-  let conf = await readConfig(
+  const conf = await readConfig(
     options.inputFS,
     fromProjectPath(options.projectRoot, filePath),
   );
 
-  if (conf == null || conf.config[configKey] == null) {
+  const value = getValueAtPath(conf?.config, configKey);
+  if (conf == null || value == null) {
     // This can occur when a config key has been removed entirely during `respondToFSEvents`
     return '';
   }
 
-  let contentHash =
-    typeof conf.config[configKey] === 'object'
-      ? hashObject(conf.config[configKey])
-      : hashString(JSON.stringify(conf.config[configKey]));
+  const contentHash =
+    typeof value === 'object'
+      ? hashObject(value)
+      : hashString(JSON.stringify(value));
 
   configKeyCache.set(cacheKey, contentHash);
 

package/src/types.js

@@ -499,7 +499,7 @@ export type Config = {|
   invalidateOnFileChange: Set<ProjectPath>,
   invalidateOnConfigKeyChange: Array<{|
     filePath: ProjectPath,
-    configKey: string,
+    configKey: string[],
   |}>,
   invalidateOnFileCreate: Array<InternalFileCreateInvalidation>,
   invalidateOnEnvChange: Set<string>,

package/test/EnvironmentManager.test.js

@@ -0,0 +1,192 @@
+// @flow strict-local
+
+import assert from 'assert';
+import nullthrows from 'nullthrows';
+import sinon from 'sinon';
+import {ATLASPACK_VERSION} from '../src/constants';
+import {DEFAULT_FEATURE_FLAGS, setFeatureFlags} from '@atlaspack/feature-flags';
+import {setAllEnvironments, getAllEnvironments} from '@atlaspack/rust';
+import {
+  loadEnvironmentsFromCache,
+  writeEnvironmentsToCache,
+} from '../src/EnvironmentManager';
+import {DEFAULT_OPTIONS} from './test-utils';
+import {LMDBLiteCache} from '@atlaspack/cache';
+
+const options = {
+  ...DEFAULT_OPTIONS,
+  cache: new LMDBLiteCache(DEFAULT_OPTIONS.cacheDir),
+};
+
+describe('EnvironmentManager', () => {
+  const env1 = {
+    id: 'd821e85f6b50315e',
+    context: 'browser',
+    engines: {browsers: ['> 0.25%']},
+    includeNodeModules: true,
+    outputFormat: 'global',
+    isLibrary: false,
+    shouldOptimize: false,
+    shouldScopeHoist: false,
+    loc: undefined,
+    sourceMap: undefined,
+    sourceType: 'module',
+    unstableSingleFileOutput: false,
+  };
+  const env2 = {
+    id: 'de92f48baa8448d2',
+    context: 'node',
+    engines: {
+      browsers: [],
+      node: '>= 8',
+    },
+    includeNodeModules: false,
+    outputFormat: 'commonjs',
+    isLibrary: true,
+    shouldOptimize: true,
+    shouldScopeHoist: true,
+    loc: null,
+    sourceMap: null,
+    sourceType: 'module',
+    unstableSingleFileOutput: false,
+  };
+
+  beforeEach(async () => {
+    await options.cache.ensure();
+
+    for (const key of options.cache.keys()) {
+      await options.cache.getNativeRef().delete(key);
+    }
+    setAllEnvironments([]);
+
+    setFeatureFlags({
+      ...DEFAULT_FEATURE_FLAGS,
+      environmentDeduplication: true,
+    });
+  });
+
+  it('should store environments by ID in the cache', async () => {
+    setAllEnvironments([env1]);
+    await writeEnvironmentsToCache(options.cache);
+
+    const cachedEnv1 = await options.cache.get(
+      `Environment/${ATLASPACK_VERSION}/${env1.id}`,
+    );
+    assert.deepEqual(cachedEnv1, env1, 'Environment 1 should be cached');
+  });
+
+  it('should list all environment IDs in the environment manager', async () => {
+    const environmentIds = [env1.id, env2.id];
+    setAllEnvironments([env1, env2]);
+    await writeEnvironmentsToCache(options.cache);
+
+    const cachedEnvIds = await options.cache.get(
+      `EnvironmentManager/${ATLASPACK_VERSION}`,
+    );
+    const cachedIdsArray = nullthrows(cachedEnvIds);
+    assert.equal(
+      cachedIdsArray.length,
+      environmentIds.length,
+      'Should have same number of IDs',
+    );
+    assert(
+      environmentIds.every((id) => cachedIdsArray.includes(id)),
+      'All environment IDs should be present in cache',
+    );
+  });
+
+  it('should write all environments to cache using writeEnvironmentsToCache', async () => {
+    setAllEnvironments([env1, env2]);
+    await writeEnvironmentsToCache(options.cache);
+
+    // Verify each environment was stored individually
+    const cachedEnv1 = await options.cache.get(
+      `Environment/${ATLASPACK_VERSION}/${env1.id}`,
+    );
+    const cachedEnv2 = await options.cache.get(
+      `Environment/${ATLASPACK_VERSION}/${env2.id}`,
+    );
+    assert.deepEqual(cachedEnv1, env1, 'Environment 1 should be cached');
+    assert.deepEqual(cachedEnv2, env2, 'Environment 2 should be cached');
+
+    // Verify environment IDs were stored in manager
+    const cachedEnvIds = await options.cache.get(
+      `EnvironmentManager/${ATLASPACK_VERSION}`,
+    );
+    const cachedIdsArray = nullthrows(cachedEnvIds);
+    assert(
+      cachedIdsArray.length === 2 &&
+        [env1.id, env2.id].every((id) => cachedIdsArray.includes(id)),
+      'Environment IDs should be stored in manager',
+    );
+  });
+
+  it('should load environments from cache on loadRequestGraph on a subsequent build', async () => {
+    // Simulate cache written on a first build
+    setAllEnvironments([env1, env2]);
+    await writeEnvironmentsToCache(options.cache);
+
+    await loadEnvironmentsFromCache(options.cache);
+
+    const loadedEnvironments = getAllEnvironments();
+    assert.equal(
+      loadedEnvironments.length,
+      2,
+      'Should load 2 environments from cache',
+    );
+
+    const env1Loaded = loadedEnvironments.find((e) => e.id === env1.id);
+    const env2Loaded = loadedEnvironments.find((e) => e.id === env2.id);
+
+    assert.deepEqual(
+      env1Loaded,
+      env1,
+      'First environment should match cached environment',
+    );
+    assert.deepEqual(
+      env2Loaded,
+      env2,
+      'Second environment should match cached environment',
+    );
+  });
+
+  it('should handle empty cache gracefully without calling setAllEnvironments', async () => {
+    const setAllEnvironmentsSpy = sinon.spy(setAllEnvironments);
+
+    await assert.doesNotReject(
+      loadEnvironmentsFromCache(options.cache),
+      'loadEnvironmentsFromCache should not throw when cache is empty',
+    );
+
+    assert.equal(
+      setAllEnvironmentsSpy.callCount,
+      0,
+      'setAllEnvironments should not be called when loading from empty cache',
+    );
+  });
+
+  it('should not load environments from a different version', async () => {
+    const setAllEnvironmentsSpy = sinon.spy(setAllEnvironments);
+    const differentVersion = '2.17.2'; // A different version than ATLASPACK_VERSION
+
+    // Store an environment with a different version
+    await options.cache.set(`Environment/${differentVersion}/${env1.id}`, env1);
+    await options.cache.set(`EnvironmentManager/${differentVersion}`, [
+      env1.id,
+    ]);
+
+    await loadEnvironmentsFromCache(options.cache);
+
+    assert.equal(
+      setAllEnvironmentsSpy.callCount,
+      0,
+      'setAllEnvironments should not be called when loading from different version',
+    );
+    const loadedEnvironments = getAllEnvironments();
+    assert.equal(
+      loadedEnvironments.length,
+      0,
+      'Should not load any environments from different version',
+    );
+  });
+});

package/test/public/Config.test.js

@@ -0,0 +1,108 @@
+// @flow strict-local
+
+import sinon from 'sinon';
+import {makeConfigProxy} from '../../src/public/Config';
+import assert from 'assert';
+
+describe('makeConfigProxy', () => {
+  it('tracks reads to nested fields', () => {
+    const onRead = sinon.spy();
+    const target = {a: {b: {c: 'd'}}};
+    const config = makeConfigProxy(onRead, target);
+    config.a.b.c;
+    assert.ok(onRead.calledWith(['a', 'b', 'c']));
+    assert.ok(onRead.calledOnce);
+  });
+
+  it('works for reading package.json dependencies', () => {
+    const packageJson = {
+      dependencies: {
+        react: '18.2.0',
+      },
+    };
+
+    const onRead = sinon.spy();
+    const config = makeConfigProxy(onRead, packageJson);
+    assert.equal(config.dependencies.react, '18.2.0');
+    // $FlowFixMe
+    assert.equal(config.dependencies.preact, undefined);
+    assert.ok(onRead.calledWith(['dependencies', 'react']));
+    assert.ok(onRead.calledWith(['dependencies', 'preact']));
+    assert.equal(onRead.callCount, 2);
+  });
+
+  it('will track reads for any missing or null keys', () => {
+    const packageJson = {
+      dependencies: {
+        react: '18.2.0',
+      },
+    };
+
+    const onRead = sinon.spy();
+    const config = makeConfigProxy(onRead, packageJson);
+
+    // $FlowFixMe
+    assert.equal(config.alias?.react, undefined);
+    assert.ok(onRead.calledWith(['alias']));
+    assert.equal(onRead.callCount, 1);
+  });
+
+  it('iterating over keys works normally and will register a read for the key being enumerated', () => {
+    const packageJson = {
+      nested: {
+        dependencies: {
+          react: '18.2.0',
+          'react-dom': '18.2.0',
+          'react-router': '6.14.2',
+        },
+      },
+    };
+
+    const onRead = sinon.spy();
+    const config = makeConfigProxy(onRead, packageJson);
+    assert.equal(Object.keys(config.nested.dependencies).length, 3);
+
+    assert.ok(onRead.calledWith(['nested', 'dependencies']));
+  });
+
+  it('if a key has an array value we will track a read for that key', () => {
+    const packageJson = {
+      scripts: ['build', 'test'],
+    };
+
+    const onRead = sinon.spy();
+    const config = makeConfigProxy(onRead, packageJson);
+    assert.equal(config.scripts[0], 'build');
+    assert.equal(onRead.callCount, 1);
+    assert.ok(onRead.calledWith(['scripts']));
+  });
+
+  it('if a key array value is iterated over we will track a read for that key', () => {
+    const packageJson = {
+      scripts: ['build', 'test'],
+    };
+
+    const onRead = sinon.spy();
+    const config = makeConfigProxy(onRead, packageJson);
+    let scriptCount = 0;
+    // eslint-disable-next-line no-unused-vars
+    for (const _script of config.scripts) {
+      scriptCount += 1;
+    }
+    assert.equal(scriptCount, 2);
+    assert.ok(onRead.calledWith(['scripts']));
+    assert.equal(onRead.callCount, 1);
+  });
+
+  it('if a key array value length is verified we will track a read for that key', () => {
+    const packageJson = {
+      scripts: ['build', 'test'],
+    };
+
+    const onRead = sinon.spy();
+    const config = makeConfigProxy(onRead, packageJson);
+    assert.equal(config.scripts.length, 2);
+    assert.ok(onRead.calledWith(['scripts']));
+    assert.equal(onRead.callCount, 1);
+  });
+});