@atlaspack/core 2.16.2-canary.58 → 2.16.2-canary.59

package/lib/EnvironmentManager.js

@@ -4,8 +4,10 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.fromEnvironmentId = fromEnvironmentId;
+exports.loadEnvironmentsFromCache = loadEnvironmentsFromCache;
 exports.toEnvironmentId = toEnvironmentId;
 exports.toEnvironmentRef = toEnvironmentRef;
+exports.writeEnvironmentsToCache = writeEnvironmentsToCache;
 function _rust() {
   const data = require("@atlaspack/rust");
   _rust = function () {
@@ -20,6 +22,14 @@ function _featureFlags() {
   };
   return data;
 }
+function _logger() {
+  const data = require("@atlaspack/logger");
+  _logger = function () {
+    return data;
+  };
+  return data;
+}
+var _constants = require("./constants");
 /*!
  * At the moment we're doing this change for `CoreEnvironment`,
  * but the same change must be made for `TypesEnvironment` in @atlaspack/types.
@@ -77,4 +87,51 @@ function fromEnvironmentId(id) {
   const env = Object.freeze((0, _rust().getEnvironment)(id));
   localEnvironmentCache.set(id, env);
   return env;
+}
+
+/**
+ * Writes all environments and their IDs to the cache
+ * @param {Cache} cache
+ * @returns {Promise<void>}
+ */
+async function writeEnvironmentsToCache(cache) {
+  const environments = (0, _rust().getAllEnvironments)();
+  const environmentIds = new Set();
+
+  // Store each environment individually
+  for (const env of environments) {
+    environmentIds.add(env.id);
+    const envKey = `Environment/${_constants.ATLASPACK_VERSION}/${env.id}`;
+    await (0, _logger().instrument)(`RequestTracker::writeToCache::cache.put(${envKey})`, async () => {
+      await cache.set(envKey, env);
+    });
+  }
+
+  // Store the list of environment IDs
+  await (0, _logger().instrument)(`RequestTracker::writeToCache::cache.put(${`EnvironmentManager/${_constants.ATLASPACK_VERSION}`})`, async () => {
+    await cache.set(`EnvironmentManager/${_constants.ATLASPACK_VERSION}`, Array.from(environmentIds));
+  });
+}
+
+/**
+ * Loads all environments and their IDs from the cache
+ * @param {Cache} cache
+ * @returns {Promise<void>}
+ */
+async function loadEnvironmentsFromCache(cache) {
+  const cachedEnvIds = await cache.get(`EnvironmentManager/${_constants.ATLASPACK_VERSION}`);
+  if (cachedEnvIds == null) {
+    return;
+  }
+  const environments = [];
+  for (const envId of cachedEnvIds) {
+    const envKey = `Environment/${_constants.ATLASPACK_VERSION}/${envId}`;
+    const cachedEnv = await cache.get(envKey);
+    if (cachedEnv != null) {
+      environments.push(cachedEnv);
+    }
+  }
+  if (environments.length > 0) {
+    (0, _rust().setAllEnvironments)(environments);
+  }
 }

package/lib/RequestTracker.js

@@ -95,6 +95,7 @@ function _perf_hooks() {
   };
   return data;
 }
+var _EnvironmentManager = require("./EnvironmentManager");
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
 function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
@@ -1023,6 +1024,9 @@ class RequestTracker {
         total,
         size: this.graph.nodes.length
       });
+      if ((0, _featureFlags().getFeatureFlag)('environmentDeduplication')) {
+        await (0, _EnvironmentManager.writeEnvironmentsToCache)(options.cache);
+      }
       let serialisedGraph = this.graph.serialize();
 
       // Delete an existing request graph cache, to prevent invalid states
@@ -1214,6 +1218,9 @@ async function loadRequestGraph(options) {
       ...commonMeta
     }
   });
+  if ((0, _featureFlags().getFeatureFlag)('environmentDeduplication')) {
+    await (0, _EnvironmentManager.loadEnvironmentsFromCache)(options.cache);
+  }
   const hasRequestGraphInCache = (0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements') ? await options.cache.has(requestGraphKey) : await options.cache.hasLargeBlob(requestGraphKey);
   if (hasRequestGraphInCache) {
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atlaspack/core",
-  "version": "2.16.2-canary.58+1e32d4eae",
+  "version": "2.16.2-canary.59+0b2f6f557",
   "license": "(MIT OR Apache-2.0)",
   "publishConfig": {
     "access": "public"
@@ -21,21 +21,21 @@
     "check-ts": "tsc --noEmit index.d.ts"
   },
   "dependencies": {
-    "@atlaspack/build-cache": "2.13.3-canary.126+1e32d4eae",
-    "@atlaspack/cache": "3.1.1-canary.58+1e32d4eae",
-    "@atlaspack/diagnostic": "2.14.1-canary.126+1e32d4eae",
-    "@atlaspack/events": "2.14.1-canary.126+1e32d4eae",
-    "@atlaspack/feature-flags": "2.14.1-canary.126+1e32d4eae",
-    "@atlaspack/fs": "2.14.5-canary.58+1e32d4eae",
-    "@atlaspack/graph": "3.4.1-canary.126+1e32d4eae",
-    "@atlaspack/logger": "2.14.5-canary.58+1e32d4eae",
-    "@atlaspack/package-manager": "2.14.5-canary.58+1e32d4eae",
-    "@atlaspack/plugin": "2.14.5-canary.58+1e32d4eae",
-    "@atlaspack/profiler": "2.14.1-canary.126+1e32d4eae",
-    "@atlaspack/rust": "3.2.1-canary.58+1e32d4eae",
-    "@atlaspack/types": "2.14.5-canary.58+1e32d4eae",
-    "@atlaspack/utils": "2.14.5-canary.58+1e32d4eae",
-    "@atlaspack/workers": "2.14.5-canary.58+1e32d4eae",
+    "@atlaspack/build-cache": "2.13.3-canary.127+0b2f6f557",
+    "@atlaspack/cache": "3.1.1-canary.59+0b2f6f557",
+    "@atlaspack/diagnostic": "2.14.1-canary.127+0b2f6f557",
+    "@atlaspack/events": "2.14.1-canary.127+0b2f6f557",
+    "@atlaspack/feature-flags": "2.14.1-canary.127+0b2f6f557",
+    "@atlaspack/fs": "2.14.5-canary.59+0b2f6f557",
+    "@atlaspack/graph": "3.4.1-canary.127+0b2f6f557",
+    "@atlaspack/logger": "2.14.5-canary.59+0b2f6f557",
+    "@atlaspack/package-manager": "2.14.5-canary.59+0b2f6f557",
+    "@atlaspack/plugin": "2.14.5-canary.59+0b2f6f557",
+    "@atlaspack/profiler": "2.14.1-canary.127+0b2f6f557",
+    "@atlaspack/rust": "3.2.1-canary.59+0b2f6f557",
+    "@atlaspack/types": "2.14.5-canary.59+0b2f6f557",
+    "@atlaspack/utils": "2.14.5-canary.59+0b2f6f557",
+    "@atlaspack/workers": "2.14.5-canary.59+0b2f6f557",
     "@mischnic/json-sourcemap": "^0.1.0",
     "@parcel/source-map": "^2.1.1",
     "base-x": "^3.0.8",
@@ -71,5 +71,5 @@
     "./src/serializerCore.js": "./src/serializerCore.browser.js"
   },
   "type": "commonjs",
-  "gitHead": "1e32d4eae6b3af3968e8a0ef97d35b4347fd4196"
+  "gitHead": "0b2f6f55794d3ff6e2f5a41f963e7e5dd8ad9f8d"
 }

package/src/EnvironmentManager.js

@@ -4,8 +4,16 @@
  * but the same change must be made for `TypesEnvironment` in @atlaspack/types.
  */
 import type {Environment as CoreEnvironment} from './types';
-import {addEnvironment, getEnvironment} from '@atlaspack/rust';
+import {type Cache} from '@atlaspack/cache';
+import {
+  addEnvironment,
+  getEnvironment,
+  getAllEnvironments,
+  setAllEnvironments,
+} from '@atlaspack/rust';
 import {getFeatureFlag} from '@atlaspack/feature-flags';
+import {instrument} from '@atlaspack/logger';
+import {ATLASPACK_VERSION} from './constants';
 
 const localEnvironmentCache = new Map<string, CoreEnvironment>();
 
@@ -74,3 +82,64 @@ export function fromEnvironmentId(id: EnvironmentRef): CoreEnvironment {
   localEnvironmentCache.set(id, env);
   return env;
 }
+
+/**
+ * Writes all environments and their IDs to the cache
+ * @param {Cache} cache
+ * @returns {Promise<void>}
+ */
+export async function writeEnvironmentsToCache(cache: Cache): Promise<void> {
+  const environments = getAllEnvironments();
+  const environmentIds = new Set<string>();
+
+  // Store each environment individually
+  for (const env of environments) {
+    environmentIds.add(env.id);
+    const envKey = `Environment/${ATLASPACK_VERSION}/${env.id}`;
+
+    await instrument(
+      `RequestTracker::writeToCache::cache.put(${envKey})`,
+      async () => {
+        await cache.set(envKey, env);
+      },
+    );
+  }
+
+  // Store the list of environment IDs
+  await instrument(
+    `RequestTracker::writeToCache::cache.put(${`EnvironmentManager/${ATLASPACK_VERSION}`})`,
+    async () => {
+      await cache.set(
+        `EnvironmentManager/${ATLASPACK_VERSION}`,
+        Array.from(environmentIds),
+      );
+    },
+  );
+}
+
+/**
+ * Loads all environments and their IDs from the cache
+ * @param {Cache} cache
+ * @returns {Promise<void>}
+ */
+export async function loadEnvironmentsFromCache(cache: Cache): Promise<void> {
+  const cachedEnvIds = await cache.get(
+    `EnvironmentManager/${ATLASPACK_VERSION}`,
+  );
+
+  if (cachedEnvIds == null) {
+    return;
+  }
+
+  const environments = [];
+  for (const envId of cachedEnvIds) {
+    const envKey = `Environment/${ATLASPACK_VERSION}/${envId}`;
+    const cachedEnv = await cache.get(envKey);
+    if (cachedEnv != null) {
+      environments.push(cachedEnv);
+    }
+  }
+  if (environments.length > 0) {
+    setAllEnvironments(environments);
+  }
+}

package/src/RequestTracker.js

@@ -30,15 +30,15 @@ import nullthrows from 'nullthrows';
 
 import {
   ATLASPACK_VERSION,
-  VALID,
-  INITIAL_BUILD,
   FILE_CREATE,
-  FILE_UPDATE,
   FILE_DELETE,
+  FILE_UPDATE,
   ENV_CHANGE,
+  ERROR,
+  INITIAL_BUILD,
   OPTION_CHANGE,
   STARTUP,
-  ERROR,
+  VALID,
 } from './constants';
 import type {AtlaspackV3} from './atlaspack-v3/AtlaspackV3';
 import {
@@ -71,6 +71,11 @@ import type {
 import {BuildAbortError, assertSignalNotAborted, hashFromOption} from './utils';
 import {performance} from 'perf_hooks';
 
+import {
+  loadEnvironmentsFromCache,
+  writeEnvironmentsToCache,
+} from './EnvironmentManager';
+
 export const requestGraphEdgeTypes = {
   subrequest: 2,
   invalidated_by_update: 3,
@@ -1572,6 +1577,10 @@ export default class RequestTracker {
         size: this.graph.nodes.length,
       });
 
+      if (getFeatureFlag('environmentDeduplication')) {
+        await writeEnvironmentsToCache(options.cache);
+      }
+
       let serialisedGraph = this.graph.serialize();
 
       // Delete an existing request graph cache, to prevent invalid states
@@ -1856,6 +1865,10 @@ async function loadRequestGraph(options): Async<RequestGraph> {
     },
   });
 
+  if (getFeatureFlag('environmentDeduplication')) {
+    await loadEnvironmentsFromCache(options.cache);
+  }
+
   const hasRequestGraphInCache = getFeatureFlag('cachePerformanceImprovements')
     ? await options.cache.has(requestGraphKey)
     : await options.cache.hasLargeBlob(requestGraphKey);

package/test/EnvironmentManager.test.js

@@ -0,0 +1,192 @@
+// @flow strict-local
+
+import assert from 'assert';
+import nullthrows from 'nullthrows';
+import sinon from 'sinon';
+import {ATLASPACK_VERSION} from '../src/constants';
+import {DEFAULT_FEATURE_FLAGS, setFeatureFlags} from '@atlaspack/feature-flags';
+import {setAllEnvironments, getAllEnvironments} from '@atlaspack/rust';
+import {
+  loadEnvironmentsFromCache,
+  writeEnvironmentsToCache,
+} from '../src/EnvironmentManager';
+import {DEFAULT_OPTIONS} from './test-utils';
+import {LMDBLiteCache} from '@atlaspack/cache';
+
+const options = {
+  ...DEFAULT_OPTIONS,
+  cache: new LMDBLiteCache(DEFAULT_OPTIONS.cacheDir),
+};
+
+describe('EnvironmentManager', () => {
+  const env1 = {
+    id: 'd821e85f6b50315e',
+    context: 'browser',
+    engines: {browsers: ['> 0.25%']},
+    includeNodeModules: true,
+    outputFormat: 'global',
+    isLibrary: false,
+    shouldOptimize: false,
+    shouldScopeHoist: false,
+    loc: undefined,
+    sourceMap: undefined,
+    sourceType: 'module',
+    unstableSingleFileOutput: false,
+  };
+  const env2 = {
+    id: 'de92f48baa8448d2',
+    context: 'node',
+    engines: {
+      browsers: [],
+      node: '>= 8',
+    },
+    includeNodeModules: false,
+    outputFormat: 'commonjs',
+    isLibrary: true,
+    shouldOptimize: true,
+    shouldScopeHoist: true,
+    loc: null,
+    sourceMap: null,
+    sourceType: 'module',
+    unstableSingleFileOutput: false,
+  };
+
+  beforeEach(async () => {
+    await options.cache.ensure();
+
+    for (const key of options.cache.keys()) {
+      await options.cache.getNativeRef().delete(key);
+    }
+    setAllEnvironments([]);
+
+    setFeatureFlags({
+      ...DEFAULT_FEATURE_FLAGS,
+      environmentDeduplication: true,
+    });
+  });
+
+  it('should store environments by ID in the cache', async () => {
+    setAllEnvironments([env1]);
+    await writeEnvironmentsToCache(options.cache);
+
+    const cachedEnv1 = await options.cache.get(
+      `Environment/${ATLASPACK_VERSION}/${env1.id}`,
+    );
+    assert.deepEqual(cachedEnv1, env1, 'Environment 1 should be cached');
+  });
+
+  it('should list all environment IDs in the environment manager', async () => {
+    const environmentIds = [env1.id, env2.id];
+    setAllEnvironments([env1, env2]);
+    await writeEnvironmentsToCache(options.cache);
+
+    const cachedEnvIds = await options.cache.get(
+      `EnvironmentManager/${ATLASPACK_VERSION}`,
+    );
+    const cachedIdsArray = nullthrows(cachedEnvIds);
+    assert.equal(
+      cachedIdsArray.length,
+      environmentIds.length,
+      'Should have same number of IDs',
+    );
+    assert(
+      environmentIds.every((id) => cachedIdsArray.includes(id)),
+      'All environment IDs should be present in cache',
+    );
+  });
+
+  it('should write all environments to cache using writeEnvironmentsToCache', async () => {
+    setAllEnvironments([env1, env2]);
+    await writeEnvironmentsToCache(options.cache);
+
+    // Verify each environment was stored individually
+    const cachedEnv1 = await options.cache.get(
+      `Environment/${ATLASPACK_VERSION}/${env1.id}`,
+    );
+    const cachedEnv2 = await options.cache.get(
+      `Environment/${ATLASPACK_VERSION}/${env2.id}`,
+    );
+    assert.deepEqual(cachedEnv1, env1, 'Environment 1 should be cached');
+    assert.deepEqual(cachedEnv2, env2, 'Environment 2 should be cached');
+
+    // Verify environment IDs were stored in manager
+    const cachedEnvIds = await options.cache.get(
+      `EnvironmentManager/${ATLASPACK_VERSION}`,
+    );
+    const cachedIdsArray = nullthrows(cachedEnvIds);
+    assert(
+      cachedIdsArray.length === 2 &&
+        [env1.id, env2.id].every((id) => cachedIdsArray.includes(id)),
+      'Environment IDs should be stored in manager',
+    );
+  });
+
+  it('should load environments from cache on loadRequestGraph on a subsequent build', async () => {
+    // Simulate cache written on a first build
+    setAllEnvironments([env1, env2]);
+    await writeEnvironmentsToCache(options.cache);
+
+    await loadEnvironmentsFromCache(options.cache);
+
+    const loadedEnvironments = getAllEnvironments();
+    assert.equal(
+      loadedEnvironments.length,
+      2,
+      'Should load 2 environments from cache',
+    );
+
+    const env1Loaded = loadedEnvironments.find((e) => e.id === env1.id);
+    const env2Loaded = loadedEnvironments.find((e) => e.id === env2.id);
+
+    assert.deepEqual(
+      env1Loaded,
+      env1,
+      'First environment should match cached environment',
+    );
+    assert.deepEqual(
+      env2Loaded,
+      env2,
+      'Second environment should match cached environment',
+    );
+  });
+
+  it('should handle empty cache gracefully without calling setAllEnvironments', async () => {
+    const setAllEnvironmentsSpy = sinon.spy(setAllEnvironments);
+
+    await assert.doesNotReject(
+      loadEnvironmentsFromCache(options.cache),
+      'loadEnvironmentsFromCache should not throw when cache is empty',
+    );
+
+    assert.equal(
+      setAllEnvironmentsSpy.callCount,
+      0,
+      'setAllEnvironments should not be called when loading from empty cache',
+    );
+  });
+
+  it('should not load environments from a different version', async () => {
+    const setAllEnvironmentsSpy = sinon.spy(setAllEnvironments);
+    const differentVersion = '2.17.2'; // A different version than ATLASPACK_VERSION
+
+    // Store an environment with a different version
+    await options.cache.set(`Environment/${differentVersion}/${env1.id}`, env1);
+    await options.cache.set(`EnvironmentManager/${differentVersion}`, [
+      env1.id,
+    ]);
+
+    await loadEnvironmentsFromCache(options.cache);
+
+    assert.equal(
+      setAllEnvironmentsSpy.callCount,
+      0,
+      'setAllEnvironments should not be called when loading from different version',
+    );
+    const loadedEnvironments = getAllEnvironments();
+    assert.equal(
+      loadedEnvironments.length,
+      0,
+      'Should not load any environments from different version',
+    );
+  });
+});