@atlaspack/core 2.16.1-canary.2 → 2.16.1-canary.4

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @atlaspack/core
 
+## 2.16.1
+
+### Patch Changes
+
+- [#525](https://github.com/atlassian-labs/atlaspack/pull/525) [`cb9da16`](https://github.com/atlassian-labs/atlaspack/commit/cb9da16fb2648e7f53c64df0313f60d5fb8970cc) Thanks [@yamadapc](https://github.com/yamadapc)! - Fix issues with large blob cache writes, run cache writes in a write transaction
+
+- Updated dependencies [[`1a2c14c`](https://github.com/atlassian-labs/atlaspack/commit/1a2c14c3cd4587551cc12e94d0680c8b71ea12bf), [`cb9da16`](https://github.com/atlassian-labs/atlaspack/commit/cb9da16fb2648e7f53c64df0313f60d5fb8970cc)]:
+  - @atlaspack/rust@3.2.0
+  - @atlaspack/cache@3.1.0
+  - @atlaspack/fs@2.14.4
+  - @atlaspack/logger@2.14.4
+  - @atlaspack/utils@2.14.4
+  - @atlaspack/package-manager@2.14.4
+  - @atlaspack/workers@2.14.4
+  - @atlaspack/types@2.14.4
+  - @atlaspack/plugin@2.14.4
+
 ## 2.16.0
 
 ### Minor Changes

package/lib/RequestTracker.js

@@ -30,6 +30,13 @@ function _buildCache() {
   };
   return data;
 }
+function _cache() {
+  const data = require("@atlaspack/cache");
+  _cache = function () {
+    return data;
+  };
+  return data;
+}
 function _featureFlags() {
   const data = require("@atlaspack/feature-flags");
   _featureFlags = function () {
@@ -45,7 +52,7 @@ function _graph() {
   return data;
 }
 function _logger() {
-  const data = _interopRequireDefault(require("@atlaspack/logger"));
+  const data = _interopRequireWildcard(require("@atlaspack/logger"));
   _logger = function () {
     return data;
   };
@@ -926,6 +933,20 @@ class RequestTracker {
     };
   }
   async writeToCache(signal) {
+    const options = this.options;
+    async function runCacheImprovements(newPath, oldPath) {
+      if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
+        (0, _assert().default)(options.cache instanceof _cache().LMDBLiteCache);
+        const result = await newPath(options.cache);
+        return result;
+      } else {
+        const result = await oldPath();
+        return result;
+      }
+    }
+    await runCacheImprovements(async cache => {
+      await cache.getNativeRef().startWriteTransaction();
+    }, () => Promise.resolve());
     let cacheKey = getCacheKey(this.options);
     let requestGraphKey = `requestGraph-${cacheKey}`;
     if (this.options.shouldDisableCache) {
@@ -939,21 +960,23 @@ class RequestTracker {
       size: this.graph.nodes.length
     });
     let serialisedGraph = this.graph.serialize();
-    if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
-      // Delete an existing request graph cache, to prevent invalid states
-      await this.options.cache.deleteLargeBlob(requestGraphKey);
-    }
+
+    // Delete an existing request graph cache, to prevent invalid states
+    await this.options.cache.deleteLargeBlob(requestGraphKey);
     const serialiseAndSet = async (key, contents) => {
       if (signal !== null && signal !== void 0 && signal.aborted) {
         throw new Error('Serialization was aborted');
       }
-      if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
-        await this.options.cache.set(key, (0, _buildCache().serialize)(contents));
-      } else {
+      await runCacheImprovements(cache => {
+        (0, _logger().instrument)(`cache.put(${key})`, () => {
+          cache.getNativeRef().putNoConfirm(key, (0, _buildCache().serialize)(contents));
+        });
+        return Promise.resolve();
+      }, async () => {
         await this.options.cache.setLargeBlob(key, (0, _buildCache().serialize)(contents), signal ? {
           signal: signal
         } : undefined);
-      }
+      });
       total += 1;
       (0, _ReporterRunner.report)({
         type: 'cache',
@@ -1014,6 +1037,9 @@ class RequestTracker {
       // If we have aborted, ignore the error and continue
       if (!(signal !== null && signal !== void 0 && signal.aborted)) throw err;
     }
+    await runCacheImprovements(async cache => {
+      await cache.getNativeRef().commitWriteTransaction();
+    }, () => Promise.resolve());
     (0, _ReporterRunner.report)({
       type: 'cache',
       phase: 'end',
@@ -1057,13 +1083,6 @@ function getRequestGraphNodeKey(index, cacheKey) {
 }
 async function readAndDeserializeRequestGraph(cache, requestGraphKey, cacheKey) {
   let bufferLength = 0;
-  if ((0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements')) {
-    let data = (0, _nullthrows().default)(await cache.get(requestGraphKey));
-    return {
-      requestGraph: RequestGraph.deserialize(data),
-      bufferLength
-    };
-  }
   const getAndDeserialize = async key => {
     let buffer = await cache.getLargeBlob(key);
     bufferLength += Buffer.byteLength(buffer);
@@ -1101,7 +1120,7 @@ async function loadRequestGraph(options) {
       snapshotKey
     }
   });
-  if (!(0, _featureFlags().getFeatureFlag)('cachePerformanceImprovements') && (await options.cache.hasLargeBlob(requestGraphKey))) {
+  if (await options.cache.hasLargeBlob(requestGraphKey)) {
     try {
       let {
         requestGraph

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atlaspack/core",
-  "version": "2.16.1-canary.2+1a2c14c3c",
+  "version": "2.16.1-canary.4+9dee30885",
   "license": "(MIT OR Apache-2.0)",
   "publishConfig": {
     "access": "public"
@@ -20,21 +20,21 @@
     "check-ts": "tsc --noEmit index.d.ts"
   },
   "dependencies": {
-    "@atlaspack/build-cache": "2.13.3-canary.65+1a2c14c3c",
-    "@atlaspack/cache": "3.0.2-canary.2+1a2c14c3c",
-    "@atlaspack/diagnostic": "2.14.1-canary.65+1a2c14c3c",
-    "@atlaspack/events": "2.14.1-canary.65+1a2c14c3c",
-    "@atlaspack/feature-flags": "2.14.1-canary.65+1a2c14c3c",
-    "@atlaspack/fs": "2.14.4-canary.2+1a2c14c3c",
-    "@atlaspack/graph": "3.4.1-canary.65+1a2c14c3c",
-    "@atlaspack/logger": "2.14.4-canary.2+1a2c14c3c",
-    "@atlaspack/package-manager": "2.14.4-canary.2+1a2c14c3c",
-    "@atlaspack/plugin": "2.14.4-canary.2+1a2c14c3c",
-    "@atlaspack/profiler": "2.14.1-canary.65+1a2c14c3c",
-    "@atlaspack/rust": "3.1.2-canary.2+1a2c14c3c",
-    "@atlaspack/types": "2.14.4-canary.2+1a2c14c3c",
-    "@atlaspack/utils": "2.14.4-canary.2+1a2c14c3c",
-    "@atlaspack/workers": "2.14.4-canary.2+1a2c14c3c",
+    "@atlaspack/build-cache": "2.13.3-canary.67+9dee30885",
+    "@atlaspack/cache": "3.0.2-canary.4+9dee30885",
+    "@atlaspack/diagnostic": "2.14.1-canary.67+9dee30885",
+    "@atlaspack/events": "2.14.1-canary.67+9dee30885",
+    "@atlaspack/feature-flags": "2.14.1-canary.67+9dee30885",
+    "@atlaspack/fs": "2.14.4-canary.4+9dee30885",
+    "@atlaspack/graph": "3.4.1-canary.67+9dee30885",
+    "@atlaspack/logger": "2.14.4-canary.4+9dee30885",
+    "@atlaspack/package-manager": "2.14.4-canary.4+9dee30885",
+    "@atlaspack/plugin": "2.14.4-canary.4+9dee30885",
+    "@atlaspack/profiler": "2.14.1-canary.67+9dee30885",
+    "@atlaspack/rust": "3.1.2-canary.4+9dee30885",
+    "@atlaspack/types": "2.14.4-canary.4+9dee30885",
+    "@atlaspack/utils": "2.14.4-canary.4+9dee30885",
+    "@atlaspack/workers": "2.14.4-canary.4+9dee30885",
     "@mischnic/json-sourcemap": "^0.1.0",
     "@parcel/source-map": "^2.1.1",
     "base-x": "^3.0.8",
@@ -67,5 +67,5 @@
     "./src/serializerCore.js": "./src/serializerCore.browser.js"
   },
   "type": "commonjs",
-  "gitHead": "1a2c14c3cd4587551cc12e94d0680c8b71ea12bf"
+  "gitHead": "9dee30885fc8c0f3654d54151d75f1e89907dafd"
 }

package/src/RequestTracker.js

@@ -4,7 +4,7 @@ import invariant, {AssertionError} from 'assert';
 import path from 'path';
 
 import {deserialize, serialize} from '@atlaspack/build-cache';
-import type {Cache} from '@atlaspack/cache';
+import {LMDBLiteCache, type Cache} from '@atlaspack/cache';
 import {getFeatureFlag} from '@atlaspack/feature-flags';
 import {ContentGraph} from '@atlaspack/graph';
 import type {
@@ -14,7 +14,7 @@ import type {
   SerializedContentGraph,
   Graph,
 } from '@atlaspack/graph';
-import logger from '@atlaspack/logger';
+import logger, {instrument} from '@atlaspack/logger';
 import {hashString} from '@atlaspack/rust';
 import type {Async, EnvMap} from '@atlaspack/types';
 import {
@@ -1457,6 +1457,28 @@ export default class RequestTracker {
   }
 
   async writeToCache(signal?: AbortSignal) {
+    const options = this.options;
+    async function runCacheImprovements<T>(
+      newPath: (cache: LMDBLiteCache) => Promise<T>,
+      oldPath: () => Promise<T>,
+    ): Promise<T> {
+      if (getFeatureFlag('cachePerformanceImprovements')) {
+        invariant(options.cache instanceof LMDBLiteCache);
+        const result = await newPath(options.cache);
+        return result;
+      } else {
+        const result = await oldPath();
+        return result;
+      }
+    }
+
+    await runCacheImprovements(
+      async (cache) => {
+        await cache.getNativeRef().startWriteTransaction();
+      },
+      () => Promise.resolve(),
+    );
+
     let cacheKey = getCacheKey(this.options);
     let requestGraphKey = `requestGraph-${cacheKey}`;
     let snapshotKey = `snapshot-${cacheKey}`;
@@ -1475,10 +1497,8 @@ export default class RequestTracker {
 
     let serialisedGraph = this.graph.serialize();
 
-    if (!getFeatureFlag('cachePerformanceImprovements')) {
-      // Delete an existing request graph cache, to prevent invalid states
-      await this.options.cache.deleteLargeBlob(requestGraphKey);
-    }
+    // Delete an existing request graph cache, to prevent invalid states
+    await this.options.cache.deleteLargeBlob(requestGraphKey);
 
     const serialiseAndSet = async (
       key: string,
@@ -1489,19 +1509,25 @@ export default class RequestTracker {
         throw new Error('Serialization was aborted');
       }
 
-      if (getFeatureFlag('cachePerformanceImprovements')) {
-        await this.options.cache.set(key, serialize(contents));
-      } else {
-        await this.options.cache.setLargeBlob(
-          key,
-          serialize(contents),
-          signal
-            ? {
-                signal: signal,
-              }
-            : undefined,
-        );
-      }
+      await runCacheImprovements(
+        (cache) => {
+          instrument(`cache.put(${key})`, () => {
+            cache.getNativeRef().putNoConfirm(key, serialize(contents));
+          });
+          return Promise.resolve();
+        },
+        async () => {
+          await this.options.cache.setLargeBlob(
+            key,
+            serialize(contents),
+            signal
+              ? {
+                  signal: signal,
+                }
+              : undefined,
+          );
+        },
+      );
 
       total += 1;
 
@@ -1592,6 +1618,13 @@ export default class RequestTracker {
       if (!signal?.aborted) throw err;
     }
 
+    await runCacheImprovements(
+      async (cache) => {
+        await cache.getNativeRef().commitWriteTransaction();
+      },
+      () => Promise.resolve(),
+    );
+
     report({type: 'cache', phase: 'end', total, size: this.graph.nodes.length});
   }
 
@@ -1641,14 +1674,6 @@ export async function readAndDeserializeRequestGraph(
 ): Async<{|requestGraph: RequestGraph, bufferLength: number|}> {
   let bufferLength = 0;
 
-  if (getFeatureFlag('cachePerformanceImprovements')) {
-    let data = nullthrows(await cache.get(requestGraphKey));
-    return {
-      requestGraph: RequestGraph.deserialize(data),
-      bufferLength,
-    };
-  }
-
   const getAndDeserialize = async (key: string) => {
     let buffer = await cache.getLargeBlob(key);
     bufferLength += Buffer.byteLength(buffer);
@@ -1699,10 +1724,7 @@ async function loadRequestGraph(options): Async<RequestGraph> {
     },
   });
 
-  if (
-    !getFeatureFlag('cachePerformanceImprovements') &&
-    (await options.cache.hasLargeBlob(requestGraphKey))
-  ) {
+  if (await options.cache.hasLargeBlob(requestGraphKey)) {
     try {
       let {requestGraph} = await readAndDeserializeRequestGraph(
         options.cache,