@lage-run/cache 0.2.4 → 0.3.0

package/CHANGELOG.json

@@ -2,7 +2,37 @@
   "name": "@lage-run/cache",
   "entries": [
     {
-      "date": "Wed, 15 Feb 2023 16:46:37 GMT",
+      "date": "Tue, 21 Feb 2023 21:30:26 GMT",
+      "tag": "@lage-run/cache_v0.3.0",
+      "version": "0.3.0",
+      "comments": {
+        "minor": [
+          {
+            "author": "kchau@microsoft.com",
+            "package": "@lage-run/cache",
+            "commit": "d7a8a3fa2bcf434c59bea26d5964dd7235998ad2",
+            "comment": "cache directory to be centralized"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Wed, 15 Feb 2023 16:51:15 GMT",
+      "tag": "@lage-run/cache_v0.2.5",
+      "version": "0.2.5",
+      "comments": {
+        "patch": [
+          {
+            "author": "beachball",
+            "package": "@lage-run/cache",
+            "comment": "Bump @lage-run/hasher to v0.1.3",
+            "commit": "b8e0ead727b1e7f7065f853e0a1ae677f4455a5b"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Wed, 15 Feb 2023 16:47:12 GMT",
       "tag": "@lage-run/cache_v0.2.4",
       "version": "0.2.4",
       "comments": {

package/CHANGELOG.md

@@ -1,12 +1,28 @@
 # Change Log - @lage-run/cache
 
-This log was last generated on Wed, 15 Feb 2023 16:46:37 GMT and should not be manually modified.
+This log was last generated on Tue, 21 Feb 2023 21:30:26 GMT and should not be manually modified.
 
 <!-- Start content -->
 
+## 0.3.0
+
+Tue, 21 Feb 2023 21:30:26 GMT
+
+### Minor changes
+
+- cache directory to be centralized (kchau@microsoft.com)
+
+## 0.2.5
+
+Wed, 15 Feb 2023 16:51:15 GMT
+
+### Patches
+
+- Bump @lage-run/hasher to v0.1.3
+
 ## 0.2.4
 
-Wed, 15 Feb 2023 16:46:37 GMT
+Wed, 15 Feb 2023 16:47:12 GMT
 
 ### Patches
 

package/lib/chunkPromise.d.ts

@@ -0,0 +1,3 @@
+type PromiseFn = () => Promise<unknown>;
+export declare function chunkPromise(promises: (Promise<unknown> | PromiseFn)[], limit?: number): Promise<void>;
+export {};

package/lib/chunkPromise.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "chunkPromise", {
+    enumerable: true,
+    get: ()=>chunkPromise
+});
+async function chunkPromise(promises, limit = 5) {
+    for(let i = 0; i < promises.length; i += limit){
+        await Promise.all(promises.slice(i, i + limit).map((p)=>typeof p === "function" ? p() : p));
+    }
+}

package/lib/getCacheDirectory.d.ts

@@ -0,0 +1,3 @@
+export declare function getCacheDirectoryRoot(root: string): string;
+export declare function getCacheDirectory(root: string, hash: string): string;
+export declare function getLogsCacheDirectory(root: string, hash: string): string;

package/lib/getCacheDirectory.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: all[name]
+    });
+}
+_export(exports, {
+    getCacheDirectoryRoot: ()=>getCacheDirectoryRoot,
+    getCacheDirectory: ()=>getCacheDirectory,
+    getLogsCacheDirectory: ()=>getLogsCacheDirectory
+});
+const _path = /*#__PURE__*/ _interopRequireDefault(require("path"));
+function _interopRequireDefault(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
+function getCacheDirectoryRoot(root) {
+    return _path.default.join(root, "node_modules", ".cache", "lage");
+}
+function getCacheDirectory(root, hash) {
+    return _path.default.join(getCacheDirectoryRoot(root), "cache", hash.substring(0, 4));
+}
+function getLogsCacheDirectory(root, hash) {
+    return _path.default.join(getCacheDirectoryRoot(root), "logs", hash.substring(0, 4));
+}

package/lib/index.d.ts

@@ -3,3 +3,4 @@ export { RemoteFallbackCacheProvider } from "./providers/RemoteFallbackCacheProv
 export { TargetHasher } from "./TargetHasher.js";
 export type { CacheOptions } from "./types/CacheOptions.js";
 export type { CacheProvider } from "./types/CacheProvider.js";
+export { getCacheDirectory, getLogsCacheDirectory, getCacheDirectoryRoot } from "./getCacheDirectory.js";

package/lib/index.js

@@ -11,8 +11,12 @@ function _export(target, all) {
 _export(exports, {
     BackfillCacheProvider: ()=>_backfillCacheProviderJs.BackfillCacheProvider,
     RemoteFallbackCacheProvider: ()=>_remoteFallbackCacheProviderJs.RemoteFallbackCacheProvider,
-    TargetHasher: ()=>_targetHasherJs.TargetHasher
+    TargetHasher: ()=>_targetHasherJs.TargetHasher,
+    getCacheDirectory: ()=>_getCacheDirectoryJs.getCacheDirectory,
+    getLogsCacheDirectory: ()=>_getCacheDirectoryJs.getLogsCacheDirectory,
+    getCacheDirectoryRoot: ()=>_getCacheDirectoryJs.getCacheDirectoryRoot
 });
 const _backfillCacheProviderJs = require("./providers/BackfillCacheProvider.js");
 const _remoteFallbackCacheProviderJs = require("./providers/RemoteFallbackCacheProvider.js");
 const _targetHasherJs = require("./TargetHasher.js");
+const _getCacheDirectoryJs = require("./getCacheDirectory.js");

package/lib/providers/BackfillCacheProvider.d.ts

@@ -16,6 +16,7 @@ export declare class BackfillCacheProvider implements CacheProvider {
     constructor(options: BackfillCacheProviderOptions);
     fetch(hash: string, target: Target): Promise<boolean>;
     put(hash: string, target: Target): Promise<void>;
-    clear(): Promise<void>;
-    purge(sinceDays: number): Promise<void>;
+    clear(concurrency?: number): Promise<void>;
+    purge(prunePeriod?: number, concurrency?: number): Promise<void>;
+    getCachePath(packagePath: string, hash: string): string;
 }

package/lib/providers/BackfillCacheProvider.js

@@ -8,10 +8,11 @@ Object.defineProperty(exports, "BackfillCacheProvider", {
 });
 const _backfillWrapperJs = require("../backfillWrapper.js");
 const _backfillCache = require("backfill-cache");
-const _workspaceTools = require("workspace-tools");
 const _util = require("util");
 const _fs = /*#__PURE__*/ _interopRequireWildcard(require("fs"));
 const _path = /*#__PURE__*/ _interopRequireWildcard(require("path"));
+const _getCacheDirectoryJs = require("../getCacheDirectory.js");
+const _chunkPromiseJs = require("../chunkPromise.js");
 function _getRequireWildcardCache(nodeInterop) {
     if (typeof WeakMap !== "function") return null;
     var cacheBabelInterop = new WeakMap();
@@ -51,25 +52,25 @@ function _interopRequireWildcard(obj, nodeInterop) {
     }
     return newObj;
 }
-const rmdir = (0, _util.promisify)(_fs.rmdir);
-const rm = (0, _util.promisify)(_fs.unlink);
+const rm = (0, _util.promisify)(_fs.rm);
 const readdir = (0, _util.promisify)(_fs.readdir);
 const stat = (0, _util.promisify)(_fs.stat);
 const MS_IN_A_DAY = 1000 * 60 * 60 * 24;
 class BackfillCacheProvider {
-    getTargetCacheStorageProvider(cwd) {
+    getTargetCacheStorageProvider(cwd, hash) {
         const { cacheOptions  } = this.options;
-        const { cacheStorageConfig , internalCacheFolder , incrementalCaching  } = (0, _backfillWrapperJs.createBackfillCacheConfig)(cwd, cacheOptions, this.backfillLogger);
+        const { cacheStorageConfig , incrementalCaching  } = (0, _backfillWrapperJs.createBackfillCacheConfig)(cwd, cacheOptions, this.backfillLogger);
+        const cachePath = this.getCachePath(cwd, hash);
         return (0, _backfillCache.getCacheStorageProvider)(cacheStorageConfig ?? {
             provider: "local"
-        }, internalCacheFolder, this.backfillLogger, cwd, incrementalCaching);
+        }, cachePath, this.backfillLogger, cwd, incrementalCaching);
     }
     async fetch(hash, target) {
         const { logger  } = this.options;
         if (!hash) {
             return false;
         }
-        const cacheStorage = this.getTargetCacheStorageProvider(target.cwd);
+        const cacheStorage = this.getTargetCacheStorageProvider(target.cwd, hash);
         try {
             return await cacheStorage.fetch(hash);
         } catch (error) {
@@ -89,7 +90,7 @@ class BackfillCacheProvider {
         if (!hash) {
             return;
         }
-        const cacheStorage = this.getTargetCacheStorageProvider(target.cwd);
+        const cacheStorage = this.getTargetCacheStorageProvider(target.cwd, hash);
         try {
             await cacheStorage.put(hash, target.outputs ?? this.options.cacheOptions.outputGlob ?? [
                 "**/*"
@@ -105,52 +106,50 @@ class BackfillCacheProvider {
         // backfill throws an error if outputGlob doesn't match any files, we will skip this error
         }
     }
-    async clear() {
-        const allPackages = (0, _workspaceTools.getPackageInfos)(this.options.root);
-        for (const info of Object.values(allPackages)){
-            const cachePath = getCachePath(info, this.options.cacheOptions.internalCacheFolder);
-            if (_fs.existsSync(cachePath)) {
-                const entries = await readdir(cachePath);
-                for (const entry of entries){
-                    const entryPath = _path.join(cachePath, entry);
-                    const entryStat = await stat(entryPath);
-                    await removeCache(entryPath, entryStat);
-                }
-            }
-        }
+    async clear(concurrency = 10) {
+        return this.purge(0, concurrency);
     }
-    async purge(sinceDays) {
-        const prunePeriod = sinceDays || 30;
+    async purge(prunePeriod = 30, concurrency = 10) {
         const now = new Date();
-        const allPackages = (0, _workspaceTools.getPackageInfos)(this.options.root);
-        for (const info of Object.values(allPackages)){
-            const cachePath = getCachePath(info, this.options.cacheOptions.internalCacheFolder);
-            if (_fs.existsSync(cachePath)) {
-                const entries = await readdir(cachePath);
-                for (const entry of entries){
-                    const entryPath = _path.join(cachePath, entry);
-                    const entryStat = await stat(entryPath);
-                    if (now.getTime() - entryStat.mtime.getTime() > prunePeriod * MS_IN_A_DAY) {
-                        await removeCache(entryPath, entryStat);
-                    }
+        const cacheTypes = [
+            "cache",
+            "logs"
+        ];
+        const entries = [];
+        for (const cacheType of cacheTypes){
+            const cacheTypeDirectory = _path.join((0, _getCacheDirectoryJs.getCacheDirectoryRoot)(this.options.root), cacheType);
+            if (_fs.existsSync(cacheTypeDirectory)) {
+                const hashPrefixes = await readdir(cacheTypeDirectory);
+                for (const prefix of hashPrefixes){
+                    const cachePath = _path.join(cacheTypeDirectory, prefix);
+                    entries.push(cachePath);
                 }
             }
         }
+        await (0, _chunkPromiseJs.chunkPromise)(entries.map((entry)=>{
+            return async ()=>{
+                const entryPath = entry;
+                const entryStat = await stat(entryPath);
+                if (now.getTime() - entryStat.mtime.getTime() > prunePeriod * MS_IN_A_DAY) {
+                    await removeCache(entryPath, entryStat);
+                }
+            };
+        }), concurrency);
+    }
+    getCachePath(packagePath, hash) {
+        return _path.relative(packagePath, (0, _getCacheDirectoryJs.getCacheDirectory)(this.options.root, hash));
     }
     constructor(options){
         this.options = options;
         this.backfillLogger = (0, _backfillWrapperJs.createBackfillLogger)();
     }
 }
-function getCachePath(info, internalCacheFolder) {
-    return _path.resolve(_path.dirname(info.packageJsonPath), internalCacheFolder ?? "node_modules/.cache/backfill");
-}
 async function removeCache(cachePath, entryStat) {
     if (entryStat.isDirectory()) {
-        rmdir(cachePath, {
+        return rm(cachePath, {
             recursive: true
         });
     } else {
-        rm(cachePath);
+        return rm(cachePath);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lage-run/cache",
-  "version": "0.2.4",
+  "version": "0.3.0",
   "description": "Cache for Lage",
   "repository": {
     "url": "https://github.com/microsoft/lage"
@@ -15,14 +15,13 @@
     "lint": "monorepo-scripts lint"
   },
   "dependencies": {
-    "@lage-run/hasher": "^0.1.2",
+    "@lage-run/hasher": "^0.1.3",
     "@lage-run/target-graph": "^0.6.2",
     "@lage-run/logger": "^1.2.2",
     "backfill-config": "^6.3.0",
     "backfill-cache": "^5.6.1",
     "backfill-logger": "^5.1.3",
-    "fast-glob": "^3.2.11",
-    "workspace-tools": "^0.30.0"
+    "fast-glob": "^3.2.11"
   },
   "devDependencies": {
     "@lage-run/monorepo-fixture": "*",