@lage-run/cache 0.2.5 → 0.4.0

package/CHANGELOG.json

@@ -2,7 +2,49 @@
   "name": "@lage-run/cache",
   "entries": [
     {
-      "date": "Wed, 15 Feb 2023 16:50:52 GMT",
+      "date": "Wed, 08 Mar 2023 00:05:07 GMT",
+      "tag": "@lage-run/cache_v0.4.0",
+      "version": "0.4.0",
+      "comments": {
+        "minor": [
+          {
+            "author": "kchau@microsoft.com",
+            "package": "@lage-run/cache",
+            "commit": "e2eb2c00d1b23e2c65943e9c64134c14e04f985f",
+            "comment": "allows global script cache"
+          },
+          {
+            "author": "beachball",
+            "package": "@lage-run/cache",
+            "comment": "Bump @lage-run/hasher to v0.2.0",
+            "commit": "e2eb2c00d1b23e2c65943e9c64134c14e04f985f"
+          },
+          {
+            "author": "beachball",
+            "package": "@lage-run/cache",
+            "comment": "Bump @lage-run/target-graph to v0.7.0",
+            "commit": "e2eb2c00d1b23e2c65943e9c64134c14e04f985f"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Tue, 21 Feb 2023 21:30:37 GMT",
+      "tag": "@lage-run/cache_v0.3.0",
+      "version": "0.3.0",
+      "comments": {
+        "minor": [
+          {
+            "author": "kchau@microsoft.com",
+            "package": "@lage-run/cache",
+            "commit": "d7a8a3fa2bcf434c59bea26d5964dd7235998ad2",
+            "comment": "cache directory to be centralized"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Wed, 15 Feb 2023 16:51:15 GMT",
       "tag": "@lage-run/cache_v0.2.5",
       "version": "0.2.5",
       "comments": {

package/CHANGELOG.md

@@ -1,12 +1,30 @@
 # Change Log - @lage-run/cache
 
-This log was last generated on Wed, 15 Feb 2023 16:50:52 GMT and should not be manually modified.
+This log was last generated on Wed, 08 Mar 2023 00:05:07 GMT and should not be manually modified.
 
 <!-- Start content -->
 
+## 0.4.0
+
+Wed, 08 Mar 2023 00:05:07 GMT
+
+### Minor changes
+
+- allows global script cache (kchau@microsoft.com)
+- Bump @lage-run/hasher to v0.2.0
+- Bump @lage-run/target-graph to v0.7.0
+
+## 0.3.0
+
+Tue, 21 Feb 2023 21:30:37 GMT
+
+### Minor changes
+
+- cache directory to be centralized (kchau@microsoft.com)
+
 ## 0.2.5
 
-Wed, 15 Feb 2023 16:50:52 GMT
+Wed, 15 Feb 2023 16:51:15 GMT
 
 ### Patches
 

package/lib/TargetHasher.js

@@ -8,11 +8,33 @@ Object.defineProperty(exports, "TargetHasher", {
 });
 const _hasher = require("@lage-run/hasher");
 const _saltJs = require("./salt.js");
+const _globHasher = require("glob-hasher");
+const _hashStringsJs = require("./hashStrings.js");
+function sortObject(unordered) {
+    return Object.keys(unordered).sort((a, b)=>a.localeCompare(b)).reduce((obj, key)=>{
+        obj[key] = unordered[key];
+        return obj;
+    }, {});
+}
 class TargetHasher {
     async hash(target) {
+        const { root  } = this.options;
         const hashKey = await (0, _saltJs.salt)(target.environmentGlob ?? this.options.environmentGlob ?? [
             "lage.config.js"
         ], `${target.id}|${JSON.stringify(this.options.cliArgs)}`, this.options.root, this.options.cacheKey || "");
+        if (target.cwd === root && target.cache) {
+            if (!target.inputs) {
+                throw new Error("Root-level targets must have `inputs` defined if it has cache enabled.");
+            }
+            const hashes = (0, _globHasher.hashGlobGit)(target.inputs, {
+                cwd: root,
+                gitignore: false
+            }) ?? {};
+            const sortedHashMap = sortObject(hashes);
+            const sortedHashes = Object.values(sortedHashMap);
+            sortedHashes.push(hashKey);
+            return (0, _hashStringsJs.hashStrings)(sortedHashes);
+        }
         const hasher = new _hasher.Hasher(target.cwd);
         return hasher.createPackageHash(hashKey);
     }

package/lib/chunkPromise.d.ts

@@ -0,0 +1,3 @@
+type PromiseFn = () => Promise<unknown>;
+export declare function chunkPromise(promises: (Promise<unknown> | PromiseFn)[], limit?: number): Promise<void>;
+export {};

package/lib/chunkPromise.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "chunkPromise", {
+    enumerable: true,
+    get: ()=>chunkPromise
+});
+async function chunkPromise(promises, limit = 5) {
+    for(let i = 0; i < promises.length; i += limit){
+        await Promise.all(promises.slice(i, i + limit).map((p)=>typeof p === "function" ? p() : p));
+    }
+}

package/lib/getCacheDirectory.d.ts

@@ -0,0 +1,3 @@
+export declare function getCacheDirectoryRoot(root: string): string;
+export declare function getCacheDirectory(root: string, hash: string): string;
+export declare function getLogsCacheDirectory(root: string, hash: string): string;

package/lib/getCacheDirectory.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: all[name]
+    });
+}
+_export(exports, {
+    getCacheDirectoryRoot: ()=>getCacheDirectoryRoot,
+    getCacheDirectory: ()=>getCacheDirectory,
+    getLogsCacheDirectory: ()=>getLogsCacheDirectory
+});
+const _path = /*#__PURE__*/ _interopRequireDefault(require("path"));
+function _interopRequireDefault(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
+function getCacheDirectoryRoot(root) {
+    return _path.default.join(root, "node_modules", ".cache", "lage");
+}
+function getCacheDirectory(root, hash) {
+    return _path.default.join(getCacheDirectoryRoot(root), "cache", hash.substring(0, 4));
+}
+function getLogsCacheDirectory(root, hash) {
+    return _path.default.join(getCacheDirectoryRoot(root), "logs", hash.substring(0, 4));
+}

package/lib/hashStrings.d.ts

@@ -0,0 +1 @@
+export declare function hashStrings(strings: string | string[]): string;

package/lib/hashStrings.js

@@ -0,0 +1,26 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "hashStrings", {
+    enumerable: true,
+    get: ()=>hashStrings
+});
+const _crypto = /*#__PURE__*/ _interopRequireDefault(require("crypto"));
+function _interopRequireDefault(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
+function hashStrings(strings) {
+    const hasher = _crypto.default.createHash("sha1");
+    const anArray = typeof strings === "string" ? [
+        strings
+    ] : strings;
+    const elements = [
+        ...anArray
+    ];
+    elements.sort((a, b)=>a.localeCompare(b));
+    elements.forEach((element)=>hasher.update(element));
+    return hasher.digest("hex");
+}

package/lib/index.d.ts

@@ -3,3 +3,4 @@ export { RemoteFallbackCacheProvider } from "./providers/RemoteFallbackCacheProv
 export { TargetHasher } from "./TargetHasher.js";
 export type { CacheOptions } from "./types/CacheOptions.js";
 export type { CacheProvider } from "./types/CacheProvider.js";
+export { getCacheDirectory, getLogsCacheDirectory, getCacheDirectoryRoot } from "./getCacheDirectory.js";

package/lib/index.js

@@ -11,8 +11,12 @@ function _export(target, all) {
 _export(exports, {
     BackfillCacheProvider: ()=>_backfillCacheProviderJs.BackfillCacheProvider,
     RemoteFallbackCacheProvider: ()=>_remoteFallbackCacheProviderJs.RemoteFallbackCacheProvider,
-    TargetHasher: ()=>_targetHasherJs.TargetHasher
+    TargetHasher: ()=>_targetHasherJs.TargetHasher,
+    getCacheDirectory: ()=>_getCacheDirectoryJs.getCacheDirectory,
+    getLogsCacheDirectory: ()=>_getCacheDirectoryJs.getLogsCacheDirectory,
+    getCacheDirectoryRoot: ()=>_getCacheDirectoryJs.getCacheDirectoryRoot
 });
 const _backfillCacheProviderJs = require("./providers/BackfillCacheProvider.js");
 const _remoteFallbackCacheProviderJs = require("./providers/RemoteFallbackCacheProvider.js");
 const _targetHasherJs = require("./TargetHasher.js");
+const _getCacheDirectoryJs = require("./getCacheDirectory.js");

package/lib/providers/BackfillCacheProvider.d.ts

@@ -16,6 +16,7 @@ export declare class BackfillCacheProvider implements CacheProvider {
     constructor(options: BackfillCacheProviderOptions);
     fetch(hash: string, target: Target): Promise<boolean>;
     put(hash: string, target: Target): Promise<void>;
-    clear(): Promise<void>;
-    purge(sinceDays: number): Promise<void>;
+    clear(concurrency?: number): Promise<void>;
+    purge(prunePeriod?: number, concurrency?: number): Promise<void>;
+    getCachePath(packagePath: string, hash: string): string;
 }

package/lib/providers/BackfillCacheProvider.js

@@ -8,10 +8,11 @@ Object.defineProperty(exports, "BackfillCacheProvider", {
 });
 const _backfillWrapperJs = require("../backfillWrapper.js");
 const _backfillCache = require("backfill-cache");
-const _workspaceTools = require("workspace-tools");
 const _util = require("util");
 const _fs = /*#__PURE__*/ _interopRequireWildcard(require("fs"));
 const _path = /*#__PURE__*/ _interopRequireWildcard(require("path"));
+const _getCacheDirectoryJs = require("../getCacheDirectory.js");
+const _chunkPromiseJs = require("../chunkPromise.js");
 function _getRequireWildcardCache(nodeInterop) {
     if (typeof WeakMap !== "function") return null;
     var cacheBabelInterop = new WeakMap();
@@ -51,25 +52,25 @@ function _interopRequireWildcard(obj, nodeInterop) {
     }
     return newObj;
 }
-const rmdir = (0, _util.promisify)(_fs.rmdir);
-const rm = (0, _util.promisify)(_fs.unlink);
+const rm = (0, _util.promisify)(_fs.rm);
 const readdir = (0, _util.promisify)(_fs.readdir);
 const stat = (0, _util.promisify)(_fs.stat);
 const MS_IN_A_DAY = 1000 * 60 * 60 * 24;
 class BackfillCacheProvider {
-    getTargetCacheStorageProvider(cwd) {
+    getTargetCacheStorageProvider(cwd, hash) {
         const { cacheOptions  } = this.options;
-        const { cacheStorageConfig , internalCacheFolder , incrementalCaching  } = (0, _backfillWrapperJs.createBackfillCacheConfig)(cwd, cacheOptions, this.backfillLogger);
+        const { cacheStorageConfig , incrementalCaching  } = (0, _backfillWrapperJs.createBackfillCacheConfig)(cwd, cacheOptions, this.backfillLogger);
+        const cachePath = this.getCachePath(cwd, hash);
         return (0, _backfillCache.getCacheStorageProvider)(cacheStorageConfig ?? {
             provider: "local"
-        }, internalCacheFolder, this.backfillLogger, cwd, incrementalCaching);
+        }, cachePath, this.backfillLogger, cwd, incrementalCaching);
     }
     async fetch(hash, target) {
         const { logger  } = this.options;
         if (!hash) {
             return false;
         }
-        const cacheStorage = this.getTargetCacheStorageProvider(target.cwd);
+        const cacheStorage = this.getTargetCacheStorageProvider(target.cwd, hash);
         try {
             return await cacheStorage.fetch(hash);
         } catch (error) {
@@ -89,7 +90,7 @@ class BackfillCacheProvider {
         if (!hash) {
             return;
         }
-        const cacheStorage = this.getTargetCacheStorageProvider(target.cwd);
+        const cacheStorage = this.getTargetCacheStorageProvider(target.cwd, hash);
         try {
             await cacheStorage.put(hash, target.outputs ?? this.options.cacheOptions.outputGlob ?? [
                 "**/*"
@@ -105,52 +106,50 @@ class BackfillCacheProvider {
         // backfill throws an error if outputGlob doesn't match any files, we will skip this error
         }
     }
-    async clear() {
-        const allPackages = (0, _workspaceTools.getPackageInfos)(this.options.root);
-        for (const info of Object.values(allPackages)){
-            const cachePath = getCachePath(info, this.options.cacheOptions.internalCacheFolder);
-            if (_fs.existsSync(cachePath)) {
-                const entries = await readdir(cachePath);
-                for (const entry of entries){
-                    const entryPath = _path.join(cachePath, entry);
-                    const entryStat = await stat(entryPath);
-                    await removeCache(entryPath, entryStat);
-                }
-            }
-        }
+    async clear(concurrency = 10) {
+        return this.purge(0, concurrency);
     }
-    async purge(sinceDays) {
-        const prunePeriod = sinceDays || 30;
+    async purge(prunePeriod = 30, concurrency = 10) {
         const now = new Date();
-        const allPackages = (0, _workspaceTools.getPackageInfos)(this.options.root);
-        for (const info of Object.values(allPackages)){
-            const cachePath = getCachePath(info, this.options.cacheOptions.internalCacheFolder);
-            if (_fs.existsSync(cachePath)) {
-                const entries = await readdir(cachePath);
-                for (const entry of entries){
-                    const entryPath = _path.join(cachePath, entry);
-                    const entryStat = await stat(entryPath);
-                    if (now.getTime() - entryStat.mtime.getTime() > prunePeriod * MS_IN_A_DAY) {
-                        await removeCache(entryPath, entryStat);
-                    }
+        const cacheTypes = [
+            "cache",
+            "logs"
+        ];
+        const entries = [];
+        for (const cacheType of cacheTypes){
+            const cacheTypeDirectory = _path.join((0, _getCacheDirectoryJs.getCacheDirectoryRoot)(this.options.root), cacheType);
+            if (_fs.existsSync(cacheTypeDirectory)) {
+                const hashPrefixes = await readdir(cacheTypeDirectory);
+                for (const prefix of hashPrefixes){
+                    const cachePath = _path.join(cacheTypeDirectory, prefix);
+                    entries.push(cachePath);
                 }
             }
         }
+        await (0, _chunkPromiseJs.chunkPromise)(entries.map((entry)=>{
+            return async ()=>{
+                const entryPath = entry;
+                const entryStat = await stat(entryPath);
+                if (now.getTime() - entryStat.mtime.getTime() > prunePeriod * MS_IN_A_DAY) {
+                    await removeCache(entryPath, entryStat);
+                }
+            };
+        }), concurrency);
+    }
+    getCachePath(packagePath, hash) {
+        return _path.relative(packagePath, (0, _getCacheDirectoryJs.getCacheDirectory)(this.options.root, hash));
     }
     constructor(options){
         this.options = options;
         this.backfillLogger = (0, _backfillWrapperJs.createBackfillLogger)();
     }
 }
-function getCachePath(info, internalCacheFolder) {
-    return _path.resolve(_path.dirname(info.packageJsonPath), internalCacheFolder ?? "node_modules/.cache/backfill");
-}
 async function removeCache(cachePath, entryStat) {
     if (entryStat.isDirectory()) {
-        rmdir(cachePath, {
+        return rm(cachePath, {
             recursive: true
         });
     } else {
-        rm(cachePath);
+        return rm(cachePath);
     }
 }

package/lib/salt.js

@@ -12,49 +12,8 @@ _export(exports, {
     "_testResetEnvHash": ()=>_testResetEnvHash,
     salt: ()=>salt
 });
-const _path = /*#__PURE__*/ _interopRequireWildcard(require("path"));
-const _crypto = /*#__PURE__*/ _interopRequireWildcard(require("crypto"));
-const _fastGlob = /*#__PURE__*/ _interopRequireWildcard(require("fast-glob"));
-const _promises = /*#__PURE__*/ _interopRequireWildcard(require("fs/promises"));
-function _getRequireWildcardCache(nodeInterop) {
-    if (typeof WeakMap !== "function") return null;
-    var cacheBabelInterop = new WeakMap();
-    var cacheNodeInterop = new WeakMap();
-    return (_getRequireWildcardCache = function(nodeInterop) {
-        return nodeInterop ? cacheNodeInterop : cacheBabelInterop;
-    })(nodeInterop);
-}
-function _interopRequireWildcard(obj, nodeInterop) {
-    if (!nodeInterop && obj && obj.__esModule) {
-        return obj;
-    }
-    if (obj === null || typeof obj !== "object" && typeof obj !== "function") {
-        return {
-            default: obj
-        };
-    }
-    var cache = _getRequireWildcardCache(nodeInterop);
-    if (cache && cache.has(obj)) {
-        return cache.get(obj);
-    }
-    var newObj = {};
-    var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor;
-    for(var key in obj){
-        if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) {
-            var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null;
-            if (desc && (desc.get || desc.set)) {
-                Object.defineProperty(newObj, key, desc);
-            } else {
-                newObj[key] = obj[key];
-            }
-        }
-    }
-    newObj.default = obj;
-    if (cache) {
-        cache.set(obj, newObj);
-    }
-    return newObj;
-}
+const _globHasher = require("glob-hasher");
+const _hashStringsJs = require("./hashStrings.js");
 let envHashes = {};
 // A promise to guarantee the getEnvHashes is done one at a time
 let oneAtATime = Promise.resolve();
@@ -63,7 +22,7 @@ function _testResetEnvHash() {
 }
 async function salt(environmentGlobFiles, command, repoRoot, customKey = "") {
     const envHash = await getEnvHash(environmentGlobFiles, repoRoot);
-    return hashStrings([
+    return (0, _hashStringsJs.hashStrings)([
         ...envHash,
         command,
         customKey
@@ -72,6 +31,12 @@ async function salt(environmentGlobFiles, command, repoRoot, customKey = "") {
 function envHashKey(environmentGlobFiles) {
     return environmentGlobFiles.sort().join("|");
 }
+function sortObject(unordered) {
+    return Object.keys(unordered).sort((a, b)=>a.localeCompare(b)).reduce((obj, key)=>{
+        obj[key] = unordered[key];
+        return obj;
+    }, {});
+}
 async function getEnvHash(environmentGlobFiles, repoRoot) {
     const key = envHashKey(environmentGlobFiles);
     // We want to make sure that we only call getEnvHashOneAtTime one at a time
@@ -85,35 +50,17 @@ async function getEnvHash(environmentGlobFiles, repoRoot) {
     });
     return oneAtATime;
 }
-async function getEnvHashOneAtTime(environmentGlobFiles, repoRoot) {
-    const envHash = [];
-    const newline = /\r\n|\r|\n/g;
-    const LF = "\n";
-    const files = _fastGlob.sync(environmentGlobFiles, {
-        cwd: repoRoot
-    });
-    files.sort((a, b)=>a.localeCompare(b));
-    for (const file of files){
-        const hasher = _crypto.createHash("sha1");
-        hasher.update(file);
-        const fileBuffer = await _promises.readFile(_path.join(repoRoot, file), "utf-8");
-        const data = fileBuffer.replace(newline, LF);
-        hasher.update(data);
-        envHash.push(hasher.digest("hex"));
-    }
+function getEnvHashOneAtTime(environmentGlobFiles, repoRoot) {
     const key = envHashKey(environmentGlobFiles);
-    envHashes[key] = envHash;
-    return envHash;
-}
-function hashStrings(strings) {
-    const hasher = _crypto.createHash("sha1");
-    const anArray = typeof strings === "string" ? [
-        strings
-    ] : strings;
-    const elements = [
-        ...anArray
-    ];
-    elements.sort((a, b)=>a.localeCompare(b));
-    elements.forEach((element)=>hasher.update(element));
-    return hasher.digest("hex");
+    if (environmentGlobFiles.length === 0) {
+        envHashes[key] = [];
+        return envHashes[key];
+    }
+    const hashes = (0, _globHasher.hashGlobGit)(environmentGlobFiles, {
+        cwd: repoRoot,
+        gitignore: false
+    });
+    const sortedHashes = sortObject(hashes);
+    envHashes[key] = Object.values(sortedHashes);
+    return envHashes[key];
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lage-run/cache",
-  "version": "0.2.5",
+  "version": "0.4.0",
   "description": "Cache for Lage",
   "repository": {
     "url": "https://github.com/microsoft/lage"
@@ -15,20 +15,17 @@
     "lint": "monorepo-scripts lint"
   },
   "dependencies": {
-    "@lage-run/hasher": "^0.1.3",
-    "@lage-run/target-graph": "^0.6.2",
+    "@lage-run/hasher": "^0.2.0",
+    "@lage-run/target-graph": "^0.7.0",
     "@lage-run/logger": "^1.2.2",
     "backfill-config": "^6.3.0",
     "backfill-cache": "^5.6.1",
     "backfill-logger": "^5.1.3",
-    "fast-glob": "^3.2.11",
-    "workspace-tools": "^0.30.0"
+    "glob-hasher": "1.1.1"
   },
   "devDependencies": {
     "@lage-run/monorepo-fixture": "*",
-    "@types/mock-fs": "4.13.1",
-    "monorepo-scripts": "*",
-    "mock-fs": "5.2.0"
+    "monorepo-scripts": "*"
   },
   "publishConfig": {
     "access": "public"