@lage-run/hasher 1.3.4 → 1.5.0

package/CHANGELOG.json

@@ -2,7 +2,43 @@
   "name": "@lage-run/hasher",
   "entries": [
     {
-      "date": "Fri, 13 Sep 2024 18:04:33 GMT",
+      "date": "Fri, 27 Sep 2024 20:03:27 GMT",
+      "version": "1.5.0",
+      "tag": "@lage-run/hasher_v1.5.0",
+      "comments": {
+        "minor": [
+          {
+            "author": "kchau@microsoft.com",
+            "package": "@lage-run/hasher",
+            "commit": "2919f9041f931dc6ef65017f7aedb9fef9dab66d",
+            "comment": "exposes getInputFiles, packageTrees"
+          },
+          {
+            "author": "beachball",
+            "package": "@lage-run/hasher",
+            "comment": "Bump @lage-run/globby to v14.1.0",
+            "commit": "not available"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Wed, 25 Sep 2024 20:28:10 GMT",
+      "version": "1.4.0",
+      "tag": "@lage-run/hasher_v1.4.0",
+      "comments": {
+        "minor": [
+          {
+            "author": "kchau@microsoft.com",
+            "package": "@lage-run/hasher",
+            "commit": "09313be6a4a5667bf8c50b654428f778f6fa4028",
+            "comment": "writes out the \"inputs\" as hashes files inside node_modules\\.cache\\lage\\hashes\\**"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Fri, 13 Sep 2024 18:05:04 GMT",
       "version": "1.3.4",
       "tag": "@lage-run/hasher_v1.3.4",
       "comments": {

package/CHANGELOG.md

@@ -1,12 +1,29 @@
 # Change Log - @lage-run/hasher
 
-<!-- This log was last generated on Fri, 13 Sep 2024 18:04:33 GMT and should not be manually modified. -->
+<!-- This log was last generated on Fri, 27 Sep 2024 20:03:27 GMT and should not be manually modified. -->
 
 <!-- Start content -->
 
+## 1.5.0
+
+Fri, 27 Sep 2024 20:03:27 GMT
+
+### Minor changes
+
+- exposes getInputFiles, packageTrees (kchau@microsoft.com)
+- Bump @lage-run/globby to v14.1.0
+
+## 1.4.0
+
+Wed, 25 Sep 2024 20:28:10 GMT
+
+### Minor changes
+
+- writes out the "inputs" as hashes files inside node_modules\.cache\lage\hashes\** (kchau@microsoft.com)
+
 ## 1.3.4
 
-Fri, 13 Sep 2024 18:04:33 GMT
+Fri, 13 Sep 2024 18:05:04 GMT
 
 ### Patches
 

package/lib/TargetHasher.d.ts

@@ -30,6 +30,11 @@ export interface TargetManifest {
  */
 export declare class TargetHasher {
     private options;
+    targetHashesLog: Record<string, {
+        fileHashes: Record<string, string>;
+        globalFileHashes: Record<string, string>;
+    }>;
+    targetHashesDirectory: string;
     logger: Logger | undefined;
     fileHasher: FileHasher;
     packageTree: PackageTree | undefined;
@@ -40,13 +45,12 @@ export declare class TargetHasher {
     lockInfo: ParsedLock | undefined;
     targetHashes: Record<string, string>;
     dependencyMap: DependencyMap;
-    memoizedEnvGlobResults: Map<string, string[]>;
-    getMemorizedEnvGlobResults(envGlob: string[]): Promise<string[]>;
     getPackageInfos(workspacePackages: WorkspaceInfo): PackageInfos;
-    expandInputPatterns(patterns: string[], target: Target): Record<string, string[]>;
     constructor(options: TargetHasherOptions);
     ensureInitialized(): void;
     initialize(): Promise<void>;
     hash(target: Target): Promise<string>;
+    writeTargetHashesManifest(): void;
+    getEnvironmentGlobHashes(root: string, target: Target): Promise<Record<string, string>>;
     cleanup(): Promise<void>;
 }

package/lib/TargetHasher.js

@@ -9,7 +9,7 @@ Object.defineProperty(exports, "TargetHasher", {
     }
 });
 const _globhasher = require("glob-hasher");
-const _fastglob = /*#__PURE__*/ _interop_require_default(require("fast-glob"));
+const _globby = require("@lage-run/globby");
 const _fs = /*#__PURE__*/ _interop_require_default(require("fs"));
 const _path = /*#__PURE__*/ _interop_require_default(require("path"));
 const _workspacetools = require("workspace-tools");
@@ -19,6 +19,7 @@ const _resolveInternalDependencies = require("./resolveInternalDependencies.js")
 const _resolveExternalDependencies = require("./resolveExternalDependencies.js");
 const _FileHasher = require("./FileHasher.js");
 const _PackageTree = require("./PackageTree.js");
+const _getInputFiles = require("./getInputFiles.js");
 function _define_property(obj, key, value) {
     if (key in obj) {
         Object.defineProperty(obj, key, {
@@ -38,17 +39,6 @@ function _interop_require_default(obj) {
     };
 }
 class TargetHasher {
-    async getMemorizedEnvGlobResults(envGlob) {
-        const key = envGlob.join("\0ENV_GLOB\0");
-        const { root  } = this.options;
-        if (!this.memoizedEnvGlobResults.has(key)) {
-            const files = await (0, _fastglob.default)(envGlob, {
-                cwd: root
-            });
-            this.memoizedEnvGlobResults.set(key, files);
-        }
-        return this.memoizedEnvGlobResults.get(key) ?? [];
-    }
     getPackageInfos(workspacePackages) {
         const { root  } = this.options;
         const packageInfos = {};
@@ -72,43 +62,6 @@ class TargetHasher {
         }
         return packageInfos;
     }
-    expandInputPatterns(patterns, target) {
-        const expandedPatterns = {};
-        for (const pattern of patterns){
-            if (pattern.startsWith("^") || pattern.startsWith("!^")) {
-                const matchPattern = pattern.replace("^", "");
-                // get all the packages that are transitive deps and add them to the list
-                const queue = [
-                    target.packageName
-                ];
-                const visited = new Set();
-                while(queue.length > 0){
-                    const pkg = queue.pop();
-                    if (visited.has(pkg)) {
-                        continue;
-                    }
-                    visited.add(pkg);
-                    if (pkg !== target.packageName) {
-                        expandedPatterns[pkg] = expandedPatterns[pkg] ?? [];
-                        expandedPatterns[pkg].push(matchPattern);
-                    }
-                    if (this.dependencyMap.dependencies.has(pkg)) {
-                        const deps = this.dependencyMap.dependencies.get(pkg);
-                        if (deps) {
-                            for (const dep of deps){
-                                queue.push(dep);
-                            }
-                        }
-                    }
-                }
-            } else {
-                const pkg = target.packageName;
-                expandedPatterns[pkg] = expandedPatterns[pkg] ?? [];
-                expandedPatterns[pkg].push(pattern);
-            }
-        }
-        return expandedPatterns;
-    }
     ensureInitialized() {
         if (!this.initializedPromise) {
             throw new Error("TargetHasher is not initialized");
@@ -121,7 +74,9 @@ class TargetHasher {
             return;
         }
         this.initializedPromise = Promise.all([
-            this.fileHasher.readManifest().then(()=>this.getMemorizedEnvGlobResults(environmentGlob)).then((files)=>this.fileHasher.hash(files)).then((hash)=>this.globalInputsHash = hash),
+            this.fileHasher.readManifest().then(()=>(0, _globby.globAsync)(environmentGlob, {
+                    cwd: root
+                })).then((files)=>this.fileHasher.hash(files)).then((hash)=>this.globalInputsHash = hash),
             (0, _workspacetools.getWorkspacesAsync)(root).then((workspaceInfo)=>this.workspaceInfo = workspaceInfo).then(()=>{
                 this.packageInfos = this.getPackageInfos(this.workspaceInfo);
                 this.dependencyMap = (0, _workspacetools.createDependencyMap)(this.packageInfos, {
@@ -142,9 +97,6 @@ class TargetHasher {
         if (this.logger !== undefined) {
             const globalInputsHash = (0, _hashStrings.hashStrings)(Object.values(this.globalInputsHash ?? {}));
             this.logger.verbose(`Global inputs hash: ${globalInputsHash}`);
-            // Log global input hashs to log file
-            const globalInputsHashJson = JSON.stringify(this.globalInputsHash, null, 2);
-            this.logger.silly(globalInputsHashJson);
         }
     }
     async hash(target) {
@@ -154,7 +106,7 @@ class TargetHasher {
             if (!target.inputs) {
                 throw new Error("Root-level targets must have `inputs` defined if it has cache enabled.");
             }
-            const files = await (0, _fastglob.default)(target.inputs, {
+            const files = await (0, _globby.globAsync)(target.inputs, {
                 cwd: root
             });
             const fileFashes = (0, _globhasher.hash)(files, {
@@ -178,22 +130,14 @@ class TargetHasher {
             ...internalDeps,
             ...externalDeps
         ].sort();
-        const inputs = target.inputs ?? [
-            "**/*"
-        ];
-        const packagePatterns = this.expandInputPatterns(inputs, target);
-        const files = [];
-        for (const [pkg, patterns] of Object.entries(packagePatterns)){
-            const packageFiles = this.packageTree.getPackageFiles(pkg, patterns);
-            files.push(...packageFiles);
-        }
+        const files = (0, _getInputFiles.getInputFiles)(target, this.dependencyMap, this.packageTree);
         const fileHashes = this.fileHasher.hash(files) ?? {}; // this list is sorted by file name
         // get target hashes
         const targetDepHashes = target.dependencies?.sort().map((targetDep)=>this.targetHashes[targetDep]);
-        const envGlobFiles = Object.values(target.environmentGlob ? this.fileHasher.hash(await this.getMemorizedEnvGlobResults(target.environmentGlob)) : this.globalInputsHash ?? {});
+        const globalFileHashes = await this.getEnvironmentGlobHashes(root, target);
         const combinedHashes = [
             // Environmental hashes
-            ...envGlobFiles,
+            ...Object.values(globalFileHashes),
             `${target.id}|${JSON.stringify(this.options.cliArgs)}`,
             this.options.cacheKey || "",
             // File content hashes based on target.inputs
@@ -204,13 +148,40 @@ class TargetHasher {
         ].filter(Boolean);
         const hashString = (0, _hashStrings.hashStrings)(combinedHashes);
         this.targetHashes[target.id] = hashString;
+        this.targetHashesLog[target.id] = {
+            fileHashes,
+            globalFileHashes
+        };
         return hashString;
     }
+    writeTargetHashesManifest() {
+        for (const [id, { fileHashes , globalFileHashes  }] of Object.entries(this.targetHashesLog)){
+            const targetHashesManifestPath = _path.default.join(this.targetHashesDirectory, `${id}.json`);
+            if (!_fs.default.existsSync(_path.default.dirname(targetHashesManifestPath))) {
+                _fs.default.mkdirSync(_path.default.dirname(targetHashesManifestPath), {
+                    recursive: true
+                });
+            }
+            _fs.default.writeFileSync(targetHashesManifestPath, JSON.stringify({
+                fileHashes,
+                globalFileHashes
+            }), "utf-8");
+        }
+    }
+    async getEnvironmentGlobHashes(root, target) {
+        const globalFileHashes = target.environmentGlob ? this.fileHasher.hash(await (0, _globby.globAsync)(target.environmentGlob ?? [], {
+            cwd: root
+        })) : this.globalInputsHash ?? {};
+        return globalFileHashes;
+    }
     async cleanup() {
+        this.writeTargetHashesManifest();
         await this.fileHasher.writeManifest();
     }
     constructor(options){
         _define_property(this, "options", void 0);
+        _define_property(this, "targetHashesLog", void 0);
+        _define_property(this, "targetHashesDirectory", void 0);
         _define_property(this, "logger", void 0);
         _define_property(this, "fileHasher", void 0);
         _define_property(this, "packageTree", void 0);
@@ -221,19 +192,24 @@ class TargetHasher {
         _define_property(this, "lockInfo", void 0);
         _define_property(this, "targetHashes", void 0);
         _define_property(this, "dependencyMap", void 0);
-        _define_property(this, "memoizedEnvGlobResults", void 0);
         this.options = options;
+        this.targetHashesLog = {};
         this.packageInfos = {};
         this.targetHashes = {};
         this.dependencyMap = {
             dependencies: new Map(),
             dependents: new Map()
         };
-        this.memoizedEnvGlobResults = new Map();
         const { root , logger  } = options;
         this.logger = logger;
         this.fileHasher = new _FileHasher.FileHasher({
             root
         });
+        this.targetHashesDirectory = _path.default.join(root, "node_modules", ".cache", "lage", "hashes");
+        if (!_fs.default.existsSync(this.targetHashesDirectory)) {
+            _fs.default.mkdirSync(this.targetHashesDirectory, {
+                recursive: true
+            });
+        }
     }
 }

package/lib/__tests__/TargetHasher.test.js

@@ -36,6 +36,25 @@ describe("The main Hasher class", ()=>{
             task
         };
     }
+    it("creates different hashes given different global environment glob", async ()=>{
+        const monorepo1 = await setupFixture("monorepo-with-global-files");
+        const hasher = new _index.TargetHasher({
+            root: monorepo1.root,
+            environmentGlob: [
+                "some-global.config.js"
+            ]
+        });
+        const target = createTarget(monorepo1.root, "package-a", "build");
+        const hash = await getHash(hasher, target);
+        const hasher2 = new _index.TargetHasher({
+            root: monorepo1.root,
+            environmentGlob: []
+        });
+        const target2 = createTarget(monorepo1.root, "package-a", "build");
+        const hash2 = await getHash(hasher2, target2);
+        expect(hash).not.toEqual(hash2);
+        monorepo1.cleanup();
+    });
     it("creates different hashes given different fixtures", async ()=>{
         const monorepo1 = await setupFixture("monorepo");
         const hasher = new _index.TargetHasher({

package/lib/expandInputPatterns.d.ts

@@ -0,0 +1,3 @@
+import { type Target } from "@lage-run/target-graph";
+import { type DependencyMap } from "workspace-tools";
+export declare function expandInputPatterns(patterns: string[], target: Target, dependencyMap: DependencyMap): Record<string, string[]>;

package/lib/expandInputPatterns.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "expandInputPatterns", {
+    enumerable: true,
+    get: function() {
+        return expandInputPatterns;
+    }
+});
+function expandInputPatterns(patterns, target, dependencyMap) {
+    const expandedPatterns = {};
+    for (const pattern of patterns){
+        if (pattern.startsWith("^") || pattern.startsWith("!^")) {
+            const matchPattern = pattern.replace("^", "");
+            // get all the packages that are transitive deps and add them to the list
+            const queue = [
+                target.packageName
+            ];
+            const visited = new Set();
+            while(queue.length > 0){
+                const pkg = queue.pop();
+                if (visited.has(pkg)) {
+                    continue;
+                }
+                visited.add(pkg);
+                if (pkg !== target.packageName) {
+                    expandedPatterns[pkg] = expandedPatterns[pkg] ?? [];
+                    expandedPatterns[pkg].push(matchPattern);
+                }
+                if (dependencyMap.dependencies.has(pkg)) {
+                    const deps = dependencyMap.dependencies.get(pkg);
+                    if (deps) {
+                        for (const dep of deps){
+                            queue.push(dep);
+                        }
+                    }
+                }
+            }
+        } else {
+            const pkg = target.packageName;
+            expandedPatterns[pkg] = expandedPatterns[pkg] ?? [];
+            expandedPatterns[pkg].push(pattern);
+        }
+    }
+    return expandedPatterns;
+}

package/lib/getInputFiles.d.ts

@@ -0,0 +1,4 @@
+import { type Target } from "@lage-run/target-graph";
+import { type DependencyMap } from "workspace-tools";
+import { type PackageTree } from "./PackageTree.js";
+export declare function getInputFiles(target: Target, dependencyMap: DependencyMap, packageTree: PackageTree): string[];

package/lib/getInputFiles.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "getInputFiles", {
+    enumerable: true,
+    get: function() {
+        return getInputFiles;
+    }
+});
+const _expandInputPatterns = require("./expandInputPatterns.js");
+function getInputFiles(target, dependencyMap, packageTree) {
+    const inputs = target.inputs ?? [
+        "**/*"
+    ];
+    const packagePatterns = (0, _expandInputPatterns.expandInputPatterns)(inputs, target, dependencyMap);
+    const files = [];
+    for (const [pkg, patterns] of Object.entries(packagePatterns)){
+        const packageFiles = packageTree.getPackageFiles(pkg, patterns);
+        files.push(...packageFiles);
+    }
+    return files;
+}

package/lib/index.d.ts

@@ -1 +1,3 @@
 export { TargetHasher } from "./TargetHasher.js";
+export { PackageTree } from "./PackageTree.js";
+export { getInputFiles } from "./getInputFiles.js";

package/lib/index.js

@@ -2,10 +2,23 @@
 Object.defineProperty(exports, "__esModule", {
     value: true
 });
-Object.defineProperty(exports, "TargetHasher", {
-    enumerable: true,
-    get: function() {
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: all[name]
+    });
+}
+_export(exports, {
+    TargetHasher: function() {
         return _TargetHasher.TargetHasher;
+    },
+    PackageTree: function() {
+        return _PackageTree.PackageTree;
+    },
+    getInputFiles: function() {
+        return _getInputFiles.getInputFiles;
     }
 });
 const _TargetHasher = require("./TargetHasher.js");
+const _PackageTree = require("./PackageTree.js");
+const _getInputFiles = require("./getInputFiles.js");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lage-run/hasher",
-  "version": "1.3.4",
+  "version": "1.5.0",
   "description": "Hasher for Lage Targets",
   "repository": {
     "url": "https://github.com/microsoft/lage"
@@ -15,10 +15,10 @@
     "lint": "monorepo-scripts lint"
   },
   "dependencies": {
+    "@lage-run/globby": "^14.1.0",
     "@lage-run/logger": "^1.3.1",
     "@lage-run/target-graph": "^0.8.10",
     "execa": "5.1.1",
-    "fast-glob": "3.3.2",
     "glob-hasher": "^1.4.2",
     "graceful-fs": "4.2.11",
     "micromatch": "4.0.8",