@lage-run/hasher 1.2.0 → 1.2.1

package/CHANGELOG.json

@@ -2,82 +2,16 @@
   "name": "@lage-run/hasher",
   "entries": [
     {
-      "date": "Tue, 25 Jun 2024 18:10:41 GMT",
-      "version": "1.2.0",
-      "tag": "@lage-run/hasher_v1.2.0",
-      "comments": {
-        "minor": [
-          {
-            "author": "kchau@microsoft.com",
-            "package": "@lage-run/hasher",
-            "commit": "e294f805313ceb5b66640f8aaff8647bdb588511",
-            "comment": "moving back to using globby instead for speed & accuracy reasons"
-          },
-          {
-            "author": "beachball",
-            "package": "@lage-run/hasher",
-            "comment": "Bump @lage-run/globby to v14.0.2",
-            "commit": "not available"
-          }
-        ]
-      }
-    },
-    {
-      "date": "Mon, 10 Jun 2024 23:50:39 GMT",
-      "version": "1.1.2",
-      "tag": "@lage-run/hasher_v1.1.2",
+      "date": "Tue, 25 Jun 2024 22:03:26 GMT",
+      "version": "1.2.1",
+      "tag": "@lage-run/hasher_v1.2.1",
       "comments": {
         "patch": [
           {
             "author": "kchau@microsoft.com_msteamsmdb",
             "package": "@lage-run/hasher",
-            "commit": "0894d97ebfedc339b77161c40fa8643d93e4486d",
-            "comment": "matching correct glob and exclude patterns"
-          }
-        ]
-      }
-    },
-    {
-      "date": "Sun, 05 May 2024 22:55:45 GMT",
-      "version": "1.1.1",
-      "tag": "@lage-run/hasher_v1.1.1",
-      "comments": {
-        "patch": [
-          {
-            "author": "kchau@microsoft.com",
-            "package": "@lage-run/hasher",
-            "commit": "1e36de04ab83fc0cde38062fc1543e4b12902166",
-            "comment": "fixing hashing issues related to rust panic"
-          }
-        ]
-      }
-    },
-    {
-      "date": "Wed, 17 Apr 2024 23:20:47 GMT",
-      "version": "1.1.0",
-      "tag": "@lage-run/hasher_v1.1.0",
-      "comments": {
-        "none": [
-          {
-            "author": "elcraig@microsoft.com",
-            "package": "@lage-run/hasher",
-            "commit": "fb4fcb8419cc778210104d7d04102fc95df13d5b",
-            "comment": "Update formatting"
-          }
-        ]
-      }
-    },
-    {
-      "date": "Fri, 15 Mar 2024 04:35:11 GMT",
-      "version": "1.1.0",
-      "tag": "@lage-run/hasher_v1.1.0",
-      "comments": {
-        "minor": [
-          {
-            "author": "kchau@microsoft.com",
-            "package": "@lage-run/hasher",
-            "commit": "71283d4af8888454c953e5228c97bfbb471c15ba",
-            "comment": "perf optimizations"
+            "commit": "440bbd19c317982a187383a5a35befbea71f22c5",
+            "comment": "reverting all the hasher changes"
           }
         ]
       }

package/CHANGELOG.md

@@ -1,41 +1,16 @@
 # Change Log - @lage-run/hasher
 
-This log was last generated on Tue, 25 Jun 2024 18:10:41 GMT and should not be manually modified.
+This log was last generated on Tue, 25 Jun 2024 22:03:26 GMT and should not be manually modified.
 
 <!-- Start content -->
 
-## 1.2.0
+## 1.2.1
 
-Tue, 25 Jun 2024 18:10:41 GMT
-
-### Minor changes
-
-- moving back to using globby instead for speed & accuracy reasons (kchau@microsoft.com)
-- Bump @lage-run/globby to v14.0.2
-
-## 1.1.2
-
-Mon, 10 Jun 2024 23:50:39 GMT
+Tue, 25 Jun 2024 22:03:26 GMT
 
 ### Patches
 
-- matching correct glob and exclude patterns (kchau@microsoft.com_msteamsmdb)
-
-## 1.1.1
-
-Sun, 05 May 2024 22:55:45 GMT
-
-### Patches
-
-- fixing hashing issues related to rust panic (kchau@microsoft.com)
-
-## 1.1.0
-
-Fri, 15 Mar 2024 04:35:11 GMT
-
-### Minor changes
-
-- perf optimizations (kchau@microsoft.com)
+- reverting all the hasher changes (kchau@microsoft.com_msteamsmdb)
 
 ## 1.0.7
 

package/lib/PackageTree.d.ts

@@ -1,16 +1,20 @@
-export interface LocalPackageTreeOptions {
+import { type PackageInfos } from "workspace-tools";
+export interface PackageTreeOptions {
     root: string;
+    packageInfos: PackageInfos;
     includeUntracked: boolean;
 }
 /**
- * Keeps a data structure to quickly find all files in a package.
+ * Package Tree keeps a data structure to quickly find all files in a package.
+ *
+ * TODO: add a watcher to make sure the tree is up to date during a "watched" run.
  */
 export declare class PackageTree {
     #private;
     private options;
-    constructor(options: LocalPackageTreeOptions);
+    constructor(options: PackageTreeOptions);
     reset(): void;
     initialize(): Promise<void>;
-    findFilesInPath(packagePath: string, patterns: string[]): Promise<string[]>;
-    cleanup(): void;
+    addToPackageTree(filePaths: string[]): Promise<void>;
+    getPackageFiles(packageName: string, patterns: string[]): string[];
 }

package/lib/PackageTree.js

@@ -8,8 +8,10 @@ Object.defineProperty(exports, "PackageTree", {
         return PackageTree;
     }
 });
+const _execa = /*#__PURE__*/ _interop_require_default(require("execa"));
 const _path = /*#__PURE__*/ _interop_require_default(require("path"));
-const _globby = require("@lage-run/globby");
+const _fs = /*#__PURE__*/ _interop_require_default(require("fs"));
+const _micromatch = /*#__PURE__*/ _interop_require_default(require("micromatch"));
 function _check_private_redeclaration(obj, privateCollection) {
     if (privateCollection.has(obj)) {
         throw new TypeError("Cannot initialize the same private elements twice on an object");
@@ -50,16 +52,6 @@ function _class_private_field_set(receiver, privateMap, value) {
     _class_apply_descriptor_set(receiver, descriptor, value);
     return value;
 }
-function _class_private_method_get(receiver, privateSet, fn) {
-    if (!privateSet.has(receiver)) {
-        throw new TypeError("attempted to get private field on non-instance");
-    }
-    return fn;
-}
-function _class_private_method_init(obj, privateSet) {
-    _check_private_redeclaration(obj, privateSet);
-    privateSet.add(obj);
-}
 function _define_property(obj, key, value) {
     if (key in obj) {
         Object.defineProperty(obj, key, {
@@ -78,43 +70,110 @@ function _interop_require_default(obj) {
         default: obj
     };
 }
-var _memoizedPackageFiles = /*#__PURE__*/ new WeakMap(), _findFilesFromGitTree = /*#__PURE__*/ new WeakSet();
+var _tree = /*#__PURE__*/ new WeakMap(), _packageFiles = /*#__PURE__*/ new WeakMap(), _memoizedPackageFiles = /*#__PURE__*/ new WeakMap();
 class PackageTree {
     reset() {
+        _class_private_field_set(this, _tree, {});
+        _class_private_field_set(this, _packageFiles, {});
         _class_private_field_set(this, _memoizedPackageFiles, {});
     }
     async initialize() {
+        const { root , includeUntracked , packageInfos  } = this.options;
         this.reset();
+        // Generate path tree of all packages in workspace (scale: ~2000 * ~3)
+        for (const info of Object.values(packageInfos)){
+            const packagePath = _path.default.dirname(info.packageJsonPath);
+            const pathParts = _path.default.relative(root, packagePath).split(/[\\/]/);
+            let currentNode = _class_private_field_get(this, _tree);
+            for (const part of pathParts){
+                currentNode[part] = currentNode[part] || {};
+                currentNode = currentNode[part];
+            }
+        }
+        // Get all files in the workspace (scale: ~2000) according to git
+        const lsFilesResults = await (0, _execa.default)("git", [
+            "ls-files",
+            "-z"
+        ], {
+            cwd: root
+        });
+        if (lsFilesResults.exitCode === 0) {
+            const files = lsFilesResults.stdout.split("\0").filter((f)=>Boolean(f) && _fs.default.existsSync(_path.default.join(root, f)));
+            this.addToPackageTree(files);
+        }
+        if (includeUntracked) {
+            // Also get all untracked files in the workspace according to git
+            const lsOtherResults = await (0, _execa.default)("git", [
+                "ls-files",
+                "-o",
+                "--exclude-standard"
+            ], {
+                cwd: root
+            });
+            if (lsOtherResults.exitCode === 0) {
+                const files = lsOtherResults.stdout.split("\0").filter(Boolean);
+                this.addToPackageTree(files);
+            }
+        }
+    }
+    async addToPackageTree(filePaths) {
+        // key: path/to/package (packageRoot), value: array of a tuple of [file, hash]
+        const packageFiles = _class_private_field_get(this, _packageFiles);
+        for (const entry of filePaths){
+            const pathParts = entry.split(/[\\/]/);
+            let node = _class_private_field_get(this, _tree);
+            const packagePathParts = [];
+            for (const part of pathParts){
+                if (node[part]) {
+                    node = node[part];
+                    packagePathParts.push(part);
+                } else {
+                    break;
+                }
+            }
+            const packageRoot = packagePathParts.join("/");
+            packageFiles[packageRoot] = packageFiles[packageRoot] || [];
+            packageFiles[packageRoot].push(entry);
+        }
     }
-    async findFilesInPath(packagePath, patterns) {
-        const key = `${packagePath}\0${patterns.join("\0")}`;
+    getPackageFiles(packageName, patterns) {
+        const { root , packageInfos  } = this.options;
+        const packagePath = _path.default.relative(root, _path.default.dirname(packageInfos[packageName].packageJsonPath)).replace(/\\/g, "/");
+        const packageFiles = _class_private_field_get(this, _packageFiles)[packagePath];
+        if (!packageFiles) {
+            return [];
+        }
+        const key = `${packageName}\0${patterns.join("\0")}`;
         if (!_class_private_field_get(this, _memoizedPackageFiles)[key]) {
-            const files = await _class_private_method_get(this, _findFilesFromGitTree, findFilesFromGitTree).call(this, packagePath, patterns);
-            _class_private_field_get(this, _memoizedPackageFiles)[key] = files;
+            const packagePatterns = patterns.map((pattern)=>{
+                if (pattern.startsWith("!")) {
+                    return `!${_path.default.join(packagePath, pattern.slice(1)).replace(/\\/g, "/")}`;
+                }
+                return _path.default.join(packagePath, pattern).replace(/\\/g, "/");
+            });
+            _class_private_field_get(this, _memoizedPackageFiles)[key] = (0, _micromatch.default)(packageFiles, packagePatterns, {
+                dot: true
+            });
         }
         return _class_private_field_get(this, _memoizedPackageFiles)[key];
     }
-    cleanup() {}
     constructor(options){
-        _class_private_method_init(this, _findFilesFromGitTree);
         _define_property(this, "options", void 0);
+        _class_private_field_init(this, _tree, {
+            writable: true,
+            value: void 0
+        });
+        _class_private_field_init(this, _packageFiles, {
+            writable: true,
+            value: void 0
+        });
         _class_private_field_init(this, _memoizedPackageFiles, {
             writable: true,
             value: void 0
         });
         this.options = options;
+        _class_private_field_set(this, _tree, {});
+        _class_private_field_set(this, _packageFiles, {});
         _class_private_field_set(this, _memoizedPackageFiles, {});
     }
 }
-async function findFilesFromGitTree(packagePath, patterns) {
-    const cwd = _path.default.isAbsolute(packagePath) ? packagePath : _path.default.join(this.options.root, packagePath);
-    return (0, _globby.globby)(patterns, {
-        cwd,
-        onlyFiles: true,
-        ignore: [
-            ".git"
-        ],
-        gitignore: true,
-        absolute: true
-    }) ?? [];
-}

package/lib/TargetHasher.d.ts

@@ -32,7 +32,7 @@ export declare class TargetHasher {
     private options;
     logger: Logger | undefined;
     fileHasher: FileHasher;
-    packageTree: PackageTree;
+    packageTree: PackageTree | undefined;
     initializedPromise: Promise<unknown> | undefined;
     packageInfos: PackageInfos;
     workspaceInfo: WorkspaceInfo | undefined;

package/lib/TargetHasher.js

@@ -9,6 +9,7 @@ Object.defineProperty(exports, "TargetHasher", {
     }
 });
 const _globhasher = require("glob-hasher");
+const _fastglob = /*#__PURE__*/ _interop_require_default(require("fast-glob"));
 const _fs = /*#__PURE__*/ _interop_require_default(require("fs"));
 const _path = /*#__PURE__*/ _interop_require_default(require("path"));
 const _workspacetools = require("workspace-tools");
@@ -108,25 +109,26 @@ class TargetHasher {
             await this.initializedPromise;
             return;
         }
-        this.workspaceInfo = (0, _workspacetools.getWorkspaces)(root);
-        this.packageInfos = this.getPackageInfos(this.workspaceInfo);
-        this.dependencyMap = (0, _workspacetools.createDependencyMap)(this.packageInfos, {
-            withDevDependencies: true,
-            withPeerDependencies: false
-        });
-        const finderInitPromise = this.packageTree.initialize();
-        this.initializedPromise = finderInitPromise.then(()=>{
-            return Promise.all([
-                this.fileHasher.readManifest().then(()=>environmentGlob.length > 0 ? this.packageTree.findFilesInPath(root, environmentGlob) : []).then((files)=>{
-                    if (this.logger !== undefined) {
-                        this.logger.verbose(`Environment glob files:`);
-                        this.logger.silly(" " + files.join("\n "));
-                    }
-                    return this.fileHasher.hash(files);
-                }).then((hash)=>this.globalInputsHash = hash),
-                (0, _workspacetools.parseLockFile)(root).then((lockInfo)=>this.lockInfo = lockInfo)
-            ]);
-        });
+        this.initializedPromise = Promise.all([
+            this.fileHasher.readManifest().then(()=>(0, _fastglob.default)(environmentGlob, {
+                    cwd: root
+                })).then((files)=>this.fileHasher.hash(files)).then((hash)=>this.globalInputsHash = hash),
+            (0, _workspacetools.getWorkspacesAsync)(root).then((workspaceInfo)=>this.workspaceInfo = workspaceInfo).then(()=>{
+                this.packageInfos = this.getPackageInfos(this.workspaceInfo);
+                this.dependencyMap = (0, _workspacetools.createDependencyMap)(this.packageInfos, {
+                    withDevDependencies: true,
+                    withPeerDependencies: false
+                });
+                this.packageTree = new _PackageTree.PackageTree({
+                    root,
+                    packageInfos: this.packageInfos,
+                    // TODO: (optimization) false if process.env.TF_BUILD || process.env.CI
+                    includeUntracked: true
+                });
+                return this.packageTree.initialize();
+            }),
+            (0, _workspacetools.parseLockFile)(root).then((lockInfo)=>this.lockInfo = lockInfo)
+        ]);
         await this.initializedPromise;
         if (this.logger !== undefined) {
             const globalInputsHash = (0, _hashStrings.hashStrings)(Object.values(this.globalInputsHash ?? {}));
@@ -143,11 +145,13 @@ class TargetHasher {
             if (!target.inputs) {
                 throw new Error("Root-level targets must have `inputs` defined if it has cache enabled.");
             }
-            const files = await this.packageTree.findFilesInPath(root, target.inputs);
+            const files = await (0, _fastglob.default)(target.inputs, {
+                cwd: root
+            });
             const fileFashes = (0, _globhasher.hash)(files, {
                 cwd: root
             }) ?? {};
-            const hashes = Object.values(fileFashes).filter((hash)=>hash !== undefined && hash !== null);
+            const hashes = Object.values(fileFashes);
             return (0, _hashStrings.hashStrings)(hashes);
         }
         // 1. add hash of target's inputs
@@ -171,9 +175,7 @@ class TargetHasher {
         const packagePatterns = this.expandInputPatterns(inputs, target);
         const files = [];
         for (const [pkg, patterns] of Object.entries(packagePatterns)){
-            const { root  } = this.options;
-            const packagePath = _path.default.relative(root, _path.default.dirname(this.packageInfos[pkg].packageJsonPath)).replace(/\\/g, "/");
-            const packageFiles = await this.packageTree.findFilesInPath(packagePath, patterns);
+            const packageFiles = this.packageTree.getPackageFiles(pkg, patterns);
             files.push(...packageFiles);
         }
         const fileHashes = this.fileHasher.hash(files) ?? {}; // this list is sorted by file name
@@ -195,8 +197,7 @@ class TargetHasher {
         return hashString;
     }
     async cleanup() {
-        this.packageTree.cleanup();
-        this.fileHasher.writeManifest();
+        await this.fileHasher.writeManifest();
     }
     constructor(options){
         _define_property(this, "options", void 0);
@@ -222,9 +223,5 @@ class TargetHasher {
         this.fileHasher = new _FileHasher.FileHasher({
             root
         });
-        this.packageTree = new _PackageTree.PackageTree({
-            root,
-            includeUntracked: true
-        });
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lage-run/hasher",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "description": "Hasher for Lage Targets",
   "repository": {
     "url": "https://github.com/microsoft/lage"
@@ -16,12 +16,13 @@
   },
   "dependencies": {
     "@lage-run/target-graph": "^0.8.9",
-    "@lage-run/globby": "^14.0.2",
     "@lage-run/logger": "^1.3.0",
     "execa": "5.1.1",
     "workspace-tools": "0.36.4",
+    "fast-glob": "3.3.1",
     "glob-hasher": "^1.4.2",
-    "graceful-fs": "4.2.11"
+    "graceful-fs": "4.2.11",
+    "micromatch": "4.0.5"
   },
   "devDependencies": {
     "@lage-run/monorepo-fixture": "*",

package/lib/__tests__/PackageTree.test.d.ts

@@ -1 +0,0 @@
-export {};

package/lib/__tests__/PackageTree.test.js

@@ -1,93 +0,0 @@
-// test out the findFilesInPath method of the PackageTree
-"use strict";
-Object.defineProperty(exports, "__esModule", {
-    value: true
-});
-const _monorepofixture = require("@lage-run/monorepo-fixture");
-const _PackageTree = require("../PackageTree");
-const _path = /*#__PURE__*/ _interop_require_default(require("path"));
-function _interop_require_default(obj) {
-    return obj && obj.__esModule ? obj : {
-        default: obj
-    };
-}
-// given various patterns that exercises the globby functionality with extgob, etc.
-describe("PackageTree", ()=>{
-    async function setupFixture(fixture = "monorepo") {
-        const monorepo = new _monorepofixture.Monorepo(fixture);
-        await monorepo.init(_path.default.join(fixturesPath, fixture));
-        return monorepo;
-    }
-    const fixturesPath = _path.default.join(__dirname, "../__fixtures__");
-    it("should find all files in a package using globby basic pattern", async ()=>{
-        const monorepo = await setupFixture("basic");
-        const packagePath = monorepo.root;
-        const patterns = [
-            "**/*"
-        ];
-        const packageTree = new _PackageTree.PackageTree({
-            root: packagePath,
-            includeUntracked: true
-        });
-        await packageTree.initialize();
-        const files = await packageTree.findFilesInPath(packagePath, patterns);
-        expect(serializeFiles(files, packagePath)).toMatchInlineSnapshot(`
-      [
-        "node_modules/package-2/package.json",
-        "package.json",
-        "src/index.ts",
-        "yarn.lock",
-      ]
-    `);
-        monorepo.cleanup();
-    });
-    it("should find all files in a package using globby with extglob", async ()=>{
-        const monorepo = await setupFixture("basic");
-        const packagePath = monorepo.root;
-        const patterns = [
-            "**/*.+(json|ts)",
-            "!yarn.lock"
-        ];
-        const packageTree = new _PackageTree.PackageTree({
-            root: packagePath,
-            includeUntracked: true
-        });
-        await packageTree.initialize();
-        const files = await packageTree.findFilesInPath(packagePath, patterns);
-        expect(serializeFiles(files, packagePath)).toMatchInlineSnapshot(`
-      [
-        "node_modules/package-2/package.json",
-        "package.json",
-        "src/index.ts",
-      ]
-    `);
-        monorepo.cleanup();
-    });
-    it("should find all files in a package using globby with extglob of sub-exclusion pattern", async ()=>{
-        const monorepo = await setupFixture("monorepo-with-deps");
-        const packagePath = monorepo.root;
-        const patterns = [
-            "**/!(package.json)"
-        ];
-        const packageTree = new _PackageTree.PackageTree({
-            root: packagePath,
-            includeUntracked: true
-        });
-        await packageTree.initialize();
-        const files = await packageTree.findFilesInPath(packagePath, patterns);
-        expect(serializeFiles(files, packagePath)).toMatchInlineSnapshot(`
-      [
-        "node_modules/.yarn-integrity",
-        "node_modules/package-a/src/index.ts",
-        "node_modules/package-b/src/index.ts",
-        "packages/package-a/src/index.ts",
-        "packages/package-b/src/index.ts",
-        "yarn.lock",
-      ]
-    `);
-        monorepo.cleanup();
-    });
-    function serializeFiles(files, packagePath) {
-        return files.map((p)=>_path.default.relative(packagePath, p)).map((p)=>p.replace(/\\/g, "/")).sort();
-    }
-});