@lage-run/hasher 1.0.7 → 1.1.1

package/CHANGELOG.json

@@ -2,7 +2,52 @@
   "name": "@lage-run/hasher",
   "entries": [
     {
-      "date": "Thu, 21 Dec 2023 09:48:33 GMT",
+      "date": "Sun, 05 May 2024 22:55:31 GMT",
+      "version": "1.1.1",
+      "tag": "@lage-run/hasher_v1.1.1",
+      "comments": {
+        "patch": [
+          {
+            "author": "kchau@microsoft.com",
+            "package": "@lage-run/hasher",
+            "commit": "1e36de04ab83fc0cde38062fc1543e4b12902166",
+            "comment": "fixing hashing issues related to rust panic"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Wed, 17 Apr 2024 23:20:47 GMT",
+      "version": "1.1.0",
+      "tag": "@lage-run/hasher_v1.1.0",
+      "comments": {
+        "none": [
+          {
+            "author": "elcraig@microsoft.com",
+            "package": "@lage-run/hasher",
+            "commit": "fb4fcb8419cc778210104d7d04102fc95df13d5b",
+            "comment": "Update formatting"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Fri, 15 Mar 2024 04:35:11 GMT",
+      "version": "1.1.0",
+      "tag": "@lage-run/hasher_v1.1.0",
+      "comments": {
+        "minor": [
+          {
+            "author": "kchau@microsoft.com",
+            "package": "@lage-run/hasher",
+            "commit": "71283d4af8888454c953e5228c97bfbb471c15ba",
+            "comment": "perf optimizations"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Thu, 21 Dec 2023 09:49:09 GMT",
       "version": "1.0.7",
       "tag": "@lage-run/hasher_v1.0.7",
       "comments": {

package/CHANGELOG.md

@@ -1,12 +1,28 @@
 # Change Log - @lage-run/hasher
 
-This log was last generated on Thu, 21 Dec 2023 09:48:33 GMT and should not be manually modified.
+This log was last generated on Sun, 05 May 2024 22:55:31 GMT and should not be manually modified.
 
 <!-- Start content -->
 
+## 1.1.1
+
+Sun, 05 May 2024 22:55:31 GMT
+
+### Patches
+
+- fixing hashing issues related to rust panic (kchau@microsoft.com)
+
+## 1.1.0
+
+Fri, 15 Mar 2024 04:35:11 GMT
+
+### Minor changes
+
+- perf optimizations (kchau@microsoft.com)
+
 ## 1.0.7
 
-Thu, 21 Dec 2023 09:48:33 GMT
+Thu, 21 Dec 2023 09:49:09 GMT
 
 ### Patches
 

package/README.md

@@ -1,2 +1,3 @@
 # @lage-run/hasher
-This package takes code from both backfill-hasher & package-dep-hash and strips out the extraneous dependencies: backfill-logger and @rushstack/node-core-lib. This is done so that `lage` can become a pure ES Module package. It also allows us to control how the hashing works should lage gains the ability to customize the `inputs` in the future.
+
+This package takes code from both backfill-hasher & package-dep-hash and strips out the extraneous dependencies: backfill-logger and @rushstack/node-core-lib. This is done so that `lage` can become a pure ES Module package. It also allows us to control how the hashing works should lage gains the ability to customize the `inputs` in the future.

package/lib/FileHasher.js

@@ -106,12 +106,13 @@ class FileHasher {
                 input: inputStream,
                 crlfDelay: Infinity
             });
+            let info = [];
             rl.on("line", (line)=>{
-                const [relativePath, mtimeStr, sizeStr, hash] = line.split("\0");
-                _class_private_field_get(this, _store)[relativePath] = {
-                    mtime: BigInt(mtimeStr),
-                    size: parseInt(sizeStr),
-                    hash
+                info = line.split("\0");
+                _class_private_field_get(this, _store)[info[0]] = {
+                    mtime: BigInt(info[1]),
+                    size: parseInt(info[2]),
+                    hash: info[3]
                 };
             });
             inputStream.on("end", ()=>{
@@ -124,11 +125,10 @@ class FileHasher {
         _gracefulfs.default.mkdirSync(_path.default.dirname(_class_private_field_get(this, _manifestFile)), {
             recursive: true
         });
-        const outputStream = _gracefulfs.default.createWriteStream(_class_private_field_get(this, _manifestFile), "utf-8");
-        for (const [relativePath, info] of Object.entries(_class_private_field_get(this, _store))){
-            outputStream.write(`${relativePath}\0${info.mtime.toString()}\0${info.size.toString()}\0${info.hash}\n`);
-        }
-        outputStream.end();
+        const outputLines = Object.entries(_class_private_field_get(this, _store)).map(([relativePath, info])=>{
+            return `${relativePath}\0${info.mtime.toString()}\0${info.size.toString()}\0${info.hash}`;
+        });
+        _gracefulfs.default.writeFileSync(_class_private_field_get(this, _manifestFile), outputLines.join("\n"), "utf-8");
     }
     hash(files) {
         const hashes = {};
@@ -156,9 +156,9 @@ class FileHasher {
             _class_private_field_get(this, _store)[file] = {
                 mtime: stat.mtimeMs,
                 size: Number(stat.size),
-                hash
+                hash: hash ?? ""
             };
-            hashes[file] = hash;
+            hashes[file] = hash ?? "";
         }
         return hashes;
     }

package/lib/PackageTree.d.ts

@@ -1,20 +1,16 @@
-import { type PackageInfos } from "workspace-tools";
-export interface PackageTreeOptions {
+export interface LocalPackageTreeOptions {
     root: string;
-    packageInfos: PackageInfos;
     includeUntracked: boolean;
 }
 /**
- * Package Tree keeps a data structure to quickly find all files in a package.
- *
- * TODO: add a watcher to make sure the tree is up to date during a "watched" run.
+ * Keeps a data structure to quickly find all files in a package.
  */
 export declare class PackageTree {
     #private;
     private options;
-    constructor(options: PackageTreeOptions);
+    constructor(options: LocalPackageTreeOptions);
     reset(): void;
     initialize(): Promise<void>;
-    addToPackageTree(filePaths: string[]): Promise<void>;
-    getPackageFiles(packageName: string, patterns: string[]): string[];
+    findFilesInPath(packagePath: string, patterns: string[]): Promise<string[]>;
+    cleanup(): void;
 }

package/lib/PackageTree.js

@@ -11,7 +11,6 @@ Object.defineProperty(exports, "PackageTree", {
 const _execa = /*#__PURE__*/ _interop_require_default(require("execa"));
 const _path = /*#__PURE__*/ _interop_require_default(require("path"));
 const _fs = /*#__PURE__*/ _interop_require_default(require("fs"));
-const _micromatch = /*#__PURE__*/ _interop_require_default(require("micromatch"));
 function _check_private_redeclaration(obj, privateCollection) {
     if (privateCollection.has(obj)) {
         throw new TypeError("Cannot initialize the same private elements twice on an object");
@@ -52,6 +51,16 @@ function _class_private_field_set(receiver, privateMap, value) {
     _class_apply_descriptor_set(receiver, descriptor, value);
     return value;
 }
+function _class_private_method_get(receiver, privateSet, fn) {
+    if (!privateSet.has(receiver)) {
+        throw new TypeError("attempted to get private field on non-instance");
+    }
+    return fn;
+}
+function _class_private_method_init(obj, privateSet) {
+    _check_private_redeclaration(obj, privateSet);
+    privateSet.add(obj);
+}
 function _define_property(obj, key, value) {
     if (key in obj) {
         Object.defineProperty(obj, key, {
@@ -70,7 +79,7 @@ function _interop_require_default(obj) {
         default: obj
     };
 }
-var _tree = /*#__PURE__*/ new WeakMap(), _packageFiles = /*#__PURE__*/ new WeakMap(), _memoizedPackageFiles = /*#__PURE__*/ new WeakMap();
+var _tree = /*#__PURE__*/ new WeakMap(), _packageFiles = /*#__PURE__*/ new WeakMap(), _memoizedPackageFiles = /*#__PURE__*/ new WeakMap(), _findFilesFromGitTree = /*#__PURE__*/ new WeakSet();
 class PackageTree {
     reset() {
         _class_private_field_set(this, _tree, {});
@@ -78,86 +87,19 @@ class PackageTree {
         _class_private_field_set(this, _memoizedPackageFiles, {});
     }
     async initialize() {
-        const { root , includeUntracked , packageInfos  } = this.options;
         this.reset();
-        // Generate path tree of all packages in workspace (scale: ~2000 * ~3)
-        for (const info of Object.values(packageInfos)){
-            const packagePath = _path.default.dirname(info.packageJsonPath);
-            const pathParts = _path.default.relative(root, packagePath).split(/[\\/]/);
-            let currentNode = _class_private_field_get(this, _tree);
-            for (const part of pathParts){
-                currentNode[part] = currentNode[part] || {};
-                currentNode = currentNode[part];
-            }
-        }
-        // Get all files in the workspace (scale: ~2000) according to git
-        const lsFilesResults = await (0, _execa.default)("git", [
-            "ls-files",
-            "-z"
-        ], {
-            cwd: root
-        });
-        if (lsFilesResults.exitCode === 0) {
-            const files = lsFilesResults.stdout.split("\0").filter((f)=>Boolean(f) && _fs.default.existsSync(_path.default.join(root, f)));
-            this.addToPackageTree(files);
-        }
-        if (includeUntracked) {
-            // Also get all untracked files in the workspace according to git
-            const lsOtherResults = await (0, _execa.default)("git", [
-                "ls-files",
-                "-o",
-                "--exclude-standard"
-            ], {
-                cwd: root
-            });
-            if (lsOtherResults.exitCode === 0) {
-                const files = lsOtherResults.stdout.split("\0").filter(Boolean);
-                this.addToPackageTree(files);
-            }
-        }
     }
-    async addToPackageTree(filePaths) {
-        // key: path/to/package (packageRoot), value: array of a tuple of [file, hash]
-        const packageFiles = _class_private_field_get(this, _packageFiles);
-        for (const entry of filePaths){
-            const pathParts = entry.split(/[\\/]/);
-            let node = _class_private_field_get(this, _tree);
-            const packagePathParts = [];
-            for (const part of pathParts){
-                if (node[part]) {
-                    node = node[part];
-                    packagePathParts.push(part);
-                } else {
-                    break;
-                }
-            }
-            const packageRoot = packagePathParts.join("/");
-            packageFiles[packageRoot] = packageFiles[packageRoot] || [];
-            packageFiles[packageRoot].push(entry);
-        }
-    }
-    getPackageFiles(packageName, patterns) {
-        const { root , packageInfos  } = this.options;
-        const packagePath = _path.default.relative(root, _path.default.dirname(packageInfos[packageName].packageJsonPath)).replace(/\\/g, "/");
-        const packageFiles = _class_private_field_get(this, _packageFiles)[packagePath];
-        if (!packageFiles) {
-            return [];
-        }
-        const key = `${packageName}\0${patterns.join("\0")}`;
+    async findFilesInPath(packagePath, patterns) {
+        const key = `${packagePath}\0${patterns.join("\0")}`;
         if (!_class_private_field_get(this, _memoizedPackageFiles)[key]) {
-            const packagePatterns = patterns.map((pattern)=>{
-                if (pattern.startsWith("!")) {
-                    return `!${_path.default.join(packagePath, pattern.slice(1)).replace(/\\/g, "/")}`;
-                }
-                return _path.default.join(packagePath, pattern).replace(/\\/g, "/");
-            });
-            _class_private_field_get(this, _memoizedPackageFiles)[key] = (0, _micromatch.default)(packageFiles, packagePatterns, {
-                dot: true
-            });
+            const files = (await _class_private_method_get(this, _findFilesFromGitTree, findFilesFromGitTree).call(this, packagePath, patterns)).map((f)=>_path.default.join(packagePath, f));
+            _class_private_field_get(this, _memoizedPackageFiles)[key] = files;
         }
         return _class_private_field_get(this, _memoizedPackageFiles)[key];
     }
+    cleanup() {}
     constructor(options){
+        _class_private_method_init(this, _findFilesFromGitTree);
         _define_property(this, "options", void 0);
         _class_private_field_init(this, _tree, {
             writable: true,
@@ -177,3 +119,40 @@ class PackageTree {
         _class_private_field_set(this, _memoizedPackageFiles, {});
     }
 }
+async function findFilesFromGitTree(packagePath, patterns) {
+    const { includeUntracked  } = this.options;
+    const cwd = _path.default.isAbsolute(packagePath) ? packagePath : _path.default.join(this.options.root, packagePath);
+    const trackedPromise = (0, _execa.default)("git", [
+        "ls-files",
+        "-z",
+        ...patterns.filter((p)=>!p.startsWith("!")),
+        ...patterns.filter((p)=>p.startsWith("!")).map((p)=>`:!:${p.slice(1)}`)
+    ], {
+        cwd
+    }).then((lsFilesResults)=>{
+        if (lsFilesResults.exitCode === 0) {
+            return lsFilesResults.stdout.split("\0").filter((f)=>Boolean(f) && _fs.default.existsSync(_path.default.join(cwd, f)));
+        }
+        return [];
+    });
+    const untrackedPromise = includeUntracked ? (0, _execa.default)("git", [
+        "ls-files",
+        "-z",
+        "-o",
+        "--exclude-standard",
+        ...patterns.filter((p)=>!p.startsWith("!")),
+        ...patterns.filter((p)=>p.startsWith("!")).map((p)=>`:!:${p.slice(1)}`)
+    ], {
+        cwd
+    }).then((lsOtherResults)=>{
+        if (lsOtherResults.exitCode === 0) {
+            return lsOtherResults.stdout.split("\0").filter((f)=>Boolean(f));
+        }
+        return [];
+    }) : Promise.resolve([]);
+    const [trackedFiles, untrackedFiles] = await Promise.all([
+        trackedPromise,
+        untrackedPromise
+    ]);
+    return trackedFiles.concat(untrackedFiles);
+}

package/lib/TargetHasher.d.ts

@@ -32,7 +32,7 @@ export declare class TargetHasher {
     private options;
     logger: Logger | undefined;
     fileHasher: FileHasher;
-    packageTree: PackageTree | undefined;
+    packageTree: PackageTree;
     initializedPromise: Promise<unknown> | undefined;
     packageInfos: PackageInfos;
     workspaceInfo: WorkspaceInfo | undefined;

package/lib/TargetHasher.js

@@ -9,7 +9,6 @@ Object.defineProperty(exports, "TargetHasher", {
     }
 });
 const _globhasher = require("glob-hasher");
-const _fastglob = /*#__PURE__*/ _interop_require_default(require("fast-glob"));
 const _fs = /*#__PURE__*/ _interop_require_default(require("fs"));
 const _path = /*#__PURE__*/ _interop_require_default(require("path"));
 const _workspacetools = require("workspace-tools");
@@ -109,26 +108,19 @@ class TargetHasher {
             await this.initializedPromise;
             return;
         }
-        this.initializedPromise = Promise.all([
-            this.fileHasher.readManifest().then(()=>(0, _fastglob.default)(environmentGlob, {
-                    cwd: root
-                })).then((files)=>this.fileHasher.hash(files)).then((hash)=>this.globalInputsHash = hash),
-            (0, _workspacetools.getWorkspacesAsync)(root).then((workspaceInfo)=>this.workspaceInfo = workspaceInfo).then(()=>{
-                this.packageInfos = this.getPackageInfos(this.workspaceInfo);
-                this.dependencyMap = (0, _workspacetools.createDependencyMap)(this.packageInfos, {
-                    withDevDependencies: true,
-                    withPeerDependencies: false
-                });
-                this.packageTree = new _PackageTree.PackageTree({
-                    root,
-                    packageInfos: this.packageInfos,
-                    // TODO: (optimization) false if process.env.TF_BUILD || process.env.CI
-                    includeUntracked: true
-                });
-                return this.packageTree.initialize();
-            }),
-            (0, _workspacetools.parseLockFile)(root).then((lockInfo)=>this.lockInfo = lockInfo)
-        ]);
+        this.workspaceInfo = (0, _workspacetools.getWorkspaces)(root);
+        this.packageInfos = this.getPackageInfos(this.workspaceInfo);
+        this.dependencyMap = (0, _workspacetools.createDependencyMap)(this.packageInfos, {
+            withDevDependencies: true,
+            withPeerDependencies: false
+        });
+        const finderInitPromise = this.packageTree.initialize();
+        this.initializedPromise = finderInitPromise.then(()=>{
+            return Promise.all([
+                this.fileHasher.readManifest().then(()=>environmentGlob.length > 0 ? this.packageTree.findFilesInPath(root, environmentGlob) : []).then((files)=>this.fileHasher.hash(files)).then((hash)=>this.globalInputsHash = hash),
+                (0, _workspacetools.parseLockFile)(root).then((lockInfo)=>this.lockInfo = lockInfo)
+            ]);
+        });
         await this.initializedPromise;
         if (this.logger !== undefined) {
             const globalInputsHash = (0, _hashStrings.hashStrings)(Object.values(this.globalInputsHash ?? {}));
@@ -145,13 +137,11 @@ class TargetHasher {
             if (!target.inputs) {
                 throw new Error("Root-level targets must have `inputs` defined if it has cache enabled.");
             }
-            const files = await (0, _fastglob.default)(target.inputs, {
-                cwd: root
-            });
+            const files = await this.packageTree.findFilesInPath(root, target.inputs);
             const fileFashes = (0, _globhasher.hash)(files, {
                 cwd: root
             }) ?? {};
-            const hashes = Object.values(fileFashes);
+            const hashes = Object.values(fileFashes).filter((hash)=>hash !== undefined && hash !== null);
             return (0, _hashStrings.hashStrings)(hashes);
         }
         // 1. add hash of target's inputs
@@ -175,7 +165,9 @@ class TargetHasher {
         const packagePatterns = this.expandInputPatterns(inputs, target);
         const files = [];
         for (const [pkg, patterns] of Object.entries(packagePatterns)){
-            const packageFiles = this.packageTree.getPackageFiles(pkg, patterns);
+            const { root  } = this.options;
+            const packagePath = _path.default.relative(root, _path.default.dirname(this.packageInfos[pkg].packageJsonPath)).replace(/\\/g, "/");
+            const packageFiles = await this.packageTree.findFilesInPath(packagePath, patterns);
             files.push(...packageFiles);
         }
         const fileHashes = this.fileHasher.hash(files) ?? {}; // this list is sorted by file name
@@ -197,7 +189,8 @@ class TargetHasher {
         return hashString;
     }
     async cleanup() {
-        await this.fileHasher.writeManifest();
+        this.packageTree.cleanup();
+        this.fileHasher.writeManifest();
     }
     constructor(options){
         _define_property(this, "options", void 0);
@@ -223,5 +216,9 @@ class TargetHasher {
         this.fileHasher = new _FileHasher.FileHasher({
             root
         });
+        this.packageTree = new _PackageTree.PackageTree({
+            root,
+            includeUntracked: true
+        });
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lage-run/hasher",
-  "version": "1.0.7",
+  "version": "1.1.1",
   "description": "Hasher for Lage Targets",
   "repository": {
     "url": "https://github.com/microsoft/lage"
@@ -19,10 +19,8 @@
     "@lage-run/logger": "^1.3.0",
     "execa": "5.1.1",
     "workspace-tools": "0.36.4",
-    "fast-glob": "3.3.1",
-    "glob-hasher": "^1.3.0",
-    "graceful-fs": "4.2.11",
-    "micromatch": "4.0.5"
+    "glob-hasher": "^1.4.2",
+    "graceful-fs": "4.2.11"
   },
   "devDependencies": {
     "@lage-run/monorepo-fixture": "*",