@lage-run/hasher 0.2.2 → 1.0.1

package/CHANGELOG.json

@@ -2,7 +2,58 @@
   "name": "@lage-run/hasher",
   "entries": [
     {
-      "date": "Wed, 29 Mar 2023 20:02:22 GMT",
+      "date": "Fri, 26 May 2023 01:08:52 GMT",
+      "tag": "@lage-run/hasher_v1.0.1",
+      "version": "1.0.1",
+      "comments": {
+        "patch": [
+          {
+            "author": "kchau@microsoft.com",
+            "package": "@lage-run/hasher",
+            "commit": "8cb78bef8aeb568c2da2ac62d0e0a62d3580847d",
+            "comment": "makes sure the presence of a file does not crash lage on the file hashing"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Mon, 08 May 2023 22:27:16 GMT",
+      "tag": "@lage-run/hasher_v1.0.0",
+      "version": "1.0.0",
+      "comments": {
+        "major": [
+          {
+            "author": "kchau@microsoft.com",
+            "package": "@lage-run/hasher",
+            "commit": "5a132808f166179bc316a279c9e11a13d3a39103",
+            "comment": "Added FileHasher, PackageTree, and its own TargetHasher"
+          },
+          {
+            "author": "beachball",
+            "package": "@lage-run/hasher",
+            "comment": "Bump @lage-run/target-graph to v0.8.6",
+            "commit": "5a132808f166179bc316a279c9e11a13d3a39103"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Thu, 30 Mar 2023 17:47:47 GMT",
+      "tag": "@lage-run/hasher_v0.2.2",
+      "version": "0.2.2",
+      "comments": {
+        "none": [
+          {
+            "author": "kchau@microsoft.com",
+            "package": "@lage-run/hasher",
+            "commit": "2f641860d4a6a97708a94823fa56c507ae00e2c9",
+            "comment": "updated hasher tests not to depend on order"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Wed, 29 Mar 2023 20:02:40 GMT",
       "tag": "@lage-run/hasher_v0.2.2",
       "version": "0.2.2",
       "comments": {

package/CHANGELOG.md

@@ -1,12 +1,29 @@
 # Change Log - @lage-run/hasher
 
-This log was last generated on Wed, 29 Mar 2023 20:02:22 GMT and should not be manually modified.
+This log was last generated on Fri, 26 May 2023 01:08:52 GMT and should not be manually modified.
 
 <!-- Start content -->
 
+## 1.0.1
+
+Fri, 26 May 2023 01:08:52 GMT
+
+### Patches
+
+- makes sure the presence of a file does not crash lage on the file hashing (kchau@microsoft.com)
+
+## 1.0.0
+
+Mon, 08 May 2023 22:27:16 GMT
+
+### Major changes
+
+- Added FileHasher, PackageTree, and its own TargetHasher (kchau@microsoft.com)
+- Bump @lage-run/target-graph to v0.8.6
+
 ## 0.2.2
 
-Wed, 29 Mar 2023 20:02:22 GMT
+Wed, 29 Mar 2023 20:02:40 GMT
 
 ### Patches
 

package/lib/FileHasher.d.ts

@@ -0,0 +1,13 @@
+interface FileHashStoreOptions {
+    root: string;
+}
+export declare class FileHasher {
+    #private;
+    private options;
+    constructor(options: FileHashStoreOptions);
+    getHashesFromGit(): Promise<void>;
+    readManifest(): Promise<void>;
+    writeManifest(): void;
+    hash(files: string[]): Record<string, string>;
+}
+export {};

package/lib/FileHasher.js

@@ -0,0 +1,181 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "FileHasher", {
+    enumerable: true,
+    get: function() {
+        return FileHasher;
+    }
+});
+const _gracefulfs = /*#__PURE__*/ _interop_require_default(require("graceful-fs"));
+const _path = /*#__PURE__*/ _interop_require_default(require("path"));
+const _globhasher = require("glob-hasher");
+const _nodereadline = require("node:readline");
+const _getPackageDeps = require("./getPackageDeps.js");
+function _check_private_redeclaration(obj, privateCollection) {
+    if (privateCollection.has(obj)) {
+        throw new TypeError("Cannot initialize the same private elements twice on an object");
+    }
+}
+function _class_apply_descriptor_get(receiver, descriptor) {
+    if (descriptor.get) {
+        return descriptor.get.call(receiver);
+    }
+    return descriptor.value;
+}
+function _class_apply_descriptor_set(receiver, descriptor, value) {
+    if (descriptor.set) {
+        descriptor.set.call(receiver, value);
+    } else {
+        if (!descriptor.writable) {
+            throw new TypeError("attempted to set read only private field");
+        }
+        descriptor.value = value;
+    }
+}
+function _class_extract_field_descriptor(receiver, privateMap, action) {
+    if (!privateMap.has(receiver)) {
+        throw new TypeError("attempted to " + action + " private field on non-instance");
+    }
+    return privateMap.get(receiver);
+}
+function _class_private_field_get(receiver, privateMap) {
+    var descriptor = _class_extract_field_descriptor(receiver, privateMap, "get");
+    return _class_apply_descriptor_get(receiver, descriptor);
+}
+function _class_private_field_init(obj, privateMap, value) {
+    _check_private_redeclaration(obj, privateMap);
+    privateMap.set(obj, value);
+}
+function _class_private_field_set(receiver, privateMap, value) {
+    var descriptor = _class_extract_field_descriptor(receiver, privateMap, "set");
+    _class_apply_descriptor_set(receiver, descriptor, value);
+    return value;
+}
+function _define_property(obj, key, value) {
+    if (key in obj) {
+        Object.defineProperty(obj, key, {
+            value: value,
+            enumerable: true,
+            configurable: true,
+            writable: true
+        });
+    } else {
+        obj[key] = value;
+    }
+    return obj;
+}
+function _interop_require_default(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
+var _store = /*#__PURE__*/ new WeakMap(), _manifestFile = /*#__PURE__*/ new WeakMap();
+class FileHasher {
+    async getHashesFromGit() {
+        const { root  } = this.options;
+        const fileHashes = await (0, _getPackageDeps.getPackageDeps)(root);
+        const files = [
+            ...fileHashes.keys()
+        ];
+        const fileStats = (0, _globhasher.stat)(files, {
+            cwd: root
+        }) ?? {};
+        for (const [relativePath, fileStat] of Object.entries(fileStats)){
+            const hash = fileHashes.get(relativePath);
+            if (hash) {
+                const { size , mtime  } = fileStat;
+                _class_private_field_get(this, _store)[relativePath] = {
+                    hash,
+                    size,
+                    mtime
+                };
+            }
+        }
+        this.writeManifest();
+    }
+    async readManifest() {
+        return new Promise((resolve)=>{
+            if (!_gracefulfs.default.existsSync(_class_private_field_get(this, _manifestFile))) {
+                this.getHashesFromGit().then(()=>resolve());
+                return;
+            }
+            const inputStream = _gracefulfs.default.createReadStream(_class_private_field_get(this, _manifestFile), "utf-8");
+            const rl = (0, _nodereadline.createInterface)({
+                input: inputStream,
+                crlfDelay: Infinity
+            });
+            rl.on("line", (line)=>{
+                const [relativePath, mtimeStr, sizeStr, hash] = line.split("\0");
+                _class_private_field_get(this, _store)[relativePath] = {
+                    mtime: BigInt(mtimeStr),
+                    size: parseInt(sizeStr),
+                    hash
+                };
+            });
+            inputStream.on("end", ()=>{
+                rl.close();
+                resolve();
+            });
+        });
+    }
+    writeManifest() {
+        _gracefulfs.default.mkdirSync(_path.default.dirname(_class_private_field_get(this, _manifestFile)), {
+            recursive: true
+        });
+        const outputStream = _gracefulfs.default.createWriteStream(_class_private_field_get(this, _manifestFile), "utf-8");
+        for (const [relativePath, info] of Object.entries(_class_private_field_get(this, _store))){
+            outputStream.write(`${relativePath}\0${info.mtime.toString()}\0${info.size.toString()}\0${info.hash}\n`);
+        }
+        outputStream.end();
+    }
+    hash(files) {
+        const hashes = {};
+        const updatedFiles = [];
+        const stats = (0, _globhasher.stat)(files, {
+            cwd: this.options.root
+        }) ?? {};
+        for (const file of files){
+            const stat = stats[file];
+            const info = _class_private_field_get(this, _store)[file];
+            if (info && stat.mtime === info.mtime && stat.size == info.size) {
+                hashes[file] = info.hash;
+            } else {
+                updatedFiles.push(file);
+            }
+        }
+        const updatedHashes = (0, _globhasher.hash)(updatedFiles, {
+            cwd: this.options.root,
+            concurrency: 4
+        }) ?? {};
+        for (const [file, hash] of Object.entries(updatedHashes)){
+            const stat = _gracefulfs.default.statSync(_path.default.join(this.options.root, file), {
+                bigint: true
+            });
+            _class_private_field_get(this, _store)[file] = {
+                mtime: stat.mtimeMs,
+                size: Number(stat.size),
+                hash
+            };
+            hashes[file] = hash;
+        }
+        return hashes;
+    }
+    constructor(options){
+        _define_property(this, "options", void 0);
+        _class_private_field_init(this, _store, {
+            writable: true,
+            value: void 0
+        });
+        _class_private_field_init(this, _manifestFile, {
+            writable: true,
+            value: void 0
+        });
+        this.options = options;
+        _class_private_field_set(this, _store, {});
+        const { root  } = options;
+        const cacheDirectory = _path.default.join(root, "node_modules", ".cache", "lage");
+        _class_private_field_set(this, _manifestFile, _path.default.join(cacheDirectory, "file_hashes.manifest"));
+    }
+}

package/lib/PackageTree.d.ts

@@ -0,0 +1,20 @@
+import { type PackageInfos } from "workspace-tools";
+export interface PackageTreeOptions {
+    root: string;
+    packageInfos: PackageInfos;
+    includeUntracked: boolean;
+}
+/**
+ * Package Tree keeps a data structure to quickly find all files in a package.
+ *
+ * TODO: add a watcher to make sure the tree is up to date during a "watched" run.
+ */
+export declare class PackageTree {
+    #private;
+    private options;
+    constructor(options: PackageTreeOptions);
+    reset(): void;
+    initialize(): Promise<void>;
+    addToPackageTree(filePaths: string[]): Promise<void>;
+    getPackageFiles(packageName: string, patterns: string[]): string[];
+}

package/lib/PackageTree.js

@@ -0,0 +1,179 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "PackageTree", {
+    enumerable: true,
+    get: function() {
+        return PackageTree;
+    }
+});
+const _execa = /*#__PURE__*/ _interop_require_default(require("execa"));
+const _path = /*#__PURE__*/ _interop_require_default(require("path"));
+const _fs = /*#__PURE__*/ _interop_require_default(require("fs"));
+const _micromatch = /*#__PURE__*/ _interop_require_default(require("micromatch"));
+function _check_private_redeclaration(obj, privateCollection) {
+    if (privateCollection.has(obj)) {
+        throw new TypeError("Cannot initialize the same private elements twice on an object");
+    }
+}
+function _class_apply_descriptor_get(receiver, descriptor) {
+    if (descriptor.get) {
+        return descriptor.get.call(receiver);
+    }
+    return descriptor.value;
+}
+function _class_apply_descriptor_set(receiver, descriptor, value) {
+    if (descriptor.set) {
+        descriptor.set.call(receiver, value);
+    } else {
+        if (!descriptor.writable) {
+            throw new TypeError("attempted to set read only private field");
+        }
+        descriptor.value = value;
+    }
+}
+function _class_extract_field_descriptor(receiver, privateMap, action) {
+    if (!privateMap.has(receiver)) {
+        throw new TypeError("attempted to " + action + " private field on non-instance");
+    }
+    return privateMap.get(receiver);
+}
+function _class_private_field_get(receiver, privateMap) {
+    var descriptor = _class_extract_field_descriptor(receiver, privateMap, "get");
+    return _class_apply_descriptor_get(receiver, descriptor);
+}
+function _class_private_field_init(obj, privateMap, value) {
+    _check_private_redeclaration(obj, privateMap);
+    privateMap.set(obj, value);
+}
+function _class_private_field_set(receiver, privateMap, value) {
+    var descriptor = _class_extract_field_descriptor(receiver, privateMap, "set");
+    _class_apply_descriptor_set(receiver, descriptor, value);
+    return value;
+}
+function _define_property(obj, key, value) {
+    if (key in obj) {
+        Object.defineProperty(obj, key, {
+            value: value,
+            enumerable: true,
+            configurable: true,
+            writable: true
+        });
+    } else {
+        obj[key] = value;
+    }
+    return obj;
+}
+function _interop_require_default(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
+var _tree = /*#__PURE__*/ new WeakMap(), _packageFiles = /*#__PURE__*/ new WeakMap(), _memoizedPackageFiles = /*#__PURE__*/ new WeakMap();
+class PackageTree {
+    reset() {
+        _class_private_field_set(this, _tree, {});
+        _class_private_field_set(this, _packageFiles, {});
+        _class_private_field_set(this, _memoizedPackageFiles, {});
+    }
+    async initialize() {
+        const { root , includeUntracked , packageInfos  } = this.options;
+        this.reset();
+        // Generate path tree of all packages in workspace (scale: ~2000 * ~3)
+        for (const info of Object.values(packageInfos)){
+            const packagePath = _path.default.dirname(info.packageJsonPath);
+            const pathParts = _path.default.relative(root, packagePath).split(/[\\/]/);
+            let currentNode = _class_private_field_get(this, _tree);
+            for (const part of pathParts){
+                currentNode[part] = currentNode[part] || {};
+                currentNode = currentNode[part];
+            }
+        }
+        // Get all files in the workspace (scale: ~2000) according to git
+        const lsFilesResults = await (0, _execa.default)("git", [
+            "ls-files",
+            "-z"
+        ], {
+            cwd: root
+        });
+        if (lsFilesResults.exitCode === 0) {
+            const files = lsFilesResults.stdout.split("\0").filter((f)=>Boolean(f) && _fs.default.existsSync(_path.default.join(root, f)));
+            this.addToPackageTree(files);
+        }
+        if (includeUntracked) {
+            // Also get all untracked files in the workspace according to git
+            const lsOtherResults = await (0, _execa.default)("git", [
+                "ls-files",
+                "-o",
+                "--exclude-standard"
+            ], {
+                cwd: root
+            });
+            if (lsOtherResults.exitCode === 0) {
+                const files = lsOtherResults.stdout.split("\0").filter(Boolean);
+                this.addToPackageTree(files);
+            }
+        }
+    }
+    async addToPackageTree(filePaths) {
+        // key: path/to/package (packageRoot), value: array of a tuple of [file, hash]
+        const packageFiles = _class_private_field_get(this, _packageFiles);
+        for (const entry of filePaths){
+            const pathParts = entry.split(/[\\/]/);
+            let node = _class_private_field_get(this, _tree);
+            const packagePathParts = [];
+            for (const part of pathParts){
+                if (node[part]) {
+                    node = node[part];
+                    packagePathParts.push(part);
+                } else {
+                    break;
+                }
+            }
+            const packageRoot = packagePathParts.join("/");
+            packageFiles[packageRoot] = packageFiles[packageRoot] || [];
+            packageFiles[packageRoot].push(entry);
+        }
+    }
+    getPackageFiles(packageName, patterns) {
+        const { root , packageInfos  } = this.options;
+        const packagePath = _path.default.relative(root, _path.default.dirname(packageInfos[packageName].packageJsonPath)).replace(/\\/g, "/");
+        const packageFiles = _class_private_field_get(this, _packageFiles)[packagePath];
+        if (!packageFiles) {
+            return [];
+        }
+        const key = `${packageName}\0${patterns.join("\0")}`;
+        if (!_class_private_field_get(this, _memoizedPackageFiles)[key]) {
+            const packagePatterns = patterns.map((pattern)=>{
+                if (pattern.startsWith("!")) {
+                    return `!${_path.default.join(packagePath, pattern.slice(1)).replace(/\\/g, "/")}`;
+                }
+                return _path.default.join(packagePath, pattern).replace(/\\/g, "/");
+            });
+            _class_private_field_get(this, _memoizedPackageFiles)[key] = (0, _micromatch.default)(packageFiles, packagePatterns, {
+                dot: true
+            });
+        }
+        return _class_private_field_get(this, _memoizedPackageFiles)[key];
+    }
+    constructor(options){
+        _define_property(this, "options", void 0);
+        _class_private_field_init(this, _tree, {
+            writable: true,
+            value: void 0
+        });
+        _class_private_field_init(this, _packageFiles, {
+            writable: true,
+            value: void 0
+        });
+        _class_private_field_init(this, _memoizedPackageFiles, {
+            writable: true,
+            value: void 0
+        });
+        this.options = options;
+        _class_private_field_set(this, _tree, {});
+        _class_private_field_set(this, _packageFiles, {});
+        _class_private_field_set(this, _memoizedPackageFiles, {});
+    }
+}

package/lib/TargetHasher.d.ts

@@ -0,0 +1,47 @@
+import type { Target } from "@lage-run/target-graph";
+import { type ParsedLock, type WorkspaceInfo, type PackageInfos } from "workspace-tools";
+import type { DependencyMap } from "workspace-tools/lib/graph/createDependencyMap.js";
+import { FileHasher } from "./FileHasher.js";
+import { PackageTree } from "./PackageTree.js";
+export interface TargetHasherOptions {
+    root: string;
+    environmentGlob: string[];
+    cacheKey?: string;
+    cliArgs?: string[];
+}
+export interface TargetManifest {
+    id: string;
+    hash: string;
+    globalInputsHash: string;
+    dependency: Record<string, string>;
+    fileHasher: FileHasher;
+    files: Record<string, {
+        mtimeMs: number;
+        size: number;
+        hash: string;
+    }>;
+}
+/**
+ * TargetHasher is a class that can be used to generate a hash of a target.
+ *
+ * Currently, it encapsulates the use of `backfill-hasher` to generate a hash.
+ */
+export declare class TargetHasher {
+    private options;
+    fileHasher: FileHasher;
+    packageTree: PackageTree | undefined;
+    initializedPromise: Promise<unknown> | undefined;
+    packageInfos: PackageInfos;
+    workspaceInfo: WorkspaceInfo | undefined;
+    globalInputsHash: Record<string, string> | undefined;
+    lockInfo: ParsedLock | undefined;
+    targetHashes: Record<string, string>;
+    dependencyMap: DependencyMap;
+    getPackageInfos(workspacePackages: WorkspaceInfo): PackageInfos;
+    expandInputPatterns(patterns: string[], target: Target): Record<string, string[]>;
+    constructor(options: TargetHasherOptions);
+    ensureInitialized(): void;
+    initialize(): Promise<void>;
+    hash(target: Target): Promise<string>;
+    cleanup(): Promise<void>;
+}