@lage-run/cli 0.21.0 → 0.22.0

package/CHANGELOG.json

@@ -2,7 +2,52 @@
   "name": "@lage-run/cli",
   "entries": [
     {
-      "date": "Wed, 25 Sep 2024 20:27:33 GMT",
+      "date": "Fri, 27 Sep 2024 20:03:27 GMT",
+      "version": "0.22.0",
+      "tag": "@lage-run/cli_v0.22.0",
+      "comments": {
+        "minor": [
+          {
+            "author": "kchau@microsoft.com",
+            "package": "@lage-run/cli",
+            "commit": "2919f9041f931dc6ef65017f7aedb9fef9dab66d",
+            "comment": "Simulate file access for lage server"
+          },
+          {
+            "author": "beachball",
+            "package": "@lage-run/cli",
+            "comment": "Bump @lage-run/globby to v14.1.0",
+            "commit": "not available"
+          },
+          {
+            "author": "beachball",
+            "package": "@lage-run/cli",
+            "comment": "Bump @lage-run/hasher to v1.5.0",
+            "commit": "not available"
+          },
+          {
+            "author": "beachball",
+            "package": "@lage-run/cli",
+            "comment": "Bump @lage-run/rpc to v1.2.1",
+            "commit": "not available"
+          },
+          {
+            "author": "beachball",
+            "package": "@lage-run/cli",
+            "comment": "Bump @lage-run/scheduler to v1.3.1",
+            "commit": "not available"
+          },
+          {
+            "author": "beachball",
+            "package": "@lage-run/cli",
+            "comment": "Bump @lage-run/worker-threads-pool to v0.8.2",
+            "commit": "not available"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Wed, 25 Sep 2024 20:28:10 GMT",
       "version": "0.21.0",
       "tag": "@lage-run/cli_v0.21.0",
       "comments": {

package/CHANGELOG.md

@@ -1,12 +1,25 @@
 # Change Log - @lage-run/cli
 
-<!-- This log was last generated on Wed, 25 Sep 2024 20:27:33 GMT and should not be manually modified. -->
+<!-- This log was last generated on Fri, 27 Sep 2024 20:03:27 GMT and should not be manually modified. -->
 
 <!-- Start content -->
 
+## 0.22.0
+
+Fri, 27 Sep 2024 20:03:27 GMT
+
+### Minor changes
+
+- Simulate file access for lage server (kchau@microsoft.com)
+- Bump @lage-run/globby to v14.1.0
+- Bump @lage-run/hasher to v1.5.0
+- Bump @lage-run/rpc to v1.2.1
+- Bump @lage-run/scheduler to v1.3.1
+- Bump @lage-run/worker-threads-pool to v0.8.2
+
 ## 0.21.0
 
-Wed, 25 Sep 2024 20:27:33 GMT
+Wed, 25 Sep 2024 20:28:10 GMT
 
 ### Minor changes
 

package/lib/commands/run/createTargetGraph.d.ts

@@ -7,7 +7,7 @@ interface CreateTargetGraphOptions {
     dependencies: boolean;
     dependents: boolean;
     since: string;
-    scope: string[];
+    scope?: string[];
     ignore: string[];
     repoWideChanges: string[];
     pipeline: PipelineDefinition;

package/lib/commands/server/action.js

@@ -13,6 +13,7 @@ const _initializeReporters = require("../initializeReporters.js");
 const _lageService = require("./lageService.js");
 const _filterArgsForTasks = require("../run/filterArgsForTasks.js");
 const _rpc = require("@lage-run/rpc");
+const _simulateFileAccess = require("./simulateFileAccess.js");
 function _interop_require_default(obj) {
     return obj && obj.__esModule ? obj : {
         default: obj
@@ -48,8 +49,12 @@ async function executeOnServer(args, client, logger) {
         task,
         taskArgs
     });
-    logger.info(`Task ${response.packageName} #${response.task} exited with code ${response.exitCode} `);
+    logger.info(`Task ${response.packageName} ${response.task} exited with code ${response.exitCode} `);
     process.exitCode = response.exitCode;
+    if (response.exitCode === 0) {
+        await (0, _simulateFileAccess.simulateFileAccess)(logger, response.inputs, response.outputs);
+    }
+    logger.info("Task execution finished");
 }
 async function serverAction(options, command) {
     const { port =5332 , host ="localhost" , timeout =1  } = options;
@@ -65,12 +70,17 @@ async function serverAction(options, command) {
     } else {
         logger.info(`Starting server on http://${host}:${port}`);
         const abortController = new AbortController();
-        const lageService = await (0, _lageService.createLageService)(process.cwd(), abortController, logger, options.concurrency);
-        const server = await (0, _rpc.createServer)(lageService, abortController);
-        server.addHook("onRequest", (req, res, next)=>{
-            resetTimer(logger, timeout, abortController, server);
-            next();
+        const lageService = await (0, _lageService.createLageService)({
+            cwd: process.cwd(),
+            serverControls: {
+                abortController,
+                countdownToShutdown: ()=>resetTimer(logger, timeout, abortController, server),
+                clearCountdown: clearTimer
+            },
+            logger,
+            maxWorkers: options.concurrency
         });
+        const server = await (0, _rpc.createServer)(lageService, abortController);
         await server.listen({
             host,
             port
@@ -86,12 +96,15 @@ async function serverAction(options, command) {
 }
 let timeoutHandle;
 function resetTimer(logger, timeout, abortController, server) {
-    if (timeoutHandle) {
-        clearTimeout(timeoutHandle);
-    }
-    timeoutHandle = setTimeout(()=>{
+    clearTimer();
+    timeoutHandle = globalThis.setTimeout(()=>{
         logger.info(`Server timed out after ${timeout} seconds`);
         abortController.abort();
         server.close();
     }, timeout * 1000);
 }
+function clearTimer() {
+    if (timeoutHandle) {
+        globalThis.clearTimeout(timeoutHandle);
+    }
+}

package/lib/commands/server/getOutputFiles.d.ts

@@ -0,0 +1,4 @@
+import type { CacheOptions } from "@lage-run/config";
+import type { PackageTree } from "@lage-run/hasher";
+import type { Target } from "@lage-run/target-graph";
+export declare function getOutputFiles(root: string, target: Target, outputGlob: CacheOptions["outputGlob"], packageTree: PackageTree): string[];

package/lib/commands/server/getOutputFiles.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "getOutputFiles", {
+    enumerable: true,
+    get: function() {
+        return getOutputFiles;
+    }
+});
+const _globby = require("@lage-run/globby");
+const _path = /*#__PURE__*/ _interop_require_default(require("path"));
+function _interop_require_default(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
+function getOutputFiles(root, target, outputGlob, packageTree) {
+    const patterns = target.outputs ?? outputGlob ?? [
+        "**/*"
+    ];
+    const sourceControlledFiles = new Set(packageTree.getPackageFiles(target.packageName ?? "", patterns));
+    const outputs = (0, _globby.glob)(patterns, {
+        cwd: target.cwd,
+        gitignore: false
+    }).map((file)=>_path.default.relative(root, _path.default.join(target.cwd, file))).filter((file)=>!sourceControlledFiles.has(file));
+    return outputs;
+}

package/lib/commands/server/lageService.d.ts

@@ -1,4 +1,13 @@
 /// <reference types="node" />
 import type { Logger } from "@lage-run/logger";
 import type { ILageService } from "@lage-run/rpc";
-export declare function createLageService(cwd: string, abortController: AbortController, logger: Logger, maxWorkers?: number): Promise<ILageService>;
+export declare function createLageService({ cwd, serverControls, logger, maxWorkers, }: {
+    cwd: string;
+    serverControls: {
+        abortController: AbortController;
+        countdownToShutdown: () => void;
+        clearCountdown: () => void;
+    };
+    logger: Logger;
+    maxWorkers?: number;
+}): Promise<ILageService>;

package/lib/commands/server/lageService.js

@@ -13,6 +13,9 @@ const _targetgraph = require("@lage-run/target-graph");
 const _workspacetools = require("workspace-tools");
 const _createTargetGraph = require("../run/createTargetGraph.js");
 const _workerthreadspool = require("@lage-run/worker-threads-pool");
+const _hasher = require("@lage-run/hasher");
+const _getOutputFiles = require("./getOutputFiles.js");
+const _globby = require("@lage-run/globby");
 function findAllTasks(pipeline) {
     const tasks = new Set();
     for (const key of Object.keys(pipeline)){
@@ -25,18 +28,15 @@ function findAllTasks(pipeline) {
     }
     return Array.from(tasks);
 }
-let targetGraph;
-let config;
+let context;
 async function initializeOnce(cwd, logger) {
-    if (!config) {
-        config = await (0, _config.getConfig)(cwd);
-    }
-    if (!targetGraph) {
-        const { pipeline  } = config;
+    if (!context) {
+        const config = await (0, _config.getConfig)(cwd);
         const root = (0, _workspacetools.getWorkspaceRoot)(cwd);
+        const { pipeline  } = config;
         const packageInfos = (0, _workspacetools.getPackageInfos)(root);
         const tasks = findAllTasks(pipeline);
-        targetGraph = (0, _createTargetGraph.createTargetGraph)({
+        const targetGraph = (0, _createTargetGraph.createTargetGraph)({
             logger,
             root,
             dependencies: false,
@@ -44,28 +44,46 @@ async function initializeOnce(cwd, logger) {
             ignore: [],
             pipeline,
             repoWideChanges: config.repoWideChanges,
-            scope: [],
+            scope: undefined,
             since: "",
             outputs: config.cacheOptions.outputGlob,
             tasks,
             packageInfos
         });
+        const dependencyMap = (0, _workspacetools.createDependencyMap)(packageInfos, {
+            withDevDependencies: true,
+            withPeerDependencies: false
+        });
+        const packageTree = new _hasher.PackageTree({
+            root,
+            packageInfos,
+            includeUntracked: true
+        });
+        await packageTree.initialize();
+        context = {
+            config,
+            targetGraph,
+            packageTree,
+            dependencyMap,
+            root
+        };
     }
-    return {
-        config,
-        targetGraph
-    };
+    return context;
 }
 let pool;
-async function createLageService(cwd, abortController, logger, maxWorkers) {
+async function createLageService({ cwd , serverControls , logger , maxWorkers  }) {
     logger.info(`Server started with ${maxWorkers} workers`);
     pool = new _workerthreadspool.WorkerPool({
         script: require.resolve("./singleTargetWorker.js"),
         maxWorkers
     });
-    abortController.signal.addEventListener("abort", ()=>{
+    serverControls.abortController.signal.addEventListener("abort", ()=>{
         pool?.close();
     });
+    pool?.on("idle", ()=>{
+        logger.info("All workers are idle, shutting down after timeout");
+        serverControls.countdownToShutdown();
+    });
     return {
         async ping () {
             return {
@@ -73,7 +91,8 @@ async function createLageService(cwd, abortController, logger, maxWorkers) {
             };
         },
         async runTarget (request) {
-            const { config , targetGraph  } = await initializeOnce(cwd, logger);
+            serverControls.clearCountdown();
+            const { config , targetGraph , dependencyMap , packageTree , root  } = await initializeOnce(cwd, logger);
             const runners = {
                 npmScript: {
                     script: require.resolve("../run/runners/NpmScriptRunner.js"),
@@ -94,7 +113,8 @@ async function createLageService(cwd, abortController, logger, maxWorkers) {
                     options: {}
                 }
             };
-            if (!targetGraph.targets.has((0, _targetgraph.getTargetId)(request.packageName, request.task))) {
+            const id = (0, _targetgraph.getTargetId)(request.packageName, request.task);
+            if (!targetGraph.targets.has(id)) {
                 logger.error(`Target not found: ${request.packageName}#${request.task}`);
                 return {
                     packageName: request.packageName,
@@ -102,7 +122,7 @@ async function createLageService(cwd, abortController, logger, maxWorkers) {
                     exitCode: 1
                 };
             }
-            const target = targetGraph.targets.get((0, _targetgraph.getTargetId)(request.packageName, request.task));
+            const target = targetGraph.targets.get(id);
             const task = {
                 target,
                 runners
@@ -121,16 +141,32 @@ async function createLageService(cwd, abortController, logger, maxWorkers) {
                     pipedStdout.unpipe(process.stdout);
                     pipedStderr.unpipe(process.stderr);
                 });
+                const globalInputs = target.environmentGlob ? (0, _globby.glob)(target.environmentGlob, {
+                    cwd: root,
+                    gitignore: true
+                }) : (0, _globby.glob)(config.cacheOptions?.environmentGlob, {
+                    cwd: root,
+                    gitignore: true
+                });
+                const inputs = ((0, _hasher.getInputFiles)(target, dependencyMap, packageTree) ?? []).concat(globalInputs);
                 return {
                     packageName: request.packageName,
                     task: request.task,
-                    exitCode: 0
+                    exitCode: 0,
+                    hash: "",
+                    inputs,
+                    outputs: (0, _getOutputFiles.getOutputFiles)(root, target, config.cacheOptions?.outputGlob, packageTree),
+                    id
                 };
             } catch (e) {
                 return {
                     packageName: request.packageName,
                     task: request.task,
-                    exitCode: 1
+                    exitCode: 1,
+                    hash: "",
+                    inputs: [],
+                    outputs: [],
+                    id
                 };
             }
         }

package/lib/commands/server/simulateFileAccess.d.ts

@@ -0,0 +1,2 @@
+import type { Logger } from "@lage-run/logger";
+export declare function simulateFileAccess(logger: Logger, inputs: string[], outputs: string[]): Promise<void>;

package/lib/commands/server/simulateFileAccess.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "simulateFileAccess", {
+    enumerable: true,
+    get: function() {
+        return simulateFileAccess;
+    }
+});
+const _path = /*#__PURE__*/ _interop_require_default(require("path"));
+const _fs = /*#__PURE__*/ _interop_require_default(require("fs"));
+const _workspacetools = require("workspace-tools");
+function _interop_require_default(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
+async function simulateFileAccess(logger, inputs, outputs) {
+    const root = (0, _workspacetools.getWorkspaceRoot)(process.cwd());
+    logger.silly("Now probing and touching inputs and outputs");
+    // probe input files
+    let fd;
+    for (const input of inputs){
+        fd = _fs.default.openSync(_path.default.join(root, input), "r");
+        _fs.default.closeSync(fd);
+    }
+    // touch output files
+    const time = new Date();
+    for (const output of outputs){
+        try {
+            _fs.default.utimesSync(_path.default.join(root, output), time, time);
+        } catch (e) {
+        // ignore
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lage-run/cli",
-  "version": "0.21.0",
+  "version": "0.22.0",
   "description": "Command Line Interface for Lage",
   "repository": {
     "type": "git",
@@ -22,15 +22,16 @@
   "dependencies": {
     "@lage-run/cache": "^1.3.3",
     "@lage-run/config": "^0.4.2",
-    "@lage-run/hasher": "^1.4.0",
+    "@lage-run/globby": "^14.1.0",
+    "@lage-run/hasher": "^1.5.0",
     "@lage-run/logger": "^1.3.1",
     "@lage-run/reporters": "^1.2.10",
-    "@lage-run/rpc": "^1.2.0",
+    "@lage-run/rpc": "^1.2.1",
     "@lage-run/runners": "^1.0.2",
-    "@lage-run/scheduler": "^1.3.0",
+    "@lage-run/scheduler": "^1.3.1",
     "@lage-run/scheduler-types": "^0.3.15",
     "@lage-run/target-graph": "^0.8.10",
-    "@lage-run/worker-threads-pool": "^0.8.1",
+    "@lage-run/worker-threads-pool": "^0.8.2",
     "chokidar": "3.5.3",
     "commander": "9.5.0",
     "execa": "5.1.1",

package/lib/commands/server/executeRemotely.d.ts

@@ -1,8 +0,0 @@
-import { type Logger } from "@lage-run/logger";
-interface ExecClientOptions {
-    logger: Logger;
-    server: string;
-    args?: string[];
-}
-export declare function executeRemotely({ logger, server, args }: ExecClientOptions): Promise<void>;
-export {};

package/lib/commands/server/executeRemotely.js

@@ -1,32 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", {
-    value: true
-});
-Object.defineProperty(exports, "executeRemotely", {
-    enumerable: true,
-    get: function() {
-        return executeRemotely;
-    }
-});
-const _filterArgsForTasks = require("../run/filterArgsForTasks.js");
-async function executeRemotely({ logger , server , args  }) {
-    const task = args?.length === 1 ? args?.[0] : args?.[1];
-    const packageName = args?.length ?? 0 > 1 ? args?.[0] : undefined;
-    if (!task) {
-        throw new Error("No task provided");
-    }
-    const { taskArgs  } = (0, _filterArgsForTasks.filterArgsForTasks)(args ?? []);
-    const rpc = (await import("@lage-run/rpc")).default;
-    const baseUrl = server.startsWith("http") ? server : `http://${server}`;
-    const client = rpc.createClient({
-        baseUrl,
-        httpVersion: baseUrl.startsWith("https") ? "2" : "1.1"
-    });
-    const response = await client.runTarget({
-        packageName,
-        task,
-        taskArgs
-    });
-    logger.info(`Task ${response.packageName}#${response.task} exited with code ${response.exitCode}`);
-    process.exitCode = response.exitCode;
-}