@lage-run/scheduler 0.7.2 → 0.8.0

package/CHANGELOG.json

@@ -2,7 +2,43 @@
   "name": "@lage-run/scheduler",
   "entries": [
     {
-      "date": "Thu, 08 Dec 2022 00:49:16 GMT",
+      "date": "Wed, 18 Jan 2023 18:18:13 GMT",
+      "tag": "@lage-run/scheduler_v0.8.0",
+      "version": "0.8.0",
+      "comments": {
+        "minor": [
+          {
+            "author": "kchau@microsoft.com",
+            "package": "@lage-run/scheduler",
+            "commit": "71b5b1452ee25148a637894d1bf99cf1cc6a80a7",
+            "comment": "Using lage for prune and clear on cache and get reporter in shape for ADO"
+          },
+          {
+            "author": "beachball",
+            "package": "@lage-run/scheduler",
+            "comment": "Bump @lage-run/scheduler-types to v0.3.0",
+            "commit": "71b5b1452ee25148a637894d1bf99cf1cc6a80a7"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Thu, 05 Jan 2023 00:40:50 GMT",
+      "tag": "@lage-run/scheduler_v0.7.3",
+      "version": "0.7.3",
+      "comments": {
+        "patch": [
+          {
+            "author": "kchau@microsoft.com",
+            "package": "@lage-run/scheduler",
+            "commit": "961843bef658c51312e02498554ad338e467e1a7",
+            "comment": "fixing progress bar to not be a bottleneck"
+          }
+        ]
+      }
+    },
+    {
+      "date": "Thu, 08 Dec 2022 00:49:28 GMT",
       "tag": "@lage-run/scheduler_v0.7.2",
       "version": "0.7.2",
       "comments": {

package/CHANGELOG.md

@@ -1,12 +1,29 @@
 # Change Log - @lage-run/scheduler
 
-This log was last generated on Thu, 08 Dec 2022 00:49:16 GMT and should not be manually modified.
+This log was last generated on Wed, 18 Jan 2023 18:18:13 GMT and should not be manually modified.
 
 <!-- Start content -->
 
+## 0.8.0
+
+Wed, 18 Jan 2023 18:18:13 GMT
+
+### Minor changes
+
+- Using lage for prune and clear on cache and get reporter in shape for ADO (kchau@microsoft.com)
+- Bump @lage-run/scheduler-types to v0.3.0
+
+## 0.7.3
+
+Thu, 05 Jan 2023 00:40:50 GMT
+
+### Patches
+
+- fixing progress bar to not be a bottleneck (kchau@microsoft.com)
+
 ## 0.7.2
 
-Thu, 08 Dec 2022 00:49:16 GMT
+Thu, 08 Dec 2022 00:49:28 GMT
 
 ### Patches
 

package/lib/SimpleScheduler.d.ts

@@ -1,6 +1,7 @@
 /// <reference types="node" />
 /// <reference types="global" />
 import { WrappedTarget } from "./WrappedTarget.js";
+import { TargetRunnerPicker } from "./runners/TargetRunnerPicker.js";
 import type { CacheProvider, TargetHasher } from "@lage-run/cache";
 import type { Logger } from "@lage-run/logger";
 import type { TargetGraph } from "@lage-run/target-graph";
@@ -40,7 +41,7 @@ export declare class SimpleScheduler implements TargetScheduler {
     abortController: AbortController;
     abortSignal: AbortSignal;
     pool: Pool;
-    workerIds: number[];
+    runnerPicker: TargetRunnerPicker;
     runPromise: Promise<any>;
     constructor(options: SimpleSchedulerOptions);
     getTargetsByPriority(): import("@lage-run/target-graph").Target[];

package/lib/SimpleScheduler.js

@@ -11,6 +11,7 @@ const _formatBytesJs = require("./formatBytes.js");
 const _categorizeTargetRunsJs = require("./categorizeTargetRuns.js");
 const _targetGraph = require("@lage-run/target-graph");
 const _wrappedTargetJs = require("./WrappedTarget.js");
+const _targetRunnerPickerJs = require("./runners/TargetRunnerPicker.js");
 function _checkPrivateRedeclaration(obj, privateCollection) {
     if (privateCollection.has(obj)) {
         throw new TypeError("Cannot initialize the same private elements twice on an object");
@@ -136,23 +137,29 @@ class SimpleScheduler {
         }
     }
     getReadyTargets() {
-        const readyTargets = [];
+        const readyTargets = new Set();
         for (const target of this.getTargetsByPriority()){
+            // Skip the start target
             if (target.id === (0, _targetGraph.getStartTargetId)()) {
                 continue;
             }
             const targetRun = this.targetRuns.get(target.id);
-            const targetDeps = targetRun.target.dependencies;
-            // filter all dependencies for those that are "ready"
-            const ready = targetDeps.every((dep)=>{
-                const fromTarget = this.targetRuns.get(dep);
-                return fromTarget.successful || dep === (0, _targetGraph.getStartTargetId)();
-            });
-            if (ready && targetRun.status === "pending") {
-                readyTargets.push(targetRun);
+            // If the target is already running, then we can't run it again
+            if (targetRun.status === "pending") {
+                const targetDeps = targetRun.target.dependencies;
+                // Target is only ready when all its deps are "successful" or that it is a root target
+                const ready = targetDeps.every((dep)=>{
+                    const fromTarget = this.targetRuns.get(dep);
+                    return fromTarget.successful || dep === (0, _targetGraph.getStartTargetId)();
+                });
+                if (ready) {
+                    readyTargets.add(targetRun);
+                }
             }
         }
-        return readyTargets;
+        return [
+            ...readyTargets
+        ];
     }
     isAllDone() {
         for (const t of this.targetRuns.values()){
@@ -220,23 +227,33 @@ class SimpleScheduler {
             },
             workerIdleMemoryLimit: options.workerIdleMemoryLimit
         });
-        this.workerIds = Array(options.concurrency).fill(0).map((_, idx)=>idx + 1);
+        this.runnerPicker = new _targetRunnerPickerJs.TargetRunnerPicker(options.runners);
     }
 }
 async function generateTargetRunPromise(target) {
     let runError;
-    const threadId = this.workerIds.shift();
-    target.threadId = threadId;
     if (target.result && target.successful && !this.rerunTargets.has(target.target.id)) {
         await target.result;
     } else {
         // This do-while loop only runs again if something causes this target to rerun (asynchronously triggering a re-run)
         do {
             this.rerunTargets.delete(target.target.id);
+            target.onQueued();
             try {
-                await target.run();
-            } catch (e) {
-                runError = e;
+                let shouldRun = true;
+                try {
+                    const runner = await this.runnerPicker.pick(target.target);
+                    shouldRun = await runner.shouldRun(target.target);
+                } catch (e) {
+                // pass - default to run anyway
+                }
+                if (shouldRun) {
+                    await target.run();
+                } else {
+                    target.status = "skipped";
+                }
+            } catch (e1) {
+                runError = e1;
             }
         }while (this.rerunTargets.has(target.target.id))
         // if a continue option is set, this merely records what errors have been encountered
@@ -248,8 +265,6 @@ async function generateTargetRunPromise(target) {
             }
         }
     }
-    this.workerIds.unshift(threadId);
-    this.workerIds.sort();
     this.logProgress();
     // finally do another round of scheduling to run next round of targets
     await this.scheduleReadyTargets();

package/lib/WrappedTarget.d.ts

@@ -43,7 +43,7 @@ export declare class WrappedTarget implements TargetRun {
     constructor(options: WrappedTargetOptions);
     onQueued(): void;
     onAbort(): void;
-    onStart(): void;
+    onStart(threadId: number): void;
     onComplete(): void;
     onFail(): void;
     onSkipped(hash: string | null): void;

package/lib/WrappedTarget.js

@@ -37,27 +37,27 @@ class WrappedTarget {
     onAbort() {
         this.status = "aborted";
         this.duration = process.hrtime(this.startTime);
-        this.options.logger.info("aborted", {
+        this.options.logger.info("", {
             target: this.target,
             status: "aborted",
             threadId: this.threadId
         });
     }
-    onStart() {
+    onStart(threadId) {
         if (this.status !== "running") {
             this.status = "running";
             this.startTime = process.hrtime();
-            this.options.logger.info("running", {
+            this.options.logger.info("", {
                 target: this.target,
                 status: "running",
-                threadId: this.threadId
+                threadId
             });
         }
     }
     onComplete() {
         this.status = "success";
         this.duration = process.hrtime(this.startTime);
-        this.options.logger.info("success", {
+        this.options.logger.info("", {
             target: this.target,
             status: "success",
             duration: this.duration,
@@ -67,7 +67,7 @@ class WrappedTarget {
     onFail() {
         this.status = "failed";
         this.duration = process.hrtime(this.startTime);
-        this.options.logger.info("failed", {
+        this.options.logger.info("", {
             target: this.target,
             status: "failed",
             duration: this.duration,
@@ -80,7 +80,7 @@ class WrappedTarget {
     onSkipped(hash) {
         this.status = "skipped";
         this.duration = process.hrtime(this.startTime);
-        this.options.logger.info(`skipped`, {
+        this.options.logger.info("", {
             target: this.target,
             status: "skipped",
             duration: this.duration,
@@ -120,10 +120,9 @@ class WrappedTarget {
     }
     async run() {
         const { target , logger , shouldCache , abortController  } = this.options;
-        this.onQueued();
         const abortSignal = abortController.signal;
         if (abortSignal.aborted) {
-            this.onStart();
+            this.onStart(0);
             this.onAbort();
             return;
         }
@@ -135,7 +134,7 @@ class WrappedTarget {
                 logger.verbose(`hash: ${hash}, cache hit? ${cacheHit}`, {
                     target
                 });
-                this.onStart();
+                this.onStart(0);
                 const cachedOutputFile = (0, _getLageOutputCacheLocationJs.getLageOutputCacheLocation)(this.target, hash ?? "");
                 if (_fs.default.existsSync(cachedOutputFile)) {
                     const cachedOutput = _fs.default.createReadStream(cachedOutputFile, "utf8");
@@ -192,13 +191,14 @@ class WrappedTarget {
         const bufferStderr = (0, _bufferTransformJs.bufferTransform)();
         this.result = pool.exec({
             target
-        }, target.weight ?? 1, (_worker, stdout, stderr)=>{
-            this.onStart();
+        }, target.weight ?? 1, (worker, stdout, stderr)=>{
+            const threadId = worker.threadId;
+            this.onStart(threadId);
             stdout.pipe(bufferStdout.transform);
             stderr.pipe(bufferStderr.transform);
             const releaseStdoutStream = logger.stream(_logger.LogLevel.verbose, stdout, {
                 target,
-                threadId: this.threadId
+                threadId
             });
             releaseStdout = ()=>{
                 releaseStdoutStream();
@@ -206,7 +206,7 @@ class WrappedTarget {
             };
             const releaseStderrStream = logger.stream(_logger.LogLevel.verbose, stderr, {
                 target,
-                threadId: this.threadId
+                threadId
             });
             releaseStderr = ()=>{
                 releaseStderrStream();

package/lib/runners/NoOpRunner.js

@@ -7,6 +7,9 @@ Object.defineProperty(exports, "NoOpRunner", {
     get: ()=>NoOpRunner
 });
 const NoOpRunner = {
+    async shouldRun () {
+        return true;
+    },
     async run () {
     // pass
     }

package/lib/runners/NpmScriptRunner.d.ts

@@ -1,4 +1,5 @@
 import type { TargetRunner, TargetRunnerOptions } from "@lage-run/scheduler-types";
+import type { Target } from "@lage-run/target-graph";
 export interface NpmScriptRunnerOptions {
     taskArgs: string[];
     nodeOptions: string;
@@ -29,5 +30,6 @@ export declare class NpmScriptRunner implements TargetRunner {
     private getNpmArgs;
     private hasNpmScript;
     private validateOptions;
+    shouldRun(target: Target): Promise<boolean>;
     run(runOptions: TargetRunnerOptions): Promise<void>;
 }

package/lib/runners/NpmScriptRunner.js

@@ -26,22 +26,22 @@ class NpmScriptRunner {
         const task = target.options?.script ?? target.task;
         const packageJsonPath = (0, _path.join)(target.cwd, "package.json");
         const packageJson = JSON.parse(await (0, _promises.readFile)(packageJsonPath, "utf8"));
-        return packageJson.scripts?.[task];
+        return !!packageJson.scripts?.[task];
     }
     validateOptions(options) {
         if (!(0, _fs.existsSync)(options.npmCmd)) {
             throw new Error(`NPM Script Runner: ${this.options.npmCmd} does not exist`);
         }
     }
+    async shouldRun(target) {
+        // By convention, do not run anything if there is no script for this task defined in package.json (counts as "success")
+        return await this.hasNpmScript(target);
+    }
     async run(runOptions) {
         const { target , weight , abortSignal  } = runOptions;
         const { nodeOptions , npmCmd , taskArgs  } = this.options;
         const task = target.options?.script ?? target.task;
         let childProcess;
-        // By convention, do not run anything if there is no script for this task defined in package.json (counts as "success")
-        if (!await this.hasNpmScript(target)) {
-            return;
-        }
         /**
      * Handling abort signal from the abort controller. Gracefully kills the process,
      * will be handled by exit handler separately to resolve the promise.

package/lib/runners/WorkerRunner.d.ts

@@ -1,4 +1,5 @@
 import type { TargetRunner, TargetRunnerOptions } from "@lage-run/scheduler-types";
+import type { Target } from "@lage-run/target-graph";
 export interface WorkerRunnerOptions {
     taskArgs: string[];
 }
@@ -41,5 +42,7 @@ export declare class WorkerRunner implements TargetRunner {
     private options;
     static gracefulKillTimeout: number;
     constructor(options: WorkerRunnerOptions);
+    shouldRun(target: Target): Promise<boolean>;
     run(runOptions: TargetRunnerOptions): Promise<void>;
+    getScriptModule(target: Target): Promise<any>;
 }

package/lib/runners/WorkerRunner.js

@@ -8,19 +8,18 @@ Object.defineProperty(exports, "WorkerRunner", {
 });
 const _url = require("url");
 class WorkerRunner {
+    async shouldRun(target) {
+        const scriptModule = await this.getScriptModule(target);
+        if (typeof scriptModule.shouldRun === "function") {
+            return await scriptModule.shouldRun(target);
+        }
+        return true;
+    }
     async run(runOptions) {
         const { target , weight , abortSignal  } = runOptions;
         const { taskArgs  } = this.options;
-        const scriptFile = target.options?.worker ?? target.options?.script;
-        if (!scriptFile) {
-            throw new Error('WorkerRunner: "script" configuration is required - e.g. { type: "worker", script: "./worker.js" }');
-        }
-        let importScript = scriptFile;
-        if (!importScript.startsWith("file://")) {
-            importScript = (0, _url.pathToFileURL)(importScript).toString();
-        }
-        const scriptModule = await import(importScript);
-        const runFn = typeof scriptModule.default === "function" ? scriptModule.default : scriptModule;
+        const scriptModule = await this.getScriptModule(target);
+        const runFn = typeof scriptModule.run === "function" ? scriptModule.run : typeof scriptModule.default === "function" ? scriptModule.default : scriptModule;
         if (typeof runFn !== "function") {
             throw new Error("WorkerRunner: worker script must export a function; you likely need to use `module.exports = function() {...}`");
         }
@@ -31,6 +30,17 @@ class WorkerRunner {
             abortSignal
         });
     }
+    async getScriptModule(target) {
+        const scriptFile = target.options?.worker ?? target.options?.script;
+        if (!scriptFile) {
+            throw new Error('WorkerRunner: "script" configuration is required - e.g. { type: "worker", script: "./worker.js" }');
+        }
+        let importScript = scriptFile;
+        if (!importScript.startsWith("file://")) {
+            importScript = (0, _url.pathToFileURL)(importScript).toString();
+        }
+        return await import(importScript);
+    }
     constructor(options){
         this.options = options;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lage-run/scheduler",
-  "version": "0.7.2",
+  "version": "0.8.0",
   "description": "Scheduler for Lage",
   "repository": {
     "url": "https://github.com/microsoft/lage"
@@ -21,7 +21,7 @@
     "@lage-run/worker-threads-pool": "^0.5.0"
   },
   "devDependencies": {
-    "@lage-run/scheduler-types": "^0.2.10",
+    "@lage-run/scheduler-types": "^0.3.0",
     "monorepo-scripts": "*"
   },
   "publishConfig": {