@lage-run/scheduler 0.11.4 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.json CHANGED
@@ -2,7 +2,58 @@
2
2
  "name": "@lage-run/scheduler",
3
3
  "entries": [
4
4
  {
5
- "date": "Wed, 26 Apr 2023 04:56:07 GMT",
5
+ "date": "Mon, 08 May 2023 22:26:52 GMT",
6
+ "tag": "@lage-run/scheduler_v1.0.0",
7
+ "version": "1.0.0",
8
+ "comments": {
9
+ "major": [
10
+ {
11
+ "author": "kchau@microsoft.com",
12
+ "package": "@lage-run/scheduler",
13
+ "commit": "5a132808f166179bc316a279c9e11a13d3a39103",
14
+ "comment": "Now takes in a TargetHasher as a dep, but retains its own calls to cache provider"
15
+ },
16
+ {
17
+ "author": "beachball",
18
+ "package": "@lage-run/scheduler",
19
+ "comment": "Bump @lage-run/target-graph to v0.8.6",
20
+ "commit": "5a132808f166179bc316a279c9e11a13d3a39103"
21
+ },
22
+ {
23
+ "author": "beachball",
24
+ "package": "@lage-run/scheduler",
25
+ "comment": "Bump @lage-run/cache to v1.1.1",
26
+ "commit": "5a132808f166179bc316a279c9e11a13d3a39103"
27
+ },
28
+ {
29
+ "author": "beachball",
30
+ "package": "@lage-run/scheduler",
31
+ "comment": "Bump @lage-run/config to v0.2.1",
32
+ "commit": "5a132808f166179bc316a279c9e11a13d3a39103"
33
+ },
34
+ {
35
+ "author": "beachball",
36
+ "package": "@lage-run/scheduler",
37
+ "comment": "Bump @lage-run/hasher to v1.0.0",
38
+ "commit": "5a132808f166179bc316a279c9e11a13d3a39103"
39
+ },
40
+ {
41
+ "author": "beachball",
42
+ "package": "@lage-run/scheduler",
43
+ "comment": "Bump @lage-run/worker-threads-pool to v0.7.0",
44
+ "commit": "5a132808f166179bc316a279c9e11a13d3a39103"
45
+ },
46
+ {
47
+ "author": "beachball",
48
+ "package": "@lage-run/scheduler",
49
+ "comment": "Bump @lage-run/scheduler-types to v0.3.9",
50
+ "commit": "5a132808f166179bc316a279c9e11a13d3a39103"
51
+ }
52
+ ]
53
+ }
54
+ },
55
+ {
56
+ "date": "Wed, 26 Apr 2023 04:56:20 GMT",
6
57
  "tag": "@lage-run/scheduler_v0.11.4",
7
58
  "version": "0.11.4",
8
59
  "comments": {
@@ -11,7 +62,7 @@
11
62
  "author": "beachball",
12
63
  "package": "@lage-run/scheduler",
13
64
  "comment": "Bump @lage-run/config to v0.2.0",
14
- "commit": "56910ce2ef67d9d3dc99b7a75df007820e9444c8"
65
+ "commit": "994af55fcc890b448fc2b5cd5e4ba8bca0832997"
15
66
  }
16
67
  ]
17
68
  }
package/CHANGELOG.md CHANGED
@@ -1,12 +1,26 @@
1
1
  # Change Log - @lage-run/scheduler
2
2
 
3
- This log was last generated on Wed, 26 Apr 2023 04:56:07 GMT and should not be manually modified.
3
+ This log was last generated on Mon, 08 May 2023 22:26:52 GMT and should not be manually modified.
4
4
 
5
5
  <!-- Start content -->
6
6
 
7
+ ## 1.0.0
8
+
9
+ Mon, 08 May 2023 22:26:52 GMT
10
+
11
+ ### Major changes
12
+
13
+ - Now takes in a TargetHasher as a dep, but retains its own calls to cache provider (kchau@microsoft.com)
14
+ - Bump @lage-run/target-graph to v0.8.6
15
+ - Bump @lage-run/cache to v1.1.1
16
+ - Bump @lage-run/config to v0.2.1
17
+ - Bump @lage-run/hasher to v1.0.0
18
+ - Bump @lage-run/worker-threads-pool to v0.7.0
19
+ - Bump @lage-run/scheduler-types to v0.3.9
20
+
7
21
  ## 0.11.4
8
22
 
9
- Wed, 26 Apr 2023 04:56:07 GMT
23
+ Wed, 26 Apr 2023 04:56:20 GMT
10
24
 
11
25
  ### Patches
12
26
 
@@ -7,6 +7,7 @@ import type { TargetGraph } from "@lage-run/target-graph";
7
7
  import type { TargetScheduler, SchedulerRunSummary } from "@lage-run/scheduler-types";
8
8
  import type { Pool } from "@lage-run/worker-threads-pool";
9
9
  import type { TargetRunnerPickerOptions } from "@lage-run/scheduler-types";
10
+ import type { TargetHasher } from "@lage-run/hasher";
10
11
  export interface SimpleSchedulerOptions {
11
12
  logger: Logger;
12
13
  concurrency: number;
@@ -22,6 +23,7 @@ export interface SimpleSchedulerOptions {
22
23
  maxWorkersPerTask: Map<string, number>;
23
24
  pool?: Pool;
24
25
  workerIdleMemoryLimit: number;
26
+ hasher: TargetHasher;
25
27
  }
26
28
  /**
27
29
  * Simple scheduler that runs all targets in a promise graph using p-graph library.
@@ -4,33 +4,48 @@ Object.defineProperty(exports, "__esModule", {
4
4
  });
5
5
  Object.defineProperty(exports, "SimpleScheduler", {
6
6
  enumerable: true,
7
- get: ()=>SimpleScheduler
7
+ get: function() {
8
+ return SimpleScheduler;
9
+ }
8
10
  });
9
- const _workerThreadsPool = require("@lage-run/worker-threads-pool");
10
- const _formatBytesJs = require("./formatBytes.js");
11
- const _categorizeTargetRunsJs = require("./categorizeTargetRuns.js");
12
- const _targetGraph = require("@lage-run/target-graph");
13
- const _wrappedTargetJs = require("./WrappedTarget.js");
14
- const _targetRunnerPickerJs = require("./runners/TargetRunnerPicker.js");
15
- function _checkPrivateRedeclaration(obj, privateCollection) {
11
+ const _workerthreadspool = require("@lage-run/worker-threads-pool");
12
+ const _formatBytes = require("./formatBytes.js");
13
+ const _categorizeTargetRuns = require("./categorizeTargetRuns.js");
14
+ const _targetgraph = require("@lage-run/target-graph");
15
+ const _WrappedTarget = require("./WrappedTarget.js");
16
+ const _TargetRunnerPicker = require("./runners/TargetRunnerPicker.js");
17
+ function _check_private_redeclaration(obj, privateCollection) {
16
18
  if (privateCollection.has(obj)) {
17
19
  throw new TypeError("Cannot initialize the same private elements twice on an object");
18
20
  }
19
21
  }
20
- function _classPrivateMethodGet(receiver, privateSet, fn) {
22
+ function _class_private_method_get(receiver, privateSet, fn) {
21
23
  if (!privateSet.has(receiver)) {
22
24
  throw new TypeError("attempted to get private field on non-instance");
23
25
  }
24
26
  return fn;
25
27
  }
26
- function _classPrivateMethodInit(obj, privateSet) {
27
- _checkPrivateRedeclaration(obj, privateSet);
28
+ function _class_private_method_init(obj, privateSet) {
29
+ _check_private_redeclaration(obj, privateSet);
28
30
  privateSet.add(obj);
29
31
  }
32
+ function _define_property(obj, key, value) {
33
+ if (key in obj) {
34
+ Object.defineProperty(obj, key, {
35
+ value: value,
36
+ enumerable: true,
37
+ configurable: true,
38
+ writable: true
39
+ });
40
+ } else {
41
+ obj[key] = value;
42
+ }
43
+ return obj;
44
+ }
30
45
  var _generateTargetRunPromise = /*#__PURE__*/ new WeakSet();
31
46
  class SimpleScheduler {
32
47
  getTargetsByPriority() {
33
- return (0, _targetGraph.sortTargetsByPriority)([
48
+ return (0, _targetgraph.sortTargetsByPriority)([
34
49
  ...this.targetRuns.values()
35
50
  ].map((run)=>run.target));
36
51
  }
@@ -68,17 +83,18 @@ class SimpleScheduler {
68
83
  this.markTargetAndDependentsPending(target.id);
69
84
  }
70
85
  } else {
71
- targetRun = new _wrappedTargetJs.WrappedTarget({
86
+ targetRun = new _WrappedTarget.WrappedTarget({
72
87
  target,
73
88
  root,
74
89
  logger,
75
90
  shouldCache,
76
91
  continueOnError,
77
92
  abortController,
78
- pool
93
+ pool,
94
+ hasher: this.options.hasher
79
95
  });
80
96
  }
81
- if (target.id === (0, _targetGraph.getStartTargetId)()) {
97
+ if (target.id === (0, _targetgraph.getStartTargetId)()) {
82
98
  targetRun.status = "success";
83
99
  }
84
100
  this.targetRuns.set(target.id, targetRun);
@@ -96,12 +112,13 @@ class SimpleScheduler {
96
112
  error = e instanceof Error ? e.message : String(e);
97
113
  } finally{
98
114
  duration = process.hrtime(startTime);
99
- targetRunByStatus = (0, _categorizeTargetRunsJs.categorizeTargetRuns)(this.targetRuns.values());
115
+ targetRunByStatus = (0, _categorizeTargetRuns.categorizeTargetRuns)(this.targetRuns.values());
100
116
  if (targetRunByStatus.failed.length + targetRunByStatus.aborted.length + targetRunByStatus.pending.length + targetRunByStatus.running.length === 0) {
101
117
  results = "success";
102
118
  }
103
119
  }
104
120
  const poolStats = pool.stats();
121
+ await this.options.hasher.cleanup();
105
122
  return {
106
123
  targetRunByStatus,
107
124
  targetRuns: this.targetRuns,
@@ -139,7 +156,7 @@ class SimpleScheduler {
139
156
  const readyTargets = new Set();
140
157
  for (const target of this.getTargetsByPriority()){
141
158
  // Skip the start target
142
- if (target.id === (0, _targetGraph.getStartTargetId)()) {
159
+ if (target.id === (0, _targetgraph.getStartTargetId)()) {
143
160
  continue;
144
161
  }
145
162
  const targetRun = this.targetRuns.get(target.id);
@@ -149,7 +166,7 @@ class SimpleScheduler {
149
166
  // Target is only ready when all its deps are "successful" or that it is a root target
150
167
  const ready = targetDeps.every((dep)=>{
151
168
  const fromTarget = this.targetRuns.get(dep);
152
- return fromTarget.successful || dep === (0, _targetGraph.getStartTargetId)();
169
+ return fromTarget.successful || dep === (0, _targetgraph.getStartTargetId)();
153
170
  });
154
171
  if (ready) {
155
172
  readyTargets.add(targetRun);
@@ -162,7 +179,7 @@ class SimpleScheduler {
162
179
  }
163
180
  isAllDone() {
164
181
  for (const t of this.targetRuns.values()){
165
- if (t.status !== "skipped" && t.status !== "success" && t.target.id !== (0, _targetGraph.getStartTargetId)()) {
182
+ if (t.status !== "skipped" && t.status !== "success" && t.target.id !== (0, _targetgraph.getStartTargetId)()) {
166
183
  return false;
167
184
  }
168
185
  }
@@ -172,16 +189,16 @@ class SimpleScheduler {
172
189
  if (this.isAllDone() || this.abortSignal.aborted) {
173
190
  return Promise.resolve();
174
191
  }
175
- this.options.logger.silly(`Max Worker Memory Usage: ${(0, _formatBytesJs.formatBytes)(this.pool.stats().maxWorkerMemoryUsage)}`);
192
+ this.options.logger.silly(`Max Worker Memory Usage: ${(0, _formatBytes.formatBytes)(this.pool.stats().maxWorkerMemoryUsage)}`);
176
193
  const promises = [];
177
194
  for (const nextTarget of this.getReadyTargets()){
178
- const runPromise = _classPrivateMethodGet(this, _generateTargetRunPromise, generateTargetRunPromise).call(this, nextTarget);
195
+ const runPromise = _class_private_method_get(this, _generateTargetRunPromise, generateTargetRunPromise).call(this, nextTarget);
179
196
  promises.push(runPromise);
180
197
  }
181
198
  await Promise.all(promises);
182
199
  }
183
200
  logProgress() {
184
- const targetRunByStatus = (0, _categorizeTargetRunsJs.categorizeTargetRuns)(this.targetRuns.values());
201
+ const targetRunByStatus = (0, _categorizeTargetRuns.categorizeTargetRuns)(this.targetRuns.values());
185
202
  const total = [
186
203
  ...this.targetRuns.values()
187
204
  ].filter((t)=>!t.target.hidden).length;
@@ -195,7 +212,7 @@ class SimpleScheduler {
195
212
  });
196
213
  }
197
214
  async cleanup() {
198
- this.options.logger.silly(`Max Worker Memory Usage: ${(0, _formatBytesJs.formatBytes)(this.pool.stats().maxWorkerMemoryUsage)}`);
215
+ this.options.logger.silly(`Max Worker Memory Usage: ${(0, _formatBytes.formatBytes)(this.pool.stats().maxWorkerMemoryUsage)}`);
199
216
  await this.pool.close();
200
217
  }
201
218
  /**
@@ -204,14 +221,22 @@ class SimpleScheduler {
204
221
  this.abortController.abort();
205
222
  }
206
223
  constructor(options){
207
- _classPrivateMethodInit(this, _generateTargetRunPromise);
224
+ _class_private_method_init(this, _generateTargetRunPromise);
225
+ _define_property(this, "options", void 0);
226
+ _define_property(this, "targetRuns", void 0);
227
+ _define_property(this, "rerunTargets", void 0);
228
+ _define_property(this, "abortController", void 0);
229
+ _define_property(this, "abortSignal", void 0);
230
+ _define_property(this, "pool", void 0);
231
+ _define_property(this, "runnerPicker", void 0);
232
+ _define_property(this, "runPromise", void 0);
208
233
  this.options = options;
209
234
  this.targetRuns = new Map();
210
235
  this.rerunTargets = new Set();
211
236
  this.abortController = new AbortController();
212
237
  this.abortSignal = this.abortController.signal;
213
238
  this.runPromise = Promise.resolve();
214
- this.pool = options.pool ?? new _workerThreadsPool.AggregatedPool({
239
+ this.pool = options.pool ?? new _workerthreadspool.AggregatedPool({
215
240
  logger: options.logger,
216
241
  maxWorkersByGroup: options.maxWorkersPerTask,
217
242
  groupBy: ({ target })=>target.task,
@@ -228,7 +253,7 @@ class SimpleScheduler {
228
253
  },
229
254
  workerIdleMemoryLimit: options.workerIdleMemoryLimit
230
255
  });
231
- this.runnerPicker = new _targetRunnerPickerJs.TargetRunnerPicker(options.workerData.runners);
256
+ this.runnerPicker = new _TargetRunnerPicker.TargetRunnerPicker(options.workerData.runners);
232
257
  }
233
258
  }
234
259
  async function generateTargetRunPromise(target) {
@@ -3,6 +3,7 @@ import type { Pool } from "@lage-run/worker-threads-pool";
3
3
  import type { TargetRun, TargetStatus } from "@lage-run/scheduler-types";
4
4
  import type { Target } from "@lage-run/target-graph";
5
5
  import type { Logger } from "@lage-run/logger";
6
+ import type { TargetHasher } from "@lage-run/hasher";
6
7
  export interface WrappedTargetOptions {
7
8
  root: string;
8
9
  target: Target;
@@ -11,6 +12,7 @@ export interface WrappedTargetOptions {
11
12
  continueOnError: boolean;
12
13
  abortController: AbortController;
13
14
  pool: Pool;
15
+ hasher: TargetHasher;
14
16
  }
15
17
  interface WorkerResult {
16
18
  stdoutBuffer: string;
@@ -4,15 +4,30 @@ Object.defineProperty(exports, "__esModule", {
4
4
  });
5
5
  Object.defineProperty(exports, "WrappedTarget", {
6
6
  enumerable: true,
7
- get: ()=>WrappedTarget
7
+ get: function() {
8
+ return WrappedTarget;
9
+ }
8
10
  });
9
- const _bufferTransformJs = require("./bufferTransform.js");
10
- const _getLageOutputCacheLocationJs = require("./getLageOutputCacheLocation.js");
11
+ const _bufferTransform = require("./bufferTransform.js");
12
+ const _getLageOutputCacheLocation = require("./getLageOutputCacheLocation.js");
11
13
  const _logger = require("@lage-run/logger");
12
- const _fs = /*#__PURE__*/ _interopRequireDefault(require("fs"));
13
- const _path = /*#__PURE__*/ _interopRequireDefault(require("path"));
14
+ const _fs = /*#__PURE__*/ _interop_require_default(require("fs"));
15
+ const _path = /*#__PURE__*/ _interop_require_default(require("path"));
14
16
  const _promises = require("fs/promises");
15
- function _interopRequireDefault(obj) {
17
+ function _define_property(obj, key, value) {
18
+ if (key in obj) {
19
+ Object.defineProperty(obj, key, {
20
+ value: value,
21
+ enumerable: true,
22
+ configurable: true,
23
+ writable: true
24
+ });
25
+ } else {
26
+ obj[key] = value;
27
+ }
28
+ return obj;
29
+ }
30
+ function _interop_require_default(obj) {
16
31
  return obj && obj.__esModule ? obj : {
17
32
  default: obj
18
33
  };
@@ -94,7 +109,7 @@ class WrappedTarget {
94
109
  }
95
110
  }
96
111
  async run() {
97
- const { target , logger , shouldCache , abortController } = this.options;
112
+ const { target , logger , shouldCache , abortController , root } = this.options;
98
113
  const abortSignal = abortController.signal;
99
114
  if (abortSignal.aborted) {
100
115
  this.onStart(0);
@@ -106,34 +121,29 @@ class WrappedTarget {
106
121
  const cacheEnabled = target.cache && shouldCache && result.hash;
107
122
  // Save output if cache is enabled & cache is hit
108
123
  if (!result.skipped && cacheEnabled) {
109
- const outputLocation = (0, _getLageOutputCacheLocationJs.getLageOutputCacheLocation)(this.options.root, result.hash);
124
+ const outputLocation = (0, _getLageOutputCacheLocation.getLageOutputCacheLocation)(root, result.hash);
110
125
  const outputPath = _path.default.dirname(outputLocation);
111
126
  await (0, _promises.mkdir)(outputPath, {
112
127
  recursive: true
113
128
  });
114
129
  const output = `${result.stdoutBuffer}\n${result.stderrBuffer}`;
115
130
  await (0, _promises.writeFile)(outputLocation, output);
116
- this.options.logger.verbose(`>> Saved cache - ${result.hash}`, {
131
+ logger.verbose(`>> Saved cache - ${result.hash}`, {
117
132
  target
118
133
  });
119
134
  }
120
135
  if (result.skipped) {
121
136
  const { hash } = result;
122
- const cachedOutputFile = (0, _getLageOutputCacheLocationJs.getLageOutputCacheLocation)(this.options.root, hash ?? "");
123
- if (_fs.default.existsSync(cachedOutputFile)) {
124
- const cachedOutput = _fs.default.createReadStream(cachedOutputFile, "utf8");
125
- this.options.logger.verbose(">> Replaying cached output", {
137
+ const cachedOutputFile = (0, _getLageOutputCacheLocation.getLageOutputCacheLocation)(root, hash ?? "");
138
+ const shouldShowCachedOutput = _fs.default.existsSync(cachedOutputFile);
139
+ if (shouldShowCachedOutput) {
140
+ const cachedOutput = _fs.default.readFileSync(cachedOutputFile, "utf8");
141
+ logger.verbose(">> Replaying cached output", {
126
142
  target
127
143
  });
128
- this.options.logger.stream(_logger.LogLevel.verbose, cachedOutput, {
144
+ logger.verbose(cachedOutput.trim(), {
129
145
  target
130
146
  });
131
- return await new Promise((resolve)=>{
132
- cachedOutput.on("close", ()=>{
133
- this.onSkipped(hash);
134
- resolve();
135
- });
136
- });
137
147
  }
138
148
  this.onSkipped(hash);
139
149
  } else {
@@ -156,21 +166,28 @@ class WrappedTarget {
156
166
  const abortSignal = abortController.signal;
157
167
  let releaseStdout;
158
168
  let releaseStderr;
159
- const bufferStdout = (0, _bufferTransformJs.bufferTransform)();
160
- const bufferStderr = (0, _bufferTransformJs.bufferTransform)();
161
- let logHandler;
169
+ const bufferStdout = (0, _bufferTransform.bufferTransform)();
170
+ const bufferStderr = (0, _bufferTransform.bufferTransform)();
171
+ let msgHandler;
162
172
  this.result = pool.exec({
163
173
  target
164
174
  }, target.weight ?? 1, (worker, stdout, stderr)=>{
165
- logHandler = (data)=>{
175
+ msgHandler = (data)=>{
166
176
  if (data.type === "log") {
167
177
  logger.log(data.level, data.msg, {
168
178
  target,
169
179
  threadId: worker.threadId
170
180
  });
181
+ } else if (data.type === "hash") {
182
+ this.options.hasher.hash(target).then((hash)=>{
183
+ worker.postMessage({
184
+ type: "hash",
185
+ hash
186
+ });
187
+ });
171
188
  }
172
189
  };
173
- worker.on("message", logHandler);
190
+ worker.on("message", msgHandler);
174
191
  const threadId = worker.threadId;
175
192
  this.onStart(threadId);
176
193
  stdout.pipe(bufferStdout.transform);
@@ -192,7 +209,7 @@ class WrappedTarget {
192
209
  stderr.unpipe(bufferStderr.transform);
193
210
  };
194
211
  }, (worker)=>{
195
- worker.off("message", logHandler);
212
+ worker.off("message", msgHandler);
196
213
  releaseStdout();
197
214
  releaseStderr();
198
215
  }, abortSignal);
@@ -217,6 +234,14 @@ class WrappedTarget {
217
234
  };
218
235
  }
219
236
  constructor(options){
237
+ _define_property(this, "options", void 0);
238
+ _define_property(this, "queueTime", void 0);
239
+ _define_property(this, "startTime", void 0);
240
+ _define_property(this, "duration", void 0);
241
+ _define_property(this, "target", void 0);
242
+ _define_property(this, "status", void 0);
243
+ _define_property(this, "threadId", void 0);
244
+ _define_property(this, "result", void 0);
220
245
  this.options = options;
221
246
  this.queueTime = [
222
247
  0,
@@ -4,7 +4,9 @@ Object.defineProperty(exports, "__esModule", {
4
4
  });
5
5
  Object.defineProperty(exports, "bufferTransform", {
6
6
  enumerable: true,
7
- get: ()=>bufferTransform
7
+ get: function() {
8
+ return bufferTransform;
9
+ }
8
10
  });
9
11
  const _stream = require("stream");
10
12
  function bufferTransform() {
@@ -1,5 +1,5 @@
1
1
  import type { CacheOptions } from "@lage-run/cache";
2
- import { RemoteFallbackCacheProvider, TargetHasher } from "@lage-run/cache";
2
+ import { RemoteFallbackCacheProvider } from "@lage-run/cache";
3
3
  import type { Logger } from "@lage-run/logger";
4
4
  interface CreateCacheOptions {
5
5
  cacheOptions?: CacheOptions;
@@ -8,8 +8,7 @@ interface CreateCacheOptions {
8
8
  skipLocalCache: boolean;
9
9
  cliArgs: string[];
10
10
  }
11
- export declare function createCache(options: CreateCacheOptions): {
11
+ export declare function createCache(options: CreateCacheOptions): Promise<{
12
12
  cacheProvider: RemoteFallbackCacheProvider;
13
- hasher: TargetHasher;
14
- };
13
+ }>;
15
14
  export {};
@@ -4,12 +4,14 @@ Object.defineProperty(exports, "__esModule", {
4
4
  });
5
5
  Object.defineProperty(exports, "createCache", {
6
6
  enumerable: true,
7
- get: ()=>createCache
7
+ get: function() {
8
+ return createCache;
9
+ }
8
10
  });
9
11
  const _cache = require("@lage-run/cache");
10
- const _isRunningFromCIJs = require("./isRunningFromCI.js");
11
- function createCache(options) {
12
- const { cacheOptions , logger , root , skipLocalCache , cliArgs } = options;
12
+ const _isRunningFromCI = require("./isRunningFromCI.js");
13
+ async function createCache(options) {
14
+ const { cacheOptions , logger , root , skipLocalCache } = options;
13
15
  const hasRemoteCacheConfig = !!cacheOptions?.cacheStorageConfig || !!process.env.BACKFILL_CACHE_PROVIDER || !!process.env.BACKFILL_CACHE_PROVIDER_OPTIONS;
14
16
  // Create Cache Provider
15
17
  const cacheProvider = new _cache.RemoteFallbackCacheProvider({
@@ -33,16 +35,9 @@ function createCache(options) {
33
35
  root,
34
36
  cacheOptions: cacheOptions ?? {}
35
37
  }) : undefined,
36
- writeRemoteCache: cacheOptions?.writeRemoteCache === true || String(process.env.LAGE_WRITE_CACHE).toLowerCase() === "true" || _isRunningFromCIJs.isRunningFromCI
37
- });
38
- const hasher = new _cache.TargetHasher({
39
- root,
40
- environmentGlob: cacheOptions?.environmentGlob ?? [],
41
- cacheKey: cacheOptions?.cacheKey,
42
- cliArgs
38
+ writeRemoteCache: cacheOptions?.writeRemoteCache === true || String(process.env.LAGE_WRITE_CACHE).toLowerCase() === "true" || _isRunningFromCI.isRunningFromCI
43
39
  });
44
40
  return {
45
- cacheProvider,
46
- hasher
41
+ cacheProvider
47
42
  };
48
43
  }
@@ -4,6 +4,8 @@ Object.defineProperty(exports, "__esModule", {
4
4
  });
5
5
  Object.defineProperty(exports, "isRunningFromCI", {
6
6
  enumerable: true,
7
- get: ()=>isRunningFromCI
7
+ get: function() {
8
+ return isRunningFromCI;
9
+ }
8
10
  });
9
11
  const isRunningFromCI = process.env.NODE_ENV !== "test" && (!!process.env.CI || !!process.env.TF_BUILD);
@@ -4,7 +4,9 @@ Object.defineProperty(exports, "__esModule", {
4
4
  });
5
5
  Object.defineProperty(exports, "categorizeTargetRuns", {
6
6
  enumerable: true,
7
- get: ()=>categorizeTargetRuns
7
+ get: function() {
8
+ return categorizeTargetRuns;
9
+ }
8
10
  });
9
11
  function categorizeTargetRuns(targetRuns) {
10
12
  const summary = {
@@ -4,7 +4,9 @@ Object.defineProperty(exports, "__esModule", {
4
4
  });
5
5
  Object.defineProperty(exports, "formatBytes", {
6
6
  enumerable: true,
7
- get: ()=>formatBytes
7
+ get: function() {
8
+ return formatBytes;
9
+ }
8
10
  });
9
11
  function formatBytes(bytes) {
10
12
  return `${(bytes / 1024 / 1024).toFixed(2)} MB`;
@@ -4,11 +4,13 @@ Object.defineProperty(exports, "__esModule", {
4
4
  });
5
5
  Object.defineProperty(exports, "getLageOutputCacheLocation", {
6
6
  enumerable: true,
7
- get: ()=>getLageOutputCacheLocation
7
+ get: function() {
8
+ return getLageOutputCacheLocation;
9
+ }
8
10
  });
9
11
  const _cache = require("@lage-run/cache");
10
- const _path = /*#__PURE__*/ _interopRequireDefault(require("path"));
11
- function _interopRequireDefault(obj) {
12
+ const _path = /*#__PURE__*/ _interop_require_default(require("path"));
13
+ function _interop_require_default(obj) {
12
14
  return obj && obj.__esModule ? obj : {
13
15
  default: obj
14
16
  };
package/lib/index.js CHANGED
@@ -9,14 +9,24 @@ function _export(target, all) {
9
9
  });
10
10
  }
11
11
  _export(exports, {
12
- NpmScriptRunner: ()=>_npmScriptRunnerJs.NpmScriptRunner,
13
- SimpleScheduler: ()=>_simpleSchedulerJs.SimpleScheduler,
14
- TargetRunnerPicker: ()=>_targetRunnerPickerJs.TargetRunnerPicker,
15
- WorkerRunner: ()=>_workerRunnerJs.WorkerRunner,
16
- NoOpRunner: ()=>_noOpRunnerJs.NoOpRunner
12
+ NpmScriptRunner: function() {
13
+ return _NpmScriptRunner.NpmScriptRunner;
14
+ },
15
+ SimpleScheduler: function() {
16
+ return _SimpleScheduler.SimpleScheduler;
17
+ },
18
+ TargetRunnerPicker: function() {
19
+ return _TargetRunnerPicker.TargetRunnerPicker;
20
+ },
21
+ WorkerRunner: function() {
22
+ return _WorkerRunner.WorkerRunner;
23
+ },
24
+ NoOpRunner: function() {
25
+ return _NoOpRunner.NoOpRunner;
26
+ }
17
27
  });
18
- const _npmScriptRunnerJs = require("./runners/NpmScriptRunner.js");
19
- const _simpleSchedulerJs = require("./SimpleScheduler.js");
20
- const _targetRunnerPickerJs = require("./runners/TargetRunnerPicker.js");
21
- const _workerRunnerJs = require("./runners/WorkerRunner.js");
22
- const _noOpRunnerJs = require("./runners/NoOpRunner.js");
28
+ const _NpmScriptRunner = require("./runners/NpmScriptRunner.js");
29
+ const _SimpleScheduler = require("./SimpleScheduler.js");
30
+ const _TargetRunnerPicker = require("./runners/TargetRunnerPicker.js");
31
+ const _WorkerRunner = require("./runners/WorkerRunner.js");
32
+ const _NoOpRunner = require("./runners/NoOpRunner.js");
@@ -4,7 +4,9 @@ Object.defineProperty(exports, "__esModule", {
4
4
  });
5
5
  Object.defineProperty(exports, "NoOpRunner", {
6
6
  enumerable: true,
7
- get: ()=>NoOpRunner
7
+ get: function() {
8
+ return NoOpRunner;
9
+ }
8
10
  });
9
11
  class NoOpRunner {
10
12
  async shouldRun() {
@@ -4,12 +4,27 @@ Object.defineProperty(exports, "__esModule", {
4
4
  });
5
5
  Object.defineProperty(exports, "NpmScriptRunner", {
6
6
  enumerable: true,
7
- get: ()=>NpmScriptRunner
7
+ get: function() {
8
+ return NpmScriptRunner;
9
+ }
8
10
  });
9
11
  const _fs = require("fs");
10
12
  const _path = require("path");
11
13
  const _promises = require("fs/promises");
12
- const _childProcess = require("child_process");
14
+ const _child_process = require("child_process");
15
+ function _define_property(obj, key, value) {
16
+ if (key in obj) {
17
+ Object.defineProperty(obj, key, {
18
+ value: value,
19
+ enumerable: true,
20
+ configurable: true,
21
+ writable: true
22
+ });
23
+ } else {
24
+ obj[key] = value;
25
+ }
26
+ return obj;
27
+ }
13
28
  class NpmScriptRunner {
14
29
  getNpmArgs(task, taskTargs) {
15
30
  const extraArgs = taskTargs.length > 0 ? [
@@ -77,7 +92,7 @@ class NpmScriptRunner {
77
92
  target.options?.nodeOptions
78
93
  ].filter((str)=>str).join(" ");
79
94
  await new Promise((resolve, reject)=>{
80
- childProcess = (0, _childProcess.spawn)(npmCmd, npmRunArgs, {
95
+ childProcess = (0, _child_process.spawn)(npmCmd, npmRunArgs, {
81
96
  cwd: target.cwd,
82
97
  stdio: [
83
98
  "inherit",
@@ -127,8 +142,9 @@ class NpmScriptRunner {
127
142
  });
128
143
  }
129
144
  constructor(options){
145
+ _define_property(this, "options", void 0);
130
146
  this.options = options;
131
147
  this.validateOptions(options);
132
148
  }
133
149
  }
134
- NpmScriptRunner.gracefulKillTimeout = 2500;
150
+ _define_property(NpmScriptRunner, "gracefulKillTimeout", 2500);
@@ -4,19 +4,34 @@ Object.defineProperty(exports, "__esModule", {
4
4
  });
5
5
  Object.defineProperty(exports, "TargetRunnerPicker", {
6
6
  enumerable: true,
7
- get: ()=>TargetRunnerPicker
7
+ get: function() {
8
+ return TargetRunnerPicker;
9
+ }
8
10
  });
9
- const _path = /*#__PURE__*/ _interopRequireDefault(require("path"));
10
- const _targetGraph = require("@lage-run/target-graph");
11
+ const _path = /*#__PURE__*/ _interop_require_default(require("path"));
12
+ const _targetgraph = require("@lage-run/target-graph");
11
13
  const _url = require("url");
12
- function _interopRequireDefault(obj) {
14
+ function _define_property(obj, key, value) {
15
+ if (key in obj) {
16
+ Object.defineProperty(obj, key, {
17
+ value: value,
18
+ enumerable: true,
19
+ configurable: true,
20
+ writable: true
21
+ });
22
+ } else {
23
+ obj[key] = value;
24
+ }
25
+ return obj;
26
+ }
27
+ function _interop_require_default(obj) {
13
28
  return obj && obj.__esModule ? obj : {
14
29
  default: obj
15
30
  };
16
31
  }
17
32
  class TargetRunnerPicker {
18
33
  async pick(target) {
19
- if (target.id === (0, _targetGraph.getStartTargetId)()) {
34
+ if (target.id === (0, _targetgraph.getStartTargetId)()) {
20
35
  return new (await import("./NoOpRunner.js")).NoOpRunner();
21
36
  }
22
37
  if (!target.type) {
@@ -38,6 +53,7 @@ class TargetRunnerPicker {
38
53
  throw new Error(`No runner found for target ${target.id}`);
39
54
  }
40
55
  constructor(options){
56
+ _define_property(this, "options", void 0);
41
57
  this.options = options;
42
58
  }
43
59
  }
@@ -4,9 +4,24 @@ Object.defineProperty(exports, "__esModule", {
4
4
  });
5
5
  Object.defineProperty(exports, "WorkerRunner", {
6
6
  enumerable: true,
7
- get: ()=>WorkerRunner
7
+ get: function() {
8
+ return WorkerRunner;
9
+ }
8
10
  });
9
11
  const _url = require("url");
12
+ function _define_property(obj, key, value) {
13
+ if (key in obj) {
14
+ Object.defineProperty(obj, key, {
15
+ value: value,
16
+ enumerable: true,
17
+ configurable: true,
18
+ writable: true
19
+ });
20
+ } else {
21
+ obj[key] = value;
22
+ }
23
+ return obj;
24
+ }
10
25
  class WorkerRunner {
11
26
  async shouldRun(target) {
12
27
  const scriptModule = await this.getScriptModule(target);
@@ -42,7 +57,8 @@ class WorkerRunner {
42
57
  return await import(importScript);
43
58
  }
44
59
  constructor(options){
60
+ _define_property(this, "options", void 0);
45
61
  this.options = options;
46
62
  }
47
63
  }
48
- WorkerRunner.gracefulKillTimeout = 2500;
64
+ _define_property(WorkerRunner, "gracefulKillTimeout", 2500);
@@ -2,13 +2,13 @@
2
2
  Object.defineProperty(exports, "__esModule", {
3
3
  value: true
4
4
  });
5
- const _createCacheProviderJs = require("../cache/createCacheProvider.js");
5
+ const _createCacheProvider = require("../cache/createCacheProvider.js");
6
6
  const _config = require("@lage-run/config");
7
- const _workerThreadsPool = require("@lage-run/worker-threads-pool");
8
- const _targetRunnerPickerJs = require("../runners/TargetRunnerPicker.js");
9
- const _workerThreads = require("worker_threads");
10
- const _logger = /*#__PURE__*/ _interopRequireDefault(require("@lage-run/logger"));
11
- function _interopRequireDefault(obj) {
7
+ const _workerthreadspool = require("@lage-run/worker-threads-pool");
8
+ const _TargetRunnerPicker = require("../runners/TargetRunnerPicker.js");
9
+ const _worker_threads = require("worker_threads");
10
+ const _logger = /*#__PURE__*/ _interop_require_default(require("@lage-run/logger"));
11
+ function _interop_require_default(obj) {
12
12
  return obj && obj.__esModule ? obj : {
13
13
  default: obj
14
14
  };
@@ -19,41 +19,54 @@ async function setup(options) {
19
19
  const logger = (0, _logger.default)();
20
20
  logger.addReporter({
21
21
  log (entry) {
22
- _workerThreads.parentPort.postMessage({
22
+ _worker_threads.parentPort.postMessage({
23
23
  type: "log",
24
24
  ...entry
25
25
  });
26
26
  },
27
27
  summarize () {}
28
28
  });
29
- const { cacheProvider , hasher } = (0, _createCacheProviderJs.createCache)({
29
+ const { cacheProvider } = await (0, _createCacheProvider.createCache)({
30
30
  root,
31
31
  logger,
32
32
  cacheOptions: config.cacheOptions,
33
33
  cliArgs: options.taskArgs,
34
34
  skipLocalCache: options.skipLocalCache
35
35
  });
36
- const runnerPicker = new _targetRunnerPickerJs.TargetRunnerPicker(runners);
36
+ const runnerPicker = new _TargetRunnerPicker.TargetRunnerPicker(runners);
37
37
  return {
38
38
  options,
39
39
  runnerPicker,
40
- cacheProvider,
41
- hasher
40
+ cacheProvider
42
41
  };
43
42
  }
44
43
  (async ()=>{
45
- const { cacheProvider , hasher , runnerPicker , options } = await setup(_workerThreads.workerData);
44
+ const { cacheProvider , runnerPicker , options } = await setup(_worker_threads.workerData);
45
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
46
+ let hashPromiseResolve = (_hash)=>{};
47
+ // main thread sends hash to worker because it keeps a global memory cache of the hashes
48
+ _worker_threads.parentPort.on("message", (data)=>{
49
+ if (data.type === "hash") {
50
+ hashPromiseResolve(data.hash);
51
+ }
52
+ });
46
53
  async function getCache(target) {
47
54
  const { shouldCache , shouldResetCache } = options;
48
55
  let hash = undefined;
49
56
  let cacheHit = false;
50
- if (!shouldCache || !target.cache || !cacheProvider || !hasher) {
57
+ if (!shouldCache || !target.cache || !cacheProvider) {
51
58
  return {
52
59
  hash,
53
60
  cacheHit
54
61
  };
55
62
  }
56
- hash = await hasher.hash(target);
63
+ // using a special pattern in communicating with the main thread to get the hash for the target
64
+ hash = await new Promise((resolve)=>{
65
+ hashPromiseResolve = resolve;
66
+ _worker_threads.parentPort.postMessage({
67
+ type: "hash"
68
+ });
69
+ });
57
70
  if (hash && !shouldResetCache) {
58
71
  cacheHit = await cacheProvider.fetch(hash, target);
59
72
  }
@@ -84,5 +97,5 @@ async function setup(options) {
84
97
  hash
85
98
  };
86
99
  }
87
- (0, _workerThreadsPool.registerWorker)(run);
100
+ (0, _workerthreadspool.registerWorker)(run);
88
101
  })();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lage-run/scheduler",
3
- "version": "0.11.4",
3
+ "version": "1.0.0",
4
4
  "description": "Scheduler for Lage",
5
5
  "repository": {
6
6
  "type": "git",
@@ -17,14 +17,15 @@
17
17
  "lint": "monorepo-scripts lint"
18
18
  },
19
19
  "dependencies": {
20
- "@lage-run/target-graph": "^0.8.4",
20
+ "@lage-run/target-graph": "^0.8.6",
21
21
  "@lage-run/logger": "^1.3.0",
22
- "@lage-run/cache": "^0.5.4",
23
- "@lage-run/config": "^0.2.0",
24
- "@lage-run/worker-threads-pool": "^0.6.1"
22
+ "@lage-run/cache": "^1.1.1",
23
+ "@lage-run/config": "^0.2.1",
24
+ "@lage-run/hasher": "^1.0.0",
25
+ "@lage-run/worker-threads-pool": "^0.7.0"
25
26
  },
26
27
  "devDependencies": {
27
- "@lage-run/scheduler-types": "^0.3.8",
28
+ "@lage-run/scheduler-types": "^0.3.9",
28
29
  "monorepo-scripts": "*"
29
30
  },
30
31
  "publishConfig": {