@lage-run/scheduler 0.11.4 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.json +53 -2
- package/CHANGELOG.md +16 -2
- package/lib/SimpleScheduler.d.ts +2 -0
- package/lib/SimpleScheduler.js +51 -26
- package/lib/WrappedTarget.d.ts +2 -0
- package/lib/WrappedTarget.js +51 -26
- package/lib/bufferTransform.js +3 -1
- package/lib/cache/createCacheProvider.d.ts +3 -4
- package/lib/cache/createCacheProvider.js +8 -13
- package/lib/cache/isRunningFromCI.js +3 -1
- package/lib/categorizeTargetRuns.js +3 -1
- package/lib/formatBytes.js +3 -1
- package/lib/getLageOutputCacheLocation.js +5 -3
- package/lib/index.js +20 -10
- package/lib/runners/NoOpRunner.js +3 -1
- package/lib/runners/NpmScriptRunner.js +20 -4
- package/lib/runners/TargetRunnerPicker.js +21 -5
- package/lib/runners/WorkerRunner.js +18 -2
- package/lib/workers/targetWorker.js +28 -15
- package/package.json +7 -6
package/CHANGELOG.json
CHANGED
|
@@ -2,7 +2,58 @@
|
|
|
2
2
|
"name": "@lage-run/scheduler",
|
|
3
3
|
"entries": [
|
|
4
4
|
{
|
|
5
|
-
"date": "
|
|
5
|
+
"date": "Mon, 08 May 2023 22:26:52 GMT",
|
|
6
|
+
"tag": "@lage-run/scheduler_v1.0.0",
|
|
7
|
+
"version": "1.0.0",
|
|
8
|
+
"comments": {
|
|
9
|
+
"major": [
|
|
10
|
+
{
|
|
11
|
+
"author": "kchau@microsoft.com",
|
|
12
|
+
"package": "@lage-run/scheduler",
|
|
13
|
+
"commit": "5a132808f166179bc316a279c9e11a13d3a39103",
|
|
14
|
+
"comment": "Now takes in a TargetHasher as a dep, but retains its own calls to cache provider"
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"author": "beachball",
|
|
18
|
+
"package": "@lage-run/scheduler",
|
|
19
|
+
"comment": "Bump @lage-run/target-graph to v0.8.6",
|
|
20
|
+
"commit": "5a132808f166179bc316a279c9e11a13d3a39103"
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"author": "beachball",
|
|
24
|
+
"package": "@lage-run/scheduler",
|
|
25
|
+
"comment": "Bump @lage-run/cache to v1.1.1",
|
|
26
|
+
"commit": "5a132808f166179bc316a279c9e11a13d3a39103"
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"author": "beachball",
|
|
30
|
+
"package": "@lage-run/scheduler",
|
|
31
|
+
"comment": "Bump @lage-run/config to v0.2.1",
|
|
32
|
+
"commit": "5a132808f166179bc316a279c9e11a13d3a39103"
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"author": "beachball",
|
|
36
|
+
"package": "@lage-run/scheduler",
|
|
37
|
+
"comment": "Bump @lage-run/hasher to v1.0.0",
|
|
38
|
+
"commit": "5a132808f166179bc316a279c9e11a13d3a39103"
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"author": "beachball",
|
|
42
|
+
"package": "@lage-run/scheduler",
|
|
43
|
+
"comment": "Bump @lage-run/worker-threads-pool to v0.7.0",
|
|
44
|
+
"commit": "5a132808f166179bc316a279c9e11a13d3a39103"
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"author": "beachball",
|
|
48
|
+
"package": "@lage-run/scheduler",
|
|
49
|
+
"comment": "Bump @lage-run/scheduler-types to v0.3.9",
|
|
50
|
+
"commit": "5a132808f166179bc316a279c9e11a13d3a39103"
|
|
51
|
+
}
|
|
52
|
+
]
|
|
53
|
+
}
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"date": "Wed, 26 Apr 2023 04:56:20 GMT",
|
|
6
57
|
"tag": "@lage-run/scheduler_v0.11.4",
|
|
7
58
|
"version": "0.11.4",
|
|
8
59
|
"comments": {
|
|
@@ -11,7 +62,7 @@
|
|
|
11
62
|
"author": "beachball",
|
|
12
63
|
"package": "@lage-run/scheduler",
|
|
13
64
|
"comment": "Bump @lage-run/config to v0.2.0",
|
|
14
|
-
"commit": "
|
|
65
|
+
"commit": "994af55fcc890b448fc2b5cd5e4ba8bca0832997"
|
|
15
66
|
}
|
|
16
67
|
]
|
|
17
68
|
}
|
package/CHANGELOG.md
CHANGED
|
@@ -1,12 +1,26 @@
|
|
|
1
1
|
# Change Log - @lage-run/scheduler
|
|
2
2
|
|
|
3
|
-
This log was last generated on
|
|
3
|
+
This log was last generated on Mon, 08 May 2023 22:26:52 GMT and should not be manually modified.
|
|
4
4
|
|
|
5
5
|
<!-- Start content -->
|
|
6
6
|
|
|
7
|
+
## 1.0.0
|
|
8
|
+
|
|
9
|
+
Mon, 08 May 2023 22:26:52 GMT
|
|
10
|
+
|
|
11
|
+
### Major changes
|
|
12
|
+
|
|
13
|
+
- Now takes in a TargetHasher as a dep, but retains its own calls to cache provider (kchau@microsoft.com)
|
|
14
|
+
- Bump @lage-run/target-graph to v0.8.6
|
|
15
|
+
- Bump @lage-run/cache to v1.1.1
|
|
16
|
+
- Bump @lage-run/config to v0.2.1
|
|
17
|
+
- Bump @lage-run/hasher to v1.0.0
|
|
18
|
+
- Bump @lage-run/worker-threads-pool to v0.7.0
|
|
19
|
+
- Bump @lage-run/scheduler-types to v0.3.9
|
|
20
|
+
|
|
7
21
|
## 0.11.4
|
|
8
22
|
|
|
9
|
-
Wed, 26 Apr 2023 04:56:
|
|
23
|
+
Wed, 26 Apr 2023 04:56:20 GMT
|
|
10
24
|
|
|
11
25
|
### Patches
|
|
12
26
|
|
package/lib/SimpleScheduler.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ import type { TargetGraph } from "@lage-run/target-graph";
|
|
|
7
7
|
import type { TargetScheduler, SchedulerRunSummary } from "@lage-run/scheduler-types";
|
|
8
8
|
import type { Pool } from "@lage-run/worker-threads-pool";
|
|
9
9
|
import type { TargetRunnerPickerOptions } from "@lage-run/scheduler-types";
|
|
10
|
+
import type { TargetHasher } from "@lage-run/hasher";
|
|
10
11
|
export interface SimpleSchedulerOptions {
|
|
11
12
|
logger: Logger;
|
|
12
13
|
concurrency: number;
|
|
@@ -22,6 +23,7 @@ export interface SimpleSchedulerOptions {
|
|
|
22
23
|
maxWorkersPerTask: Map<string, number>;
|
|
23
24
|
pool?: Pool;
|
|
24
25
|
workerIdleMemoryLimit: number;
|
|
26
|
+
hasher: TargetHasher;
|
|
25
27
|
}
|
|
26
28
|
/**
|
|
27
29
|
* Simple scheduler that runs all targets in a promise graph using p-graph library.
|
package/lib/SimpleScheduler.js
CHANGED
|
@@ -4,33 +4,48 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
});
|
|
5
5
|
Object.defineProperty(exports, "SimpleScheduler", {
|
|
6
6
|
enumerable: true,
|
|
7
|
-
get: ()
|
|
7
|
+
get: function() {
|
|
8
|
+
return SimpleScheduler;
|
|
9
|
+
}
|
|
8
10
|
});
|
|
9
|
-
const
|
|
10
|
-
const
|
|
11
|
-
const
|
|
12
|
-
const
|
|
13
|
-
const
|
|
14
|
-
const
|
|
15
|
-
function
|
|
11
|
+
const _workerthreadspool = require("@lage-run/worker-threads-pool");
|
|
12
|
+
const _formatBytes = require("./formatBytes.js");
|
|
13
|
+
const _categorizeTargetRuns = require("./categorizeTargetRuns.js");
|
|
14
|
+
const _targetgraph = require("@lage-run/target-graph");
|
|
15
|
+
const _WrappedTarget = require("./WrappedTarget.js");
|
|
16
|
+
const _TargetRunnerPicker = require("./runners/TargetRunnerPicker.js");
|
|
17
|
+
function _check_private_redeclaration(obj, privateCollection) {
|
|
16
18
|
if (privateCollection.has(obj)) {
|
|
17
19
|
throw new TypeError("Cannot initialize the same private elements twice on an object");
|
|
18
20
|
}
|
|
19
21
|
}
|
|
20
|
-
function
|
|
22
|
+
function _class_private_method_get(receiver, privateSet, fn) {
|
|
21
23
|
if (!privateSet.has(receiver)) {
|
|
22
24
|
throw new TypeError("attempted to get private field on non-instance");
|
|
23
25
|
}
|
|
24
26
|
return fn;
|
|
25
27
|
}
|
|
26
|
-
function
|
|
27
|
-
|
|
28
|
+
function _class_private_method_init(obj, privateSet) {
|
|
29
|
+
_check_private_redeclaration(obj, privateSet);
|
|
28
30
|
privateSet.add(obj);
|
|
29
31
|
}
|
|
32
|
+
function _define_property(obj, key, value) {
|
|
33
|
+
if (key in obj) {
|
|
34
|
+
Object.defineProperty(obj, key, {
|
|
35
|
+
value: value,
|
|
36
|
+
enumerable: true,
|
|
37
|
+
configurable: true,
|
|
38
|
+
writable: true
|
|
39
|
+
});
|
|
40
|
+
} else {
|
|
41
|
+
obj[key] = value;
|
|
42
|
+
}
|
|
43
|
+
return obj;
|
|
44
|
+
}
|
|
30
45
|
var _generateTargetRunPromise = /*#__PURE__*/ new WeakSet();
|
|
31
46
|
class SimpleScheduler {
|
|
32
47
|
getTargetsByPriority() {
|
|
33
|
-
return (0,
|
|
48
|
+
return (0, _targetgraph.sortTargetsByPriority)([
|
|
34
49
|
...this.targetRuns.values()
|
|
35
50
|
].map((run)=>run.target));
|
|
36
51
|
}
|
|
@@ -68,17 +83,18 @@ class SimpleScheduler {
|
|
|
68
83
|
this.markTargetAndDependentsPending(target.id);
|
|
69
84
|
}
|
|
70
85
|
} else {
|
|
71
|
-
targetRun = new
|
|
86
|
+
targetRun = new _WrappedTarget.WrappedTarget({
|
|
72
87
|
target,
|
|
73
88
|
root,
|
|
74
89
|
logger,
|
|
75
90
|
shouldCache,
|
|
76
91
|
continueOnError,
|
|
77
92
|
abortController,
|
|
78
|
-
pool
|
|
93
|
+
pool,
|
|
94
|
+
hasher: this.options.hasher
|
|
79
95
|
});
|
|
80
96
|
}
|
|
81
|
-
if (target.id === (0,
|
|
97
|
+
if (target.id === (0, _targetgraph.getStartTargetId)()) {
|
|
82
98
|
targetRun.status = "success";
|
|
83
99
|
}
|
|
84
100
|
this.targetRuns.set(target.id, targetRun);
|
|
@@ -96,12 +112,13 @@ class SimpleScheduler {
|
|
|
96
112
|
error = e instanceof Error ? e.message : String(e);
|
|
97
113
|
} finally{
|
|
98
114
|
duration = process.hrtime(startTime);
|
|
99
|
-
targetRunByStatus = (0,
|
|
115
|
+
targetRunByStatus = (0, _categorizeTargetRuns.categorizeTargetRuns)(this.targetRuns.values());
|
|
100
116
|
if (targetRunByStatus.failed.length + targetRunByStatus.aborted.length + targetRunByStatus.pending.length + targetRunByStatus.running.length === 0) {
|
|
101
117
|
results = "success";
|
|
102
118
|
}
|
|
103
119
|
}
|
|
104
120
|
const poolStats = pool.stats();
|
|
121
|
+
await this.options.hasher.cleanup();
|
|
105
122
|
return {
|
|
106
123
|
targetRunByStatus,
|
|
107
124
|
targetRuns: this.targetRuns,
|
|
@@ -139,7 +156,7 @@ class SimpleScheduler {
|
|
|
139
156
|
const readyTargets = new Set();
|
|
140
157
|
for (const target of this.getTargetsByPriority()){
|
|
141
158
|
// Skip the start target
|
|
142
|
-
if (target.id === (0,
|
|
159
|
+
if (target.id === (0, _targetgraph.getStartTargetId)()) {
|
|
143
160
|
continue;
|
|
144
161
|
}
|
|
145
162
|
const targetRun = this.targetRuns.get(target.id);
|
|
@@ -149,7 +166,7 @@ class SimpleScheduler {
|
|
|
149
166
|
// Target is only ready when all its deps are "successful" or that it is a root target
|
|
150
167
|
const ready = targetDeps.every((dep)=>{
|
|
151
168
|
const fromTarget = this.targetRuns.get(dep);
|
|
152
|
-
return fromTarget.successful || dep === (0,
|
|
169
|
+
return fromTarget.successful || dep === (0, _targetgraph.getStartTargetId)();
|
|
153
170
|
});
|
|
154
171
|
if (ready) {
|
|
155
172
|
readyTargets.add(targetRun);
|
|
@@ -162,7 +179,7 @@ class SimpleScheduler {
|
|
|
162
179
|
}
|
|
163
180
|
isAllDone() {
|
|
164
181
|
for (const t of this.targetRuns.values()){
|
|
165
|
-
if (t.status !== "skipped" && t.status !== "success" && t.target.id !== (0,
|
|
182
|
+
if (t.status !== "skipped" && t.status !== "success" && t.target.id !== (0, _targetgraph.getStartTargetId)()) {
|
|
166
183
|
return false;
|
|
167
184
|
}
|
|
168
185
|
}
|
|
@@ -172,16 +189,16 @@ class SimpleScheduler {
|
|
|
172
189
|
if (this.isAllDone() || this.abortSignal.aborted) {
|
|
173
190
|
return Promise.resolve();
|
|
174
191
|
}
|
|
175
|
-
this.options.logger.silly(`Max Worker Memory Usage: ${(0,
|
|
192
|
+
this.options.logger.silly(`Max Worker Memory Usage: ${(0, _formatBytes.formatBytes)(this.pool.stats().maxWorkerMemoryUsage)}`);
|
|
176
193
|
const promises = [];
|
|
177
194
|
for (const nextTarget of this.getReadyTargets()){
|
|
178
|
-
const runPromise =
|
|
195
|
+
const runPromise = _class_private_method_get(this, _generateTargetRunPromise, generateTargetRunPromise).call(this, nextTarget);
|
|
179
196
|
promises.push(runPromise);
|
|
180
197
|
}
|
|
181
198
|
await Promise.all(promises);
|
|
182
199
|
}
|
|
183
200
|
logProgress() {
|
|
184
|
-
const targetRunByStatus = (0,
|
|
201
|
+
const targetRunByStatus = (0, _categorizeTargetRuns.categorizeTargetRuns)(this.targetRuns.values());
|
|
185
202
|
const total = [
|
|
186
203
|
...this.targetRuns.values()
|
|
187
204
|
].filter((t)=>!t.target.hidden).length;
|
|
@@ -195,7 +212,7 @@ class SimpleScheduler {
|
|
|
195
212
|
});
|
|
196
213
|
}
|
|
197
214
|
async cleanup() {
|
|
198
|
-
this.options.logger.silly(`Max Worker Memory Usage: ${(0,
|
|
215
|
+
this.options.logger.silly(`Max Worker Memory Usage: ${(0, _formatBytes.formatBytes)(this.pool.stats().maxWorkerMemoryUsage)}`);
|
|
199
216
|
await this.pool.close();
|
|
200
217
|
}
|
|
201
218
|
/**
|
|
@@ -204,14 +221,22 @@ class SimpleScheduler {
|
|
|
204
221
|
this.abortController.abort();
|
|
205
222
|
}
|
|
206
223
|
constructor(options){
|
|
207
|
-
|
|
224
|
+
_class_private_method_init(this, _generateTargetRunPromise);
|
|
225
|
+
_define_property(this, "options", void 0);
|
|
226
|
+
_define_property(this, "targetRuns", void 0);
|
|
227
|
+
_define_property(this, "rerunTargets", void 0);
|
|
228
|
+
_define_property(this, "abortController", void 0);
|
|
229
|
+
_define_property(this, "abortSignal", void 0);
|
|
230
|
+
_define_property(this, "pool", void 0);
|
|
231
|
+
_define_property(this, "runnerPicker", void 0);
|
|
232
|
+
_define_property(this, "runPromise", void 0);
|
|
208
233
|
this.options = options;
|
|
209
234
|
this.targetRuns = new Map();
|
|
210
235
|
this.rerunTargets = new Set();
|
|
211
236
|
this.abortController = new AbortController();
|
|
212
237
|
this.abortSignal = this.abortController.signal;
|
|
213
238
|
this.runPromise = Promise.resolve();
|
|
214
|
-
this.pool = options.pool ?? new
|
|
239
|
+
this.pool = options.pool ?? new _workerthreadspool.AggregatedPool({
|
|
215
240
|
logger: options.logger,
|
|
216
241
|
maxWorkersByGroup: options.maxWorkersPerTask,
|
|
217
242
|
groupBy: ({ target })=>target.task,
|
|
@@ -228,7 +253,7 @@ class SimpleScheduler {
|
|
|
228
253
|
},
|
|
229
254
|
workerIdleMemoryLimit: options.workerIdleMemoryLimit
|
|
230
255
|
});
|
|
231
|
-
this.runnerPicker = new
|
|
256
|
+
this.runnerPicker = new _TargetRunnerPicker.TargetRunnerPicker(options.workerData.runners);
|
|
232
257
|
}
|
|
233
258
|
}
|
|
234
259
|
async function generateTargetRunPromise(target) {
|
package/lib/WrappedTarget.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type { Pool } from "@lage-run/worker-threads-pool";
|
|
|
3
3
|
import type { TargetRun, TargetStatus } from "@lage-run/scheduler-types";
|
|
4
4
|
import type { Target } from "@lage-run/target-graph";
|
|
5
5
|
import type { Logger } from "@lage-run/logger";
|
|
6
|
+
import type { TargetHasher } from "@lage-run/hasher";
|
|
6
7
|
export interface WrappedTargetOptions {
|
|
7
8
|
root: string;
|
|
8
9
|
target: Target;
|
|
@@ -11,6 +12,7 @@ export interface WrappedTargetOptions {
|
|
|
11
12
|
continueOnError: boolean;
|
|
12
13
|
abortController: AbortController;
|
|
13
14
|
pool: Pool;
|
|
15
|
+
hasher: TargetHasher;
|
|
14
16
|
}
|
|
15
17
|
interface WorkerResult {
|
|
16
18
|
stdoutBuffer: string;
|
package/lib/WrappedTarget.js
CHANGED
|
@@ -4,15 +4,30 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
});
|
|
5
5
|
Object.defineProperty(exports, "WrappedTarget", {
|
|
6
6
|
enumerable: true,
|
|
7
|
-
get: ()
|
|
7
|
+
get: function() {
|
|
8
|
+
return WrappedTarget;
|
|
9
|
+
}
|
|
8
10
|
});
|
|
9
|
-
const
|
|
10
|
-
const
|
|
11
|
+
const _bufferTransform = require("./bufferTransform.js");
|
|
12
|
+
const _getLageOutputCacheLocation = require("./getLageOutputCacheLocation.js");
|
|
11
13
|
const _logger = require("@lage-run/logger");
|
|
12
|
-
const _fs = /*#__PURE__*/
|
|
13
|
-
const _path = /*#__PURE__*/
|
|
14
|
+
const _fs = /*#__PURE__*/ _interop_require_default(require("fs"));
|
|
15
|
+
const _path = /*#__PURE__*/ _interop_require_default(require("path"));
|
|
14
16
|
const _promises = require("fs/promises");
|
|
15
|
-
function
|
|
17
|
+
function _define_property(obj, key, value) {
|
|
18
|
+
if (key in obj) {
|
|
19
|
+
Object.defineProperty(obj, key, {
|
|
20
|
+
value: value,
|
|
21
|
+
enumerable: true,
|
|
22
|
+
configurable: true,
|
|
23
|
+
writable: true
|
|
24
|
+
});
|
|
25
|
+
} else {
|
|
26
|
+
obj[key] = value;
|
|
27
|
+
}
|
|
28
|
+
return obj;
|
|
29
|
+
}
|
|
30
|
+
function _interop_require_default(obj) {
|
|
16
31
|
return obj && obj.__esModule ? obj : {
|
|
17
32
|
default: obj
|
|
18
33
|
};
|
|
@@ -94,7 +109,7 @@ class WrappedTarget {
|
|
|
94
109
|
}
|
|
95
110
|
}
|
|
96
111
|
async run() {
|
|
97
|
-
const { target , logger , shouldCache , abortController } = this.options;
|
|
112
|
+
const { target , logger , shouldCache , abortController , root } = this.options;
|
|
98
113
|
const abortSignal = abortController.signal;
|
|
99
114
|
if (abortSignal.aborted) {
|
|
100
115
|
this.onStart(0);
|
|
@@ -106,34 +121,29 @@ class WrappedTarget {
|
|
|
106
121
|
const cacheEnabled = target.cache && shouldCache && result.hash;
|
|
107
122
|
// Save output if cache is enabled & cache is hit
|
|
108
123
|
if (!result.skipped && cacheEnabled) {
|
|
109
|
-
const outputLocation = (0,
|
|
124
|
+
const outputLocation = (0, _getLageOutputCacheLocation.getLageOutputCacheLocation)(root, result.hash);
|
|
110
125
|
const outputPath = _path.default.dirname(outputLocation);
|
|
111
126
|
await (0, _promises.mkdir)(outputPath, {
|
|
112
127
|
recursive: true
|
|
113
128
|
});
|
|
114
129
|
const output = `${result.stdoutBuffer}\n${result.stderrBuffer}`;
|
|
115
130
|
await (0, _promises.writeFile)(outputLocation, output);
|
|
116
|
-
|
|
131
|
+
logger.verbose(`>> Saved cache - ${result.hash}`, {
|
|
117
132
|
target
|
|
118
133
|
});
|
|
119
134
|
}
|
|
120
135
|
if (result.skipped) {
|
|
121
136
|
const { hash } = result;
|
|
122
|
-
const cachedOutputFile = (0,
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
137
|
+
const cachedOutputFile = (0, _getLageOutputCacheLocation.getLageOutputCacheLocation)(root, hash ?? "");
|
|
138
|
+
const shouldShowCachedOutput = _fs.default.existsSync(cachedOutputFile);
|
|
139
|
+
if (shouldShowCachedOutput) {
|
|
140
|
+
const cachedOutput = _fs.default.readFileSync(cachedOutputFile, "utf8");
|
|
141
|
+
logger.verbose(">> Replaying cached output", {
|
|
126
142
|
target
|
|
127
143
|
});
|
|
128
|
-
|
|
144
|
+
logger.verbose(cachedOutput.trim(), {
|
|
129
145
|
target
|
|
130
146
|
});
|
|
131
|
-
return await new Promise((resolve)=>{
|
|
132
|
-
cachedOutput.on("close", ()=>{
|
|
133
|
-
this.onSkipped(hash);
|
|
134
|
-
resolve();
|
|
135
|
-
});
|
|
136
|
-
});
|
|
137
147
|
}
|
|
138
148
|
this.onSkipped(hash);
|
|
139
149
|
} else {
|
|
@@ -156,21 +166,28 @@ class WrappedTarget {
|
|
|
156
166
|
const abortSignal = abortController.signal;
|
|
157
167
|
let releaseStdout;
|
|
158
168
|
let releaseStderr;
|
|
159
|
-
const bufferStdout = (0,
|
|
160
|
-
const bufferStderr = (0,
|
|
161
|
-
let
|
|
169
|
+
const bufferStdout = (0, _bufferTransform.bufferTransform)();
|
|
170
|
+
const bufferStderr = (0, _bufferTransform.bufferTransform)();
|
|
171
|
+
let msgHandler;
|
|
162
172
|
this.result = pool.exec({
|
|
163
173
|
target
|
|
164
174
|
}, target.weight ?? 1, (worker, stdout, stderr)=>{
|
|
165
|
-
|
|
175
|
+
msgHandler = (data)=>{
|
|
166
176
|
if (data.type === "log") {
|
|
167
177
|
logger.log(data.level, data.msg, {
|
|
168
178
|
target,
|
|
169
179
|
threadId: worker.threadId
|
|
170
180
|
});
|
|
181
|
+
} else if (data.type === "hash") {
|
|
182
|
+
this.options.hasher.hash(target).then((hash)=>{
|
|
183
|
+
worker.postMessage({
|
|
184
|
+
type: "hash",
|
|
185
|
+
hash
|
|
186
|
+
});
|
|
187
|
+
});
|
|
171
188
|
}
|
|
172
189
|
};
|
|
173
|
-
worker.on("message",
|
|
190
|
+
worker.on("message", msgHandler);
|
|
174
191
|
const threadId = worker.threadId;
|
|
175
192
|
this.onStart(threadId);
|
|
176
193
|
stdout.pipe(bufferStdout.transform);
|
|
@@ -192,7 +209,7 @@ class WrappedTarget {
|
|
|
192
209
|
stderr.unpipe(bufferStderr.transform);
|
|
193
210
|
};
|
|
194
211
|
}, (worker)=>{
|
|
195
|
-
worker.off("message",
|
|
212
|
+
worker.off("message", msgHandler);
|
|
196
213
|
releaseStdout();
|
|
197
214
|
releaseStderr();
|
|
198
215
|
}, abortSignal);
|
|
@@ -217,6 +234,14 @@ class WrappedTarget {
|
|
|
217
234
|
};
|
|
218
235
|
}
|
|
219
236
|
constructor(options){
|
|
237
|
+
_define_property(this, "options", void 0);
|
|
238
|
+
_define_property(this, "queueTime", void 0);
|
|
239
|
+
_define_property(this, "startTime", void 0);
|
|
240
|
+
_define_property(this, "duration", void 0);
|
|
241
|
+
_define_property(this, "target", void 0);
|
|
242
|
+
_define_property(this, "status", void 0);
|
|
243
|
+
_define_property(this, "threadId", void 0);
|
|
244
|
+
_define_property(this, "result", void 0);
|
|
220
245
|
this.options = options;
|
|
221
246
|
this.queueTime = [
|
|
222
247
|
0,
|
package/lib/bufferTransform.js
CHANGED
|
@@ -4,7 +4,9 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
});
|
|
5
5
|
Object.defineProperty(exports, "bufferTransform", {
|
|
6
6
|
enumerable: true,
|
|
7
|
-
get: ()
|
|
7
|
+
get: function() {
|
|
8
|
+
return bufferTransform;
|
|
9
|
+
}
|
|
8
10
|
});
|
|
9
11
|
const _stream = require("stream");
|
|
10
12
|
function bufferTransform() {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CacheOptions } from "@lage-run/cache";
|
|
2
|
-
import { RemoteFallbackCacheProvider
|
|
2
|
+
import { RemoteFallbackCacheProvider } from "@lage-run/cache";
|
|
3
3
|
import type { Logger } from "@lage-run/logger";
|
|
4
4
|
interface CreateCacheOptions {
|
|
5
5
|
cacheOptions?: CacheOptions;
|
|
@@ -8,8 +8,7 @@ interface CreateCacheOptions {
|
|
|
8
8
|
skipLocalCache: boolean;
|
|
9
9
|
cliArgs: string[];
|
|
10
10
|
}
|
|
11
|
-
export declare function createCache(options: CreateCacheOptions): {
|
|
11
|
+
export declare function createCache(options: CreateCacheOptions): Promise<{
|
|
12
12
|
cacheProvider: RemoteFallbackCacheProvider;
|
|
13
|
-
|
|
14
|
-
};
|
|
13
|
+
}>;
|
|
15
14
|
export {};
|
|
@@ -4,12 +4,14 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
});
|
|
5
5
|
Object.defineProperty(exports, "createCache", {
|
|
6
6
|
enumerable: true,
|
|
7
|
-
get: ()
|
|
7
|
+
get: function() {
|
|
8
|
+
return createCache;
|
|
9
|
+
}
|
|
8
10
|
});
|
|
9
11
|
const _cache = require("@lage-run/cache");
|
|
10
|
-
const
|
|
11
|
-
function createCache(options) {
|
|
12
|
-
const { cacheOptions , logger , root , skipLocalCache
|
|
12
|
+
const _isRunningFromCI = require("./isRunningFromCI.js");
|
|
13
|
+
async function createCache(options) {
|
|
14
|
+
const { cacheOptions , logger , root , skipLocalCache } = options;
|
|
13
15
|
const hasRemoteCacheConfig = !!cacheOptions?.cacheStorageConfig || !!process.env.BACKFILL_CACHE_PROVIDER || !!process.env.BACKFILL_CACHE_PROVIDER_OPTIONS;
|
|
14
16
|
// Create Cache Provider
|
|
15
17
|
const cacheProvider = new _cache.RemoteFallbackCacheProvider({
|
|
@@ -33,16 +35,9 @@ function createCache(options) {
|
|
|
33
35
|
root,
|
|
34
36
|
cacheOptions: cacheOptions ?? {}
|
|
35
37
|
}) : undefined,
|
|
36
|
-
writeRemoteCache: cacheOptions?.writeRemoteCache === true || String(process.env.LAGE_WRITE_CACHE).toLowerCase() === "true" ||
|
|
37
|
-
});
|
|
38
|
-
const hasher = new _cache.TargetHasher({
|
|
39
|
-
root,
|
|
40
|
-
environmentGlob: cacheOptions?.environmentGlob ?? [],
|
|
41
|
-
cacheKey: cacheOptions?.cacheKey,
|
|
42
|
-
cliArgs
|
|
38
|
+
writeRemoteCache: cacheOptions?.writeRemoteCache === true || String(process.env.LAGE_WRITE_CACHE).toLowerCase() === "true" || _isRunningFromCI.isRunningFromCI
|
|
43
39
|
});
|
|
44
40
|
return {
|
|
45
|
-
cacheProvider
|
|
46
|
-
hasher
|
|
41
|
+
cacheProvider
|
|
47
42
|
};
|
|
48
43
|
}
|
|
@@ -4,6 +4,8 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
});
|
|
5
5
|
Object.defineProperty(exports, "isRunningFromCI", {
|
|
6
6
|
enumerable: true,
|
|
7
|
-
get: ()
|
|
7
|
+
get: function() {
|
|
8
|
+
return isRunningFromCI;
|
|
9
|
+
}
|
|
8
10
|
});
|
|
9
11
|
const isRunningFromCI = process.env.NODE_ENV !== "test" && (!!process.env.CI || !!process.env.TF_BUILD);
|
|
@@ -4,7 +4,9 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
});
|
|
5
5
|
Object.defineProperty(exports, "categorizeTargetRuns", {
|
|
6
6
|
enumerable: true,
|
|
7
|
-
get: ()
|
|
7
|
+
get: function() {
|
|
8
|
+
return categorizeTargetRuns;
|
|
9
|
+
}
|
|
8
10
|
});
|
|
9
11
|
function categorizeTargetRuns(targetRuns) {
|
|
10
12
|
const summary = {
|
package/lib/formatBytes.js
CHANGED
|
@@ -4,7 +4,9 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
});
|
|
5
5
|
Object.defineProperty(exports, "formatBytes", {
|
|
6
6
|
enumerable: true,
|
|
7
|
-
get: ()
|
|
7
|
+
get: function() {
|
|
8
|
+
return formatBytes;
|
|
9
|
+
}
|
|
8
10
|
});
|
|
9
11
|
function formatBytes(bytes) {
|
|
10
12
|
return `${(bytes / 1024 / 1024).toFixed(2)} MB`;
|
|
@@ -4,11 +4,13 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
});
|
|
5
5
|
Object.defineProperty(exports, "getLageOutputCacheLocation", {
|
|
6
6
|
enumerable: true,
|
|
7
|
-
get: ()
|
|
7
|
+
get: function() {
|
|
8
|
+
return getLageOutputCacheLocation;
|
|
9
|
+
}
|
|
8
10
|
});
|
|
9
11
|
const _cache = require("@lage-run/cache");
|
|
10
|
-
const _path = /*#__PURE__*/
|
|
11
|
-
function
|
|
12
|
+
const _path = /*#__PURE__*/ _interop_require_default(require("path"));
|
|
13
|
+
function _interop_require_default(obj) {
|
|
12
14
|
return obj && obj.__esModule ? obj : {
|
|
13
15
|
default: obj
|
|
14
16
|
};
|
package/lib/index.js
CHANGED
|
@@ -9,14 +9,24 @@ function _export(target, all) {
|
|
|
9
9
|
});
|
|
10
10
|
}
|
|
11
11
|
_export(exports, {
|
|
12
|
-
NpmScriptRunner: ()
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
12
|
+
NpmScriptRunner: function() {
|
|
13
|
+
return _NpmScriptRunner.NpmScriptRunner;
|
|
14
|
+
},
|
|
15
|
+
SimpleScheduler: function() {
|
|
16
|
+
return _SimpleScheduler.SimpleScheduler;
|
|
17
|
+
},
|
|
18
|
+
TargetRunnerPicker: function() {
|
|
19
|
+
return _TargetRunnerPicker.TargetRunnerPicker;
|
|
20
|
+
},
|
|
21
|
+
WorkerRunner: function() {
|
|
22
|
+
return _WorkerRunner.WorkerRunner;
|
|
23
|
+
},
|
|
24
|
+
NoOpRunner: function() {
|
|
25
|
+
return _NoOpRunner.NoOpRunner;
|
|
26
|
+
}
|
|
17
27
|
});
|
|
18
|
-
const
|
|
19
|
-
const
|
|
20
|
-
const
|
|
21
|
-
const
|
|
22
|
-
const
|
|
28
|
+
const _NpmScriptRunner = require("./runners/NpmScriptRunner.js");
|
|
29
|
+
const _SimpleScheduler = require("./SimpleScheduler.js");
|
|
30
|
+
const _TargetRunnerPicker = require("./runners/TargetRunnerPicker.js");
|
|
31
|
+
const _WorkerRunner = require("./runners/WorkerRunner.js");
|
|
32
|
+
const _NoOpRunner = require("./runners/NoOpRunner.js");
|
|
@@ -4,12 +4,27 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
});
|
|
5
5
|
Object.defineProperty(exports, "NpmScriptRunner", {
|
|
6
6
|
enumerable: true,
|
|
7
|
-
get: ()
|
|
7
|
+
get: function() {
|
|
8
|
+
return NpmScriptRunner;
|
|
9
|
+
}
|
|
8
10
|
});
|
|
9
11
|
const _fs = require("fs");
|
|
10
12
|
const _path = require("path");
|
|
11
13
|
const _promises = require("fs/promises");
|
|
12
|
-
const
|
|
14
|
+
const _child_process = require("child_process");
|
|
15
|
+
function _define_property(obj, key, value) {
|
|
16
|
+
if (key in obj) {
|
|
17
|
+
Object.defineProperty(obj, key, {
|
|
18
|
+
value: value,
|
|
19
|
+
enumerable: true,
|
|
20
|
+
configurable: true,
|
|
21
|
+
writable: true
|
|
22
|
+
});
|
|
23
|
+
} else {
|
|
24
|
+
obj[key] = value;
|
|
25
|
+
}
|
|
26
|
+
return obj;
|
|
27
|
+
}
|
|
13
28
|
class NpmScriptRunner {
|
|
14
29
|
getNpmArgs(task, taskTargs) {
|
|
15
30
|
const extraArgs = taskTargs.length > 0 ? [
|
|
@@ -77,7 +92,7 @@ class NpmScriptRunner {
|
|
|
77
92
|
target.options?.nodeOptions
|
|
78
93
|
].filter((str)=>str).join(" ");
|
|
79
94
|
await new Promise((resolve, reject)=>{
|
|
80
|
-
childProcess = (0,
|
|
95
|
+
childProcess = (0, _child_process.spawn)(npmCmd, npmRunArgs, {
|
|
81
96
|
cwd: target.cwd,
|
|
82
97
|
stdio: [
|
|
83
98
|
"inherit",
|
|
@@ -127,8 +142,9 @@ class NpmScriptRunner {
|
|
|
127
142
|
});
|
|
128
143
|
}
|
|
129
144
|
constructor(options){
|
|
145
|
+
_define_property(this, "options", void 0);
|
|
130
146
|
this.options = options;
|
|
131
147
|
this.validateOptions(options);
|
|
132
148
|
}
|
|
133
149
|
}
|
|
134
|
-
NpmScriptRunner
|
|
150
|
+
_define_property(NpmScriptRunner, "gracefulKillTimeout", 2500);
|
|
@@ -4,19 +4,34 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
});
|
|
5
5
|
Object.defineProperty(exports, "TargetRunnerPicker", {
|
|
6
6
|
enumerable: true,
|
|
7
|
-
get: ()
|
|
7
|
+
get: function() {
|
|
8
|
+
return TargetRunnerPicker;
|
|
9
|
+
}
|
|
8
10
|
});
|
|
9
|
-
const _path = /*#__PURE__*/
|
|
10
|
-
const
|
|
11
|
+
const _path = /*#__PURE__*/ _interop_require_default(require("path"));
|
|
12
|
+
const _targetgraph = require("@lage-run/target-graph");
|
|
11
13
|
const _url = require("url");
|
|
12
|
-
function
|
|
14
|
+
function _define_property(obj, key, value) {
|
|
15
|
+
if (key in obj) {
|
|
16
|
+
Object.defineProperty(obj, key, {
|
|
17
|
+
value: value,
|
|
18
|
+
enumerable: true,
|
|
19
|
+
configurable: true,
|
|
20
|
+
writable: true
|
|
21
|
+
});
|
|
22
|
+
} else {
|
|
23
|
+
obj[key] = value;
|
|
24
|
+
}
|
|
25
|
+
return obj;
|
|
26
|
+
}
|
|
27
|
+
function _interop_require_default(obj) {
|
|
13
28
|
return obj && obj.__esModule ? obj : {
|
|
14
29
|
default: obj
|
|
15
30
|
};
|
|
16
31
|
}
|
|
17
32
|
class TargetRunnerPicker {
|
|
18
33
|
async pick(target) {
|
|
19
|
-
if (target.id === (0,
|
|
34
|
+
if (target.id === (0, _targetgraph.getStartTargetId)()) {
|
|
20
35
|
return new (await import("./NoOpRunner.js")).NoOpRunner();
|
|
21
36
|
}
|
|
22
37
|
if (!target.type) {
|
|
@@ -38,6 +53,7 @@ class TargetRunnerPicker {
|
|
|
38
53
|
throw new Error(`No runner found for target ${target.id}`);
|
|
39
54
|
}
|
|
40
55
|
constructor(options){
|
|
56
|
+
_define_property(this, "options", void 0);
|
|
41
57
|
this.options = options;
|
|
42
58
|
}
|
|
43
59
|
}
|
|
@@ -4,9 +4,24 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
});
|
|
5
5
|
Object.defineProperty(exports, "WorkerRunner", {
|
|
6
6
|
enumerable: true,
|
|
7
|
-
get: ()
|
|
7
|
+
get: function() {
|
|
8
|
+
return WorkerRunner;
|
|
9
|
+
}
|
|
8
10
|
});
|
|
9
11
|
const _url = require("url");
|
|
12
|
+
function _define_property(obj, key, value) {
|
|
13
|
+
if (key in obj) {
|
|
14
|
+
Object.defineProperty(obj, key, {
|
|
15
|
+
value: value,
|
|
16
|
+
enumerable: true,
|
|
17
|
+
configurable: true,
|
|
18
|
+
writable: true
|
|
19
|
+
});
|
|
20
|
+
} else {
|
|
21
|
+
obj[key] = value;
|
|
22
|
+
}
|
|
23
|
+
return obj;
|
|
24
|
+
}
|
|
10
25
|
class WorkerRunner {
|
|
11
26
|
async shouldRun(target) {
|
|
12
27
|
const scriptModule = await this.getScriptModule(target);
|
|
@@ -42,7 +57,8 @@ class WorkerRunner {
|
|
|
42
57
|
return await import(importScript);
|
|
43
58
|
}
|
|
44
59
|
constructor(options){
|
|
60
|
+
_define_property(this, "options", void 0);
|
|
45
61
|
this.options = options;
|
|
46
62
|
}
|
|
47
63
|
}
|
|
48
|
-
WorkerRunner
|
|
64
|
+
_define_property(WorkerRunner, "gracefulKillTimeout", 2500);
|
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", {
|
|
3
3
|
value: true
|
|
4
4
|
});
|
|
5
|
-
const
|
|
5
|
+
const _createCacheProvider = require("../cache/createCacheProvider.js");
|
|
6
6
|
const _config = require("@lage-run/config");
|
|
7
|
-
const
|
|
8
|
-
const
|
|
9
|
-
const
|
|
10
|
-
const _logger = /*#__PURE__*/
|
|
11
|
-
function
|
|
7
|
+
const _workerthreadspool = require("@lage-run/worker-threads-pool");
|
|
8
|
+
const _TargetRunnerPicker = require("../runners/TargetRunnerPicker.js");
|
|
9
|
+
const _worker_threads = require("worker_threads");
|
|
10
|
+
const _logger = /*#__PURE__*/ _interop_require_default(require("@lage-run/logger"));
|
|
11
|
+
function _interop_require_default(obj) {
|
|
12
12
|
return obj && obj.__esModule ? obj : {
|
|
13
13
|
default: obj
|
|
14
14
|
};
|
|
@@ -19,41 +19,54 @@ async function setup(options) {
|
|
|
19
19
|
const logger = (0, _logger.default)();
|
|
20
20
|
logger.addReporter({
|
|
21
21
|
log (entry) {
|
|
22
|
-
|
|
22
|
+
_worker_threads.parentPort.postMessage({
|
|
23
23
|
type: "log",
|
|
24
24
|
...entry
|
|
25
25
|
});
|
|
26
26
|
},
|
|
27
27
|
summarize () {}
|
|
28
28
|
});
|
|
29
|
-
const { cacheProvider
|
|
29
|
+
const { cacheProvider } = await (0, _createCacheProvider.createCache)({
|
|
30
30
|
root,
|
|
31
31
|
logger,
|
|
32
32
|
cacheOptions: config.cacheOptions,
|
|
33
33
|
cliArgs: options.taskArgs,
|
|
34
34
|
skipLocalCache: options.skipLocalCache
|
|
35
35
|
});
|
|
36
|
-
const runnerPicker = new
|
|
36
|
+
const runnerPicker = new _TargetRunnerPicker.TargetRunnerPicker(runners);
|
|
37
37
|
return {
|
|
38
38
|
options,
|
|
39
39
|
runnerPicker,
|
|
40
|
-
cacheProvider
|
|
41
|
-
hasher
|
|
40
|
+
cacheProvider
|
|
42
41
|
};
|
|
43
42
|
}
|
|
44
43
|
(async ()=>{
|
|
45
|
-
const { cacheProvider ,
|
|
44
|
+
const { cacheProvider , runnerPicker , options } = await setup(_worker_threads.workerData);
|
|
45
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
46
|
+
let hashPromiseResolve = (_hash)=>{};
|
|
47
|
+
// main thread sends hash to worker because it keeps a global memory cache of the hashes
|
|
48
|
+
_worker_threads.parentPort.on("message", (data)=>{
|
|
49
|
+
if (data.type === "hash") {
|
|
50
|
+
hashPromiseResolve(data.hash);
|
|
51
|
+
}
|
|
52
|
+
});
|
|
46
53
|
async function getCache(target) {
|
|
47
54
|
const { shouldCache , shouldResetCache } = options;
|
|
48
55
|
let hash = undefined;
|
|
49
56
|
let cacheHit = false;
|
|
50
|
-
if (!shouldCache || !target.cache || !cacheProvider
|
|
57
|
+
if (!shouldCache || !target.cache || !cacheProvider) {
|
|
51
58
|
return {
|
|
52
59
|
hash,
|
|
53
60
|
cacheHit
|
|
54
61
|
};
|
|
55
62
|
}
|
|
56
|
-
hash
|
|
63
|
+
// using a special pattern in communicating with the main thread to get the hash for the target
|
|
64
|
+
hash = await new Promise((resolve)=>{
|
|
65
|
+
hashPromiseResolve = resolve;
|
|
66
|
+
_worker_threads.parentPort.postMessage({
|
|
67
|
+
type: "hash"
|
|
68
|
+
});
|
|
69
|
+
});
|
|
57
70
|
if (hash && !shouldResetCache) {
|
|
58
71
|
cacheHit = await cacheProvider.fetch(hash, target);
|
|
59
72
|
}
|
|
@@ -84,5 +97,5 @@ async function setup(options) {
|
|
|
84
97
|
hash
|
|
85
98
|
};
|
|
86
99
|
}
|
|
87
|
-
(0,
|
|
100
|
+
(0, _workerthreadspool.registerWorker)(run);
|
|
88
101
|
})();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lage-run/scheduler",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "1.0.0",
|
|
4
4
|
"description": "Scheduler for Lage",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -17,14 +17,15 @@
|
|
|
17
17
|
"lint": "monorepo-scripts lint"
|
|
18
18
|
},
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"@lage-run/target-graph": "^0.8.
|
|
20
|
+
"@lage-run/target-graph": "^0.8.6",
|
|
21
21
|
"@lage-run/logger": "^1.3.0",
|
|
22
|
-
"@lage-run/cache": "^
|
|
23
|
-
"@lage-run/config": "^0.2.
|
|
24
|
-
"@lage-run/
|
|
22
|
+
"@lage-run/cache": "^1.1.1",
|
|
23
|
+
"@lage-run/config": "^0.2.1",
|
|
24
|
+
"@lage-run/hasher": "^1.0.0",
|
|
25
|
+
"@lage-run/worker-threads-pool": "^0.7.0"
|
|
25
26
|
},
|
|
26
27
|
"devDependencies": {
|
|
27
|
-
"@lage-run/scheduler-types": "^0.3.
|
|
28
|
+
"@lage-run/scheduler-types": "^0.3.9",
|
|
28
29
|
"monorepo-scripts": "*"
|
|
29
30
|
},
|
|
30
31
|
"publishConfig": {
|