hammerkit 1.4.0 → 1.5.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.devcontainer/Dockerfile +5 -0
- package/.devcontainer/devcontainer.json +40 -0
- package/README.md +13 -86
- package/SUMMARY.md +43 -0
- package/build.schema.json +56 -2
- package/dist/cli.js +97 -0
- package/dist/cli.spec.js +5 -0
- package/dist/console/console-context-mock.js +11 -0
- package/dist/console/get-logger.js +23 -0
- package/dist/docker/pull.js +5 -5
- package/dist/docker/remove-container.js +21 -0
- package/dist/docker/stream.js +14 -10
- package/dist/docker/using-container.js +28 -0
- package/dist/environment/get-process-env.js +4 -4
- package/dist/environment/replace-env-variables.js +2 -3
- package/dist/executer/abort.js +13 -0
- package/dist/executer/check-readiness.js +25 -0
- package/dist/executer/create-scheduler-state.js +32 -0
- package/dist/executer/docker-node.js +123 -0
- package/dist/executer/docker-service.js +113 -0
- package/dist/executer/environment-mock.js +17 -0
- package/dist/executer/event-cache.js +165 -0
- package/dist/executer/execute-command.js +42 -0
- package/dist/executer/execute-docker.js +58 -194
- package/dist/executer/execute.spec.js +37 -37
- package/dist/executer/execution-steps.js +57 -0
- package/dist/executer/get-docker-executor.js +6 -100
- package/dist/executer/hierarchy-scheduler.js +94 -0
- package/dist/executer/kubernetes-service.js +64 -0
- package/dist/executer/label-values.js +65 -0
- package/dist/executer/local-node.js +57 -0
- package/dist/executer/print-container-options.js +17 -0
- package/dist/{logging/log-strategy.js → executer/process-item.js} +1 -1
- package/dist/executer/process-listener.js +3 -0
- package/dist/executer/process-manager.js +111 -0
- package/dist/executer/process-manager.spec.js +91 -0
- package/dist/executer/{executor.js → process.js} +1 -1
- package/dist/executer/{execute-result.js → readonly-state.js} +1 -1
- package/dist/executer/schedule.js +139 -0
- package/dist/executer/scheduler/check-for-loop.js +21 -0
- package/dist/executer/scheduler/enqueue-next.js +27 -0
- package/dist/{testing/test-context.js → executer/scheduler/node-state.js} +1 -1
- package/dist/executer/scheduler/scheduler-result.js +3 -0
- package/dist/executer/scheduler/scheduler-state.js +3 -0
- package/dist/executer/scheduler/service-state.js +3 -0
- package/dist/executer/service-dns.js +6 -0
- package/dist/executer/set-user-permission.js +16 -0
- package/dist/executer/start-node.js +59 -0
- package/dist/executer/state-listener.js +3 -0
- package/dist/executer/state.js +49 -0
- package/dist/executer/states.js +4 -159
- package/dist/executer/watch-node.js +57 -0
- package/dist/executer/watch-service.js +54 -0
- package/dist/executer/work-scope.js +10 -0
- package/dist/file/get-file-context-mock.spec.js +5 -5
- package/dist/file/get-file-context.js +41 -18
- package/dist/file/move-files.js +1 -1
- package/dist/get-work-context.js +29 -0
- package/dist/index.js +8 -4
- package/dist/log.js +192 -50
- package/dist/logging/grouped-logger.js +31 -19
- package/dist/logging/interactive-logger.js +20 -23
- package/dist/logging/live-logger.js +7 -17
- package/dist/optimizer/calculate-checksum.js +1 -1
- package/dist/optimizer/get-cache-directory.js +8 -13
- package/dist/optimizer/get-work-node-cache-stats.js +68 -21
- package/dist/optimizer/read-work-node-cache.js +2 -2
- package/dist/optimizer/work-node-cache-description.js +12 -11
- package/dist/optimizer/write-work-node-cache.js +9 -8
- package/dist/{executer/execution-context.js → parser/build-file-service.js} +1 -1
- package/dist/{optimizer → parser}/cache-method.js +0 -0
- package/dist/parser/get-build-file.js +1 -1
- package/dist/parser/parse-boolean.js +29 -0
- package/dist/parser/parse-boolean.spec.js +48 -0
- package/dist/parser/parse-build-file-references.js +2 -2
- package/dist/parser/parse-build-file-services.js +58 -0
- package/dist/parser/parse-build-file-task-command.js +6 -5
- package/dist/parser/parse-build-file-task-source.js +5 -5
- package/dist/parser/parse-build-file.js +27 -15
- package/dist/parser/parse-context.js +13 -0
- package/dist/parser/parse-envs.js +4 -3
- package/dist/parser/parse-healthcheck.js +28 -0
- package/dist/parser/parse-label-arguments.js +23 -0
- package/dist/parser/parse-service-selector.js +39 -0
- package/dist/parser/parse-string-array.js +6 -5
- package/dist/parser/parse-string-map.js +30 -0
- package/dist/parser/parse-string.js +20 -0
- package/dist/parser/read-build-file.js +16 -5
- package/dist/parser/read-env-file.js +3 -3
- package/dist/planner/utils/get-container-mounts.js +24 -0
- package/dist/planner/utils/normalize-path.js +17 -0
- package/dist/planner/utils/parse-work-node-mount.js +3 -14
- package/dist/planner/utils/parse-work-node-mount.spec.js +10 -10
- package/dist/planner/utils/parse-work-service-volume.js +21 -0
- package/dist/planner/utils/plan-work-command.js +1 -1
- package/dist/planner/utils/plan-work-dependency.js +17 -5
- package/dist/planner/utils/plan-work-node.js +203 -128
- package/dist/planner/utils/plan-work-nodes.js +59 -14
- package/dist/planner/utils/plan-work-tree.js +8 -4
- package/dist/planner/utils/plan-work-volume.js +25 -0
- package/dist/planner/validate.js +56 -8
- package/dist/planner/work-context.js +20 -0
- package/dist/planner/work-mount.js +3 -0
- package/dist/planner/work-node-id.js +4 -4
- package/dist/planner/work-node-id.spec.js +11 -8
- package/dist/planner/work-node-path.js +0 -1
- package/dist/planner/work-node-status.js +121 -18
- package/dist/planner/work-node.js +3 -3
- package/dist/planner/work-service-id.js +13 -0
- package/dist/planner/work-service-volume.js +3 -0
- package/dist/planner/work-service.js +6 -0
- package/dist/planner/work-services.js +3 -0
- package/dist/planner/work-volume.js +3 -0
- package/dist/program.js +155 -90
- package/dist/program.spec.js +20 -0
- package/dist/start-watch-processes.js +28 -0
- package/dist/testing/cli.js +73 -0
- package/dist/testing/create-build-file.js +1 -1
- package/dist/testing/example-test-suite.js +54 -0
- package/dist/testing/expect.js +30 -18
- package/dist/testing/get-logger.js +23 -0
- package/dist/testing/get-node.js +2 -2
- package/dist/testing/get-test-context.js +5 -3
- package/dist/testing/get-test-suite.js +2 -50
- package/dist/testing/integration/cache.spec.js +25 -28
- package/dist/testing/integration/cancellation.spec.js +19 -21
- package/dist/testing/integration/clean.spec.js +25 -28
- package/dist/testing/integration/cmd.spec.js +5 -8
- package/dist/testing/integration/concurrency.spec.js +8 -17
- package/dist/testing/integration/docker.spec.js +6 -9
- package/dist/testing/integration/env.spec.js +27 -34
- package/dist/testing/integration/error.spec.js +5 -9
- package/dist/testing/integration/extend.spec.js +32 -0
- package/dist/testing/integration/glob.spec.js +35 -28
- package/dist/testing/integration/include.spec.js +17 -16
- package/dist/testing/integration/invalid.spec.js +3 -2
- package/dist/testing/integration/invalid_loop.spec.js +13 -0
- package/dist/testing/integration/kubernetes.spec.js +15 -0
- package/dist/testing/integration/labels.spec.js +47 -0
- package/dist/testing/integration/local.spec.js +4 -7
- package/dist/testing/integration/monorepo.spec.js +5 -9
- package/dist/testing/integration/multi-line.spec.js +4 -4
- package/dist/testing/integration/program.spec.js +3 -10
- package/dist/testing/integration/reference.spec.js +10 -13
- package/dist/testing/integration/services.spec.js +14 -0
- package/dist/testing/integration/store.spec.js +42 -38
- package/dist/testing/integration/unknown.spec.js +4 -4
- package/dist/testing/integration/validate.spec.js +4 -4
- package/dist/testing/integration/watch.spec.js +34 -33
- package/dist/testing/lazy.js +17 -0
- package/dist/testing/start-watch-processes.js +29 -0
- package/dist/testing/test-suite-setup.js +3 -0
- package/dist/utils/abort-event.js +14 -1
- package/dist/utils/abortable-function.js +71 -0
- package/dist/utils/debouncer.spec.js +8 -8
- package/dist/utils/environment-config.js +13 -0
- package/dist/utils/fail-never.js +1 -1
- package/dist/utils/sleep.spec.js +3 -3
- package/package.json +24 -21
- package/dist/console/get-console-context-mock.js +0 -33
- package/dist/executer/clean.js +0 -17
- package/dist/executer/execute-local.js +0 -59
- package/dist/executer/execute.js +0 -139
- package/dist/executer/execution-context-event.js +0 -3
- package/dist/executer/execution-context-mock.js +0 -3
- package/dist/executer/executor-mock.js +0 -61
- package/dist/executer/get-environment-mock.js +0 -16
- package/dist/executer/get-execution-context-mock.js +0 -18
- package/dist/executer/get-local-executor.js +0 -56
- package/dist/executer/restore.js +0 -19
- package/dist/executer/store.js +0 -19
- package/dist/logging/get-logger.js +0 -23
- package/dist/logging/message-to-console.js +0 -16
- package/dist/optimizer/optimize.js +0 -32
- package/dist/optimizer/work-node-description.js +0 -12
- package/dist/planner/utils/find-build-task.js +0 -18
- package/dist/testing/create-work-tree.js +0 -11
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.kubernetesService = void 0;
|
|
4
|
+
const child_process_1 = require("child_process");
|
|
5
|
+
const log_1 = require("../log");
|
|
6
|
+
const abort_event_1 = require("../utils/abort-event");
|
|
7
|
+
const sleep_1 = require("../utils/sleep");
|
|
8
|
+
function kubernetesService(service, stateKey, state, env) {
|
|
9
|
+
return async (abort) => {
|
|
10
|
+
const status = env.status.service(service);
|
|
11
|
+
do {
|
|
12
|
+
await startForward(service, stateKey, state, status, abort);
|
|
13
|
+
if (!abort.signal.aborted) {
|
|
14
|
+
await (0, sleep_1.sleep)(1000);
|
|
15
|
+
}
|
|
16
|
+
} while (!abort.signal.aborted);
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
exports.kubernetesService = kubernetesService;
|
|
20
|
+
function startForward(service, stateKey, state, status, abort) {
|
|
21
|
+
const cmd = `kubectl port-forward ${service.selector.type}/${service.selector.name} --kubeconfig ${service.kubeconfig} --context ${service.context} ${service.ports.map((p) => `${p.hostPort}:${p.containerPort}`).join(' ')}`;
|
|
22
|
+
const ps = (0, child_process_1.exec)(cmd, {});
|
|
23
|
+
return new Promise((resolve) => {
|
|
24
|
+
ps.stdout?.on('data', async (data) => {
|
|
25
|
+
for (const log of (0, log_1.getLogs)(data)) {
|
|
26
|
+
status.console('stdout', log);
|
|
27
|
+
}
|
|
28
|
+
state.patchService({
|
|
29
|
+
service,
|
|
30
|
+
type: 'running',
|
|
31
|
+
dns: { host: 'host-gateway' },
|
|
32
|
+
stateKey,
|
|
33
|
+
});
|
|
34
|
+
});
|
|
35
|
+
ps.stderr?.on('data', async (data) => {
|
|
36
|
+
for (const log of (0, log_1.getLogs)(data)) {
|
|
37
|
+
status.console('stderr', log);
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
ps.on('error', (err) => {
|
|
41
|
+
status.write('error', (0, log_1.getErrorMessage)(err));
|
|
42
|
+
state.patchService({
|
|
43
|
+
service,
|
|
44
|
+
type: 'end',
|
|
45
|
+
reason: 'crash',
|
|
46
|
+
stateKey,
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
ps.on('close', (code) => {
|
|
50
|
+
status.write('info', `exit with ${code}`);
|
|
51
|
+
state.patchService({
|
|
52
|
+
service,
|
|
53
|
+
type: 'end',
|
|
54
|
+
reason: 'crash',
|
|
55
|
+
stateKey,
|
|
56
|
+
});
|
|
57
|
+
resolve();
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
(0, abort_event_1.listenOnAbort)(abort.signal, () => {
|
|
61
|
+
ps.kill();
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=kubernetes-service.js.map
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.matchesAnyLabel = exports.hasDependencyWithMatch = exports.hasNeedsWithMatch = exports.matchesAllLabels = exports.hasLabels = void 0;
|
|
4
|
+
function hasLabels(labels) {
|
|
5
|
+
return Object.keys(labels).length > 0;
|
|
6
|
+
}
|
|
7
|
+
exports.hasLabels = hasLabels;
|
|
8
|
+
function matchesAllLabels(filterLabels, node, nodes) {
|
|
9
|
+
for (const [labelKey, filterValues] of Object.entries(filterLabels)) {
|
|
10
|
+
const nodeValues = node.labels[labelKey];
|
|
11
|
+
if (nodeValues === null || nodeValues === undefined || nodeValues.length === 0) {
|
|
12
|
+
if (!hasNeedsWithMatch(node, nodes, (depNode) => matchesAllLabels(filterLabels, depNode, nodes))) {
|
|
13
|
+
return false;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
for (const nodeValue of nodeValues || []) {
|
|
17
|
+
if (filterValues.indexOf(nodeValue) === -1) {
|
|
18
|
+
if (!hasNeedsWithMatch(node, nodes, (depNode) => matchesAllLabels(filterLabels, depNode, nodes))) {
|
|
19
|
+
return false;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
26
|
+
exports.matchesAllLabels = matchesAllLabels;
|
|
27
|
+
function hasNeedsWithMatch(root, nodes, matcher) {
|
|
28
|
+
for (const node of Object.values(nodes)) {
|
|
29
|
+
if (node.deps.some((d) => d.id === root.id)) {
|
|
30
|
+
if (matcher(node)) {
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return false;
|
|
36
|
+
}
|
|
37
|
+
exports.hasNeedsWithMatch = hasNeedsWithMatch;
|
|
38
|
+
function hasDependencyWithMatch(root, matcher) {
|
|
39
|
+
for (const dep of root.deps) {
|
|
40
|
+
if (matcher(dep)) {
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
exports.hasDependencyWithMatch = hasDependencyWithMatch;
|
|
47
|
+
function matchesAnyLabel(filterLabels, node) {
|
|
48
|
+
if (!hasLabels(filterLabels)) {
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
for (const [labelKey, filterValues] of Object.entries(filterLabels)) {
|
|
52
|
+
const nodeValues = node.labels[labelKey];
|
|
53
|
+
if (nodeValues === null || nodeValues === undefined || nodeValues.length === 0) {
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
for (const nodeValue of nodeValues) {
|
|
57
|
+
if (filterValues.indexOf(nodeValue) >= 0) {
|
|
58
|
+
return true;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return hasDependencyWithMatch(node, (depNode) => matchesAnyLabel(filterLabels, depNode));
|
|
63
|
+
}
|
|
64
|
+
exports.matchesAnyLabel = matchesAnyLabel;
|
|
65
|
+
//# sourceMappingURL=label-values.js.map
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.localNode = void 0;
|
|
4
|
+
const abort_1 = require("./abort");
|
|
5
|
+
const execute_command_1 = require("./execute-command");
|
|
6
|
+
const write_work_node_cache_1 = require("../optimizer/write-work-node-cache");
|
|
7
|
+
const log_1 = require("../log");
|
|
8
|
+
const states_1 = require("./states");
|
|
9
|
+
function localNode(node, stateKey, state, environment) {
|
|
10
|
+
return async (abort, started) => {
|
|
11
|
+
const status = environment.status.task(node);
|
|
12
|
+
status.write('info', `execute ${node.name} locally`);
|
|
13
|
+
try {
|
|
14
|
+
for (const cmd of node.cmds) {
|
|
15
|
+
(0, abort_1.checkForAbort)(abort.signal);
|
|
16
|
+
status.write('info', `execute cmd "${cmd.cmd}" locally`);
|
|
17
|
+
const exitCode = await (0, execute_command_1.executeCommand)(status, abort.signal, cmd.path, cmd.cmd, node.envs);
|
|
18
|
+
if (exitCode !== 0) {
|
|
19
|
+
state.patchNode({
|
|
20
|
+
type: 'crash',
|
|
21
|
+
node,
|
|
22
|
+
exitCode,
|
|
23
|
+
stateKey,
|
|
24
|
+
}, stateKey);
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
await (0, write_work_node_cache_1.writeWorkNodeCache)(node, environment);
|
|
29
|
+
state.patchNode({
|
|
30
|
+
type: 'completed',
|
|
31
|
+
node: node,
|
|
32
|
+
cached: false,
|
|
33
|
+
stateKey,
|
|
34
|
+
duration: (0, states_1.getDuration)(started),
|
|
35
|
+
}, stateKey);
|
|
36
|
+
}
|
|
37
|
+
catch (e) {
|
|
38
|
+
if (e instanceof abort_1.AbortError) {
|
|
39
|
+
state.patchNode({
|
|
40
|
+
type: 'canceled',
|
|
41
|
+
node,
|
|
42
|
+
stateKey,
|
|
43
|
+
}, stateKey);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
state.patchNode({
|
|
47
|
+
type: 'error',
|
|
48
|
+
node,
|
|
49
|
+
stateKey,
|
|
50
|
+
errorMessage: (0, log_1.getErrorMessage)(e),
|
|
51
|
+
}, stateKey);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
exports.localNode = localNode;
|
|
57
|
+
//# sourceMappingURL=local-node.js.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.printContainerOptions = void 0;
|
|
4
|
+
function printContainerOptions(status, containerOptions) {
|
|
5
|
+
status.write('debug', `create container with image ${containerOptions.Image} with ${containerOptions.Entrypoint}`);
|
|
6
|
+
for (const mount of containerOptions.HostConfig?.Binds || []) {
|
|
7
|
+
status.write('debug', `bind ${mount}`);
|
|
8
|
+
}
|
|
9
|
+
for (const mount of containerOptions.HostConfig?.Binds || []) {
|
|
10
|
+
status.write('debug', `bind ${mount}`);
|
|
11
|
+
}
|
|
12
|
+
for (const link of containerOptions.HostConfig?.Links || []) {
|
|
13
|
+
status.write('debug', `link ${link}`);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
exports.printContainerOptions = printContainerOptions;
|
|
17
|
+
//# sourceMappingURL=print-container-options.js.map
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ProcessManager = void 0;
|
|
4
|
+
const abort_event_1 = require("../utils/abort-event");
|
|
5
|
+
const log_1 = require("../log");
|
|
6
|
+
class ProcessManager {
|
|
7
|
+
constructor(env, workerLimit) {
|
|
8
|
+
this.env = env;
|
|
9
|
+
this.workerLimit = workerLimit;
|
|
10
|
+
this.processes = [];
|
|
11
|
+
this.pendingProcesses = [];
|
|
12
|
+
this.listeners = [];
|
|
13
|
+
}
|
|
14
|
+
on(listener) {
|
|
15
|
+
this.listeners.push(listener);
|
|
16
|
+
}
|
|
17
|
+
onComplete() {
|
|
18
|
+
if (this.processes.length === 0 && this.pendingProcesses.length === 0) {
|
|
19
|
+
return Promise.resolve();
|
|
20
|
+
}
|
|
21
|
+
return new Promise((resolve) => {
|
|
22
|
+
this.listeners.push((evt, processes, pending) => {
|
|
23
|
+
if (processes.length === 0 && pending.length === 0) {
|
|
24
|
+
resolve();
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
abort(context) {
|
|
30
|
+
this.pendingProcesses = this.pendingProcesses.filter((p) => p.context.id !== context.id);
|
|
31
|
+
const processes = this.processes.filter((p) => p.context.id === context.id);
|
|
32
|
+
for (const process of processes) {
|
|
33
|
+
process.abortController.abort();
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
background(context, process) {
|
|
37
|
+
const abortController = new AbortController();
|
|
38
|
+
(0, abort_event_1.listenOnAbort)(this.env.abortCtrl.signal, () => {
|
|
39
|
+
abortController.abort();
|
|
40
|
+
});
|
|
41
|
+
this.startProcess({ process, context, abortController });
|
|
42
|
+
return abortController;
|
|
43
|
+
}
|
|
44
|
+
task(context, process) {
|
|
45
|
+
const abortController = new AbortController();
|
|
46
|
+
(0, abort_event_1.listenOnAbort)(this.env.abortCtrl.signal, () => {
|
|
47
|
+
abortController.abort();
|
|
48
|
+
});
|
|
49
|
+
const hasFreeWorker = this.workerLimit === 0 || this.workerLimit > this.processes.length;
|
|
50
|
+
if (hasFreeWorker) {
|
|
51
|
+
const hasBlockingProcess = this.processes.some((p) => p.context.id === context.id);
|
|
52
|
+
if (hasBlockingProcess) {
|
|
53
|
+
this.abort(context);
|
|
54
|
+
this.pendingProcesses.push({ context, process, abortController });
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
this.startProcess({ process, context, abortController });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
this.pendingProcesses.push({ context, process, abortController });
|
|
62
|
+
}
|
|
63
|
+
return abortController;
|
|
64
|
+
}
|
|
65
|
+
enqueuePendingProcesses() {
|
|
66
|
+
const nextProcess = this.pendingProcesses[0];
|
|
67
|
+
if (nextProcess) {
|
|
68
|
+
const hasBlockingProcess = this.processes.some((p) => p.context.id === nextProcess.context.id);
|
|
69
|
+
if (hasBlockingProcess) {
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
const pendingProcess = this.pendingProcesses.shift();
|
|
74
|
+
if (pendingProcess) {
|
|
75
|
+
this.startProcess(pendingProcess);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
startProcess(pendingProcess) {
|
|
79
|
+
const started = new Date();
|
|
80
|
+
const item = {
|
|
81
|
+
context: pendingProcess.context,
|
|
82
|
+
started,
|
|
83
|
+
abortController: pendingProcess.abortController,
|
|
84
|
+
promise: pendingProcess
|
|
85
|
+
.process(pendingProcess.abortController, started)
|
|
86
|
+
.catch((err) => {
|
|
87
|
+
this.env.status.context(pendingProcess.context).write('error', (0, log_1.getErrorMessage)(err));
|
|
88
|
+
})
|
|
89
|
+
.finally(() => {
|
|
90
|
+
const index = this.processes.indexOf(item);
|
|
91
|
+
this.processes.splice(index, 1);
|
|
92
|
+
this.notifyListeners({ context: item.context, type: 'ended' });
|
|
93
|
+
this.enqueuePendingProcesses();
|
|
94
|
+
}),
|
|
95
|
+
};
|
|
96
|
+
this.processes.push(item);
|
|
97
|
+
this.notifyListeners({ context: item.context, type: 'started' });
|
|
98
|
+
}
|
|
99
|
+
notifyListeners(evt) {
|
|
100
|
+
for (const listener of this.listeners) {
|
|
101
|
+
try {
|
|
102
|
+
listener(evt, this.processes, this.pendingProcesses.map((p) => ({ context: p.context })));
|
|
103
|
+
}
|
|
104
|
+
catch (e) {
|
|
105
|
+
this.env.status.context(evt.context).write('error', (0, log_1.getErrorMessage)(e));
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
exports.ProcessManager = ProcessManager;
|
|
111
|
+
//# sourceMappingURL=process-manager.js.map
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const process_manager_1 = require("./process-manager");
|
|
4
|
+
const environment_mock_1 = require("./environment-mock");
|
|
5
|
+
const abort_event_1 = require("../utils/abort-event");
|
|
6
|
+
describe('process-manager', () => {
|
|
7
|
+
it('should complete on success', async () => {
|
|
8
|
+
const manager = new process_manager_1.ProcessManager((0, environment_mock_1.environmentMock)(process.cwd()), 0);
|
|
9
|
+
let currentIndex = 0;
|
|
10
|
+
const expectedEvents = ['started', 'ended'];
|
|
11
|
+
manager.on((evt) => {
|
|
12
|
+
expect(evt.type).toBe(expectedEvents[currentIndex++]);
|
|
13
|
+
});
|
|
14
|
+
manager.task({ type: 'task', name: 'test-success', id: 'a' }, () => new Promise((resolve) => setTimeout(() => resolve(), 100)));
|
|
15
|
+
await manager.onComplete();
|
|
16
|
+
expect(currentIndex).toBe(2);
|
|
17
|
+
});
|
|
18
|
+
it('should complete on error', async () => {
|
|
19
|
+
const manager = new process_manager_1.ProcessManager((0, environment_mock_1.environmentMock)(process.cwd()), 0);
|
|
20
|
+
let currentIndex = 0;
|
|
21
|
+
const expectedEvents = ['started', 'ended'];
|
|
22
|
+
manager.on((evt) => {
|
|
23
|
+
expect(evt.type).toBe(expectedEvents[currentIndex++]);
|
|
24
|
+
});
|
|
25
|
+
manager.task({ type: 'task', name: 'test-error', id: 'a' }, () => new Promise((resolve, reject) => setTimeout(() => reject(new Error('3')), 100)));
|
|
26
|
+
await manager.onComplete();
|
|
27
|
+
expect(currentIndex).toBe(2);
|
|
28
|
+
});
|
|
29
|
+
it('should not start more than worker count', async () => {
|
|
30
|
+
const manager = new process_manager_1.ProcessManager((0, environment_mock_1.environmentMock)(process.cwd()), 1);
|
|
31
|
+
const processes = [];
|
|
32
|
+
let concurrencyCount = 0;
|
|
33
|
+
let hasOversteppedLimits = false;
|
|
34
|
+
manager.on((evt) => {
|
|
35
|
+
if (evt.type === 'started') {
|
|
36
|
+
concurrencyCount++;
|
|
37
|
+
}
|
|
38
|
+
else if (evt.type === 'ended') {
|
|
39
|
+
concurrencyCount--;
|
|
40
|
+
processes.push(evt.context.id);
|
|
41
|
+
}
|
|
42
|
+
if (concurrencyCount > 1) {
|
|
43
|
+
hasOversteppedLimits = true;
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
manager.task({ type: 'task', name: 'test-success', id: 'a' }, () => new Promise((resolve) => setTimeout(() => resolve(), 100)));
|
|
47
|
+
manager.task({ type: 'task', name: 'test-success', id: 'b' }, () => new Promise((resolve) => setTimeout(() => resolve(), 100)));
|
|
48
|
+
manager.task({ type: 'task', name: 'test-success', id: 'c' }, () => new Promise((resolve) => setTimeout(() => resolve(), 100)));
|
|
49
|
+
await manager.onComplete();
|
|
50
|
+
expect(processes).toEqual(['a', 'b', 'c']);
|
|
51
|
+
expect(hasOversteppedLimits).toBeFalsy();
|
|
52
|
+
expect(concurrencyCount).toBe(0);
|
|
53
|
+
});
|
|
54
|
+
it('should terminate on abort', async () => {
|
|
55
|
+
const env = (0, environment_mock_1.environmentMock)(process.cwd());
|
|
56
|
+
const manager = new process_manager_1.ProcessManager(env, 0);
|
|
57
|
+
manager.task({ type: 'task', name: 'test-success', id: 'a' }, (abort) => new Promise((resolve) => {
|
|
58
|
+
(0, abort_event_1.listenOnAbort)(abort.signal, resolve);
|
|
59
|
+
}));
|
|
60
|
+
setTimeout(() => {
|
|
61
|
+
env.abortCtrl.abort();
|
|
62
|
+
}, 200);
|
|
63
|
+
await manager.onComplete();
|
|
64
|
+
});
|
|
65
|
+
it('should abort if task with same id is already started', async () => {
|
|
66
|
+
const manager = new process_manager_1.ProcessManager((0, environment_mock_1.environmentMock)(process.cwd()), 0);
|
|
67
|
+
const processes = [];
|
|
68
|
+
let concurrencyCount = 0;
|
|
69
|
+
let hasOversteppedLimits = false;
|
|
70
|
+
manager.on((evt) => {
|
|
71
|
+
if (evt.type === 'started') {
|
|
72
|
+
concurrencyCount++;
|
|
73
|
+
}
|
|
74
|
+
else if (evt.type === 'ended') {
|
|
75
|
+
concurrencyCount--;
|
|
76
|
+
processes.push(evt.context.id);
|
|
77
|
+
}
|
|
78
|
+
if (concurrencyCount > 1) {
|
|
79
|
+
hasOversteppedLimits = true;
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
const abortController = manager.task({ type: 'task', name: 'test-success', id: 'a' }, () => new Promise((resolve) => setTimeout(() => resolve(), 200)));
|
|
83
|
+
manager.task({ type: 'task', name: 'test-success', id: 'a' }, () => new Promise((resolve) => setTimeout(() => resolve(), 100)));
|
|
84
|
+
expect(abortController.signal.aborted).toBeTruthy();
|
|
85
|
+
await manager.onComplete();
|
|
86
|
+
expect(processes).toEqual(['a', 'a']);
|
|
87
|
+
expect(hasOversteppedLimits).toBeFalsy();
|
|
88
|
+
expect(concurrencyCount).toBe(0);
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
//# sourceMappingURL=process-manager.spec.js.map
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.schedule = void 0;
|
|
4
|
+
const work_service_1 = require("../planner/work-service");
|
|
5
|
+
const docker_service_1 = require("./docker-service");
|
|
6
|
+
const kubernetes_service_1 = require("./kubernetes-service");
|
|
7
|
+
const work_node_1 = require("../planner/work-node");
|
|
8
|
+
const docker_node_1 = require("./docker-node");
|
|
9
|
+
const local_node_1 = require("./local-node");
|
|
10
|
+
const work_node_status_1 = require("../planner/work-node-status");
|
|
11
|
+
const start_watch_processes_1 = require("../start-watch-processes");
|
|
12
|
+
const start_node_1 = require("./start-node");
|
|
13
|
+
async function schedule(processManager, state, environment) {
|
|
14
|
+
if (state.current.watch) {
|
|
15
|
+
(0, start_watch_processes_1.startWatchProcesses)(state, processManager, environment);
|
|
16
|
+
}
|
|
17
|
+
state.on((currentState) => {
|
|
18
|
+
for (const [, nodeState] of Object.entries(currentState.node)) {
|
|
19
|
+
if (nodeState.type === 'pending') {
|
|
20
|
+
const hasOpenDeps = nodeState.node.deps.some((dep) => currentState.node[dep.id].type !== 'completed');
|
|
21
|
+
if (hasOpenDeps) {
|
|
22
|
+
continue;
|
|
23
|
+
}
|
|
24
|
+
state.patchNode({
|
|
25
|
+
type: 'starting',
|
|
26
|
+
node: nodeState.node,
|
|
27
|
+
started: new Date(),
|
|
28
|
+
stateKey: null,
|
|
29
|
+
}, nodeState.stateKey);
|
|
30
|
+
processManager.background({
|
|
31
|
+
type: 'task',
|
|
32
|
+
name: nodeState.node.name,
|
|
33
|
+
id: nodeState.node.id + '-cache',
|
|
34
|
+
}, async () => {
|
|
35
|
+
// TODO abort improvement
|
|
36
|
+
await (0, start_node_1.startNode)(nodeState, state, environment);
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
else if (nodeState.type === 'ready') {
|
|
40
|
+
const endedNeeds = nodeState.node.needs
|
|
41
|
+
.map((need) => currentState.service[need.id])
|
|
42
|
+
.filter((service) => service.type === 'end' || service.type === 'canceled');
|
|
43
|
+
if (endedNeeds.length > 0) {
|
|
44
|
+
state.patchNode({
|
|
45
|
+
type: 'error',
|
|
46
|
+
node: nodeState.node,
|
|
47
|
+
stateKey: nodeState.stateKey,
|
|
48
|
+
errorMessage: endedNeeds
|
|
49
|
+
.map((n) => `service ${n.service.name} ${n.type === 'end' ? 'has ended ' + n.reason : 'was canceled'}`)
|
|
50
|
+
.join(', '),
|
|
51
|
+
}, nodeState.stateKey);
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
const pendingNeeds = nodeState.node.needs
|
|
55
|
+
.map((need) => currentState.service[need.id])
|
|
56
|
+
.filter((service) => service.type === 'pending');
|
|
57
|
+
if (pendingNeeds.length > 0) {
|
|
58
|
+
for (const pendingNeed of pendingNeeds) {
|
|
59
|
+
state.patchService({
|
|
60
|
+
type: 'starting',
|
|
61
|
+
service: pendingNeed.service,
|
|
62
|
+
stateKey: null,
|
|
63
|
+
});
|
|
64
|
+
processManager.background({
|
|
65
|
+
type: 'service',
|
|
66
|
+
name: pendingNeed.service.name,
|
|
67
|
+
id: pendingNeed.service.id + '-cache',
|
|
68
|
+
}, async () => {
|
|
69
|
+
// TODO abort improvement
|
|
70
|
+
await (0, start_node_1.startService)(pendingNeed.service, state, environment);
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
const hasNotReadyNeeds = nodeState.node.needs.some((need) => currentState.service[need.id].type !== 'running');
|
|
76
|
+
if (hasNotReadyNeeds) {
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
const serviceContainers = {};
|
|
80
|
+
for (const need of nodeState.node.needs) {
|
|
81
|
+
const serviceState = currentState.service[need.id];
|
|
82
|
+
if (serviceState.type === 'running') {
|
|
83
|
+
serviceContainers[need.id] = serviceState.dns;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
state.patchNode({
|
|
87
|
+
type: 'running',
|
|
88
|
+
node: nodeState.node,
|
|
89
|
+
stateKey: nodeState.stateKey,
|
|
90
|
+
started: nodeState.started,
|
|
91
|
+
}, nodeState.stateKey);
|
|
92
|
+
const ctx = (0, work_node_status_1.logContext)('task', nodeState.node);
|
|
93
|
+
if ((0, work_node_1.isContainerWorkNode)(nodeState.node)) {
|
|
94
|
+
processManager.task(ctx, (0, docker_node_1.dockerNode)(nodeState.node, nodeState.stateKey, serviceContainers, state, environment));
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
processManager.task(ctx, (0, local_node_1.localNode)(nodeState.node, nodeState.stateKey, state, environment));
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
for (const [serviceId, serviceState] of Object.entries(currentState.service)) {
|
|
102
|
+
if (serviceState.type === 'ready') {
|
|
103
|
+
const ctx = (0, work_node_status_1.logContext)('service', serviceState.service);
|
|
104
|
+
state.patchService({
|
|
105
|
+
type: 'starting',
|
|
106
|
+
service: serviceState.service,
|
|
107
|
+
stateKey: serviceState.stateKey,
|
|
108
|
+
});
|
|
109
|
+
processManager.background(ctx, (0, work_service_1.isContainerWorkService)(serviceState.service)
|
|
110
|
+
? (0, docker_service_1.dockerService)(serviceState.service, serviceState.stateKey, state, environment)
|
|
111
|
+
: (0, kubernetes_service_1.kubernetesService)(serviceState.service, serviceState.stateKey, state, environment));
|
|
112
|
+
}
|
|
113
|
+
if (serviceState.type !== 'ready' && serviceState.type !== 'running') {
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
let hasNeed = false;
|
|
117
|
+
for (const nodeState of Object.values(currentState.node)) {
|
|
118
|
+
if (nodeState.type === 'running' || nodeState.type === 'starting' || nodeState.type === 'pending') {
|
|
119
|
+
if (nodeState.node.needs.some((n) => n.id === serviceId)) {
|
|
120
|
+
hasNeed = true;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (!hasNeed) {
|
|
125
|
+
const ctx = (0, work_node_status_1.logContext)('service', serviceState.service);
|
|
126
|
+
environment.status.service(serviceState.service).write('info', 'stop unused service');
|
|
127
|
+
processManager.abort(ctx);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
await processManager.onComplete();
|
|
132
|
+
const success = !Object.values(state.current.node).some((n) => n.type !== 'completed');
|
|
133
|
+
return {
|
|
134
|
+
state: state.current,
|
|
135
|
+
success,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
exports.schedule = schedule;
|
|
139
|
+
//# sourceMappingURL=schedule.js.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.checkForLoop = void 0;
|
|
4
|
+
const plan_work_nodes_1 = require("../../planner/utils/plan-work-nodes");
|
|
5
|
+
const validate_1 = require("../../planner/validate");
|
|
6
|
+
function checkForLoop(state) {
|
|
7
|
+
for (const nodeState of (0, plan_work_nodes_1.iterateWorkNodes)(state.node)) {
|
|
8
|
+
const cyclePath = (0, validate_1.hasCycle)(nodeState.node, []);
|
|
9
|
+
if (cyclePath && cyclePath.length > 0) {
|
|
10
|
+
const errorMessage = `task cycle detected ${cyclePath.map((n) => n.name).join(' -> ')}`;
|
|
11
|
+
state.node[nodeState.node.id] = {
|
|
12
|
+
type: 'error',
|
|
13
|
+
node: nodeState.node,
|
|
14
|
+
stateKey: null,
|
|
15
|
+
errorMessage,
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
exports.checkForLoop = checkForLoop;
|
|
21
|
+
//# sourceMappingURL=check-for-loop.js.map
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.checkCacheState = void 0;
|
|
4
|
+
const read_work_node_cache_1 = require("../../optimizer/read-work-node-cache");
|
|
5
|
+
const get_work_node_cache_stats_1 = require("../../optimizer/get-work-node-cache-stats");
|
|
6
|
+
async function checkCacheState(node, defaultCacheMethod, environment) {
|
|
7
|
+
const status = environment.status.task(node);
|
|
8
|
+
const caching = node.caching ?? defaultCacheMethod;
|
|
9
|
+
const currentStats = await (0, get_work_node_cache_stats_1.getWorkNodeCacheStats)(node, environment);
|
|
10
|
+
const stateKey = (0, get_work_node_cache_stats_1.getStateKey)(currentStats, caching);
|
|
11
|
+
if (caching === 'none') {
|
|
12
|
+
status.write('debug', `${node.name} is skipping cache check, because caching is disabled`);
|
|
13
|
+
return { cached: false, stateKey };
|
|
14
|
+
}
|
|
15
|
+
const cache = await (0, read_work_node_cache_1.readCache)(node, environment);
|
|
16
|
+
if (cache === null) {
|
|
17
|
+
status.write('debug', `${node.name} is skipping cache check, because there was none found`);
|
|
18
|
+
return { cached: false, stateKey };
|
|
19
|
+
}
|
|
20
|
+
const cacheKey = (0, get_work_node_cache_stats_1.getStateKey)(cache, caching);
|
|
21
|
+
return {
|
|
22
|
+
cached: cacheKey === stateKey,
|
|
23
|
+
stateKey,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
exports.checkCacheState = checkCacheState;
|
|
27
|
+
//# sourceMappingURL=enqueue-next.js.map
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.setUserPermission = void 0;
|
|
4
|
+
const execute_docker_1 = require("./execute-docker");
|
|
5
|
+
async function setUserPermission(directory, status, environment, container, user) {
|
|
6
|
+
status.write('debug', `set permission on ${directory}`);
|
|
7
|
+
const result = await (0, execute_docker_1.execCommand)(status, environment, container, '/', ['chown', user, directory], null, undefined, undefined);
|
|
8
|
+
if (result.type === 'canceled') {
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
if (result.type === 'timeout' || result.result.ExitCode !== 0) {
|
|
12
|
+
status.write('warn', `unable to set permissions for ${directory}`);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
exports.setUserPermission = setUserPermission;
|
|
16
|
+
//# sourceMappingURL=set-user-permission.js.map
|