hammerkit 1.24.0 → 1.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/optimizer/get-cache-directory.js +5 -0
- package/dist/planner/work-secret.js +6 -2
- package/dist/program.js +2 -0
- package/dist/remote/remote-commands.js +16 -12
- package/dist/remote/remote-work-commands.js +75 -3
- package/dist/schema/reference-parser.js +8 -2
- package/dist/secrets/provider-values.js +4 -0
- package/dist/server/accounts-api.js +42 -0
- package/dist/server/approvals-api.js +11 -6
- package/dist/server/definitions-api.js +65 -21
- package/dist/server/definitions.js +54 -142
- package/dist/server/executor.js +180 -181
- package/dist/server/git-checkout.js +55 -3
- package/dist/server/plan.js +296 -0
- package/dist/server/policy.js +98 -50
- package/dist/server/proposals-api.js +30 -8
- package/dist/server/proposals.js +50 -3
- package/dist/server/repo-access.js +127 -0
- package/dist/server/repo.js +94 -0
- package/dist/server/run-environment.js +20 -0
- package/dist/server/run-plan.js +44 -0
- package/dist/server/run-request.js +23 -5
- package/dist/server/run-store.js +61 -0
- package/dist/server/run-worker-command.js +17 -0
- package/dist/server/run-worker-main.js +7 -0
- package/dist/server/run-worker.js +179 -0
- package/dist/server/runs.js +189 -36
- package/dist/server/server-command.js +18 -0
- package/dist/server/server-config.js +203 -42
- package/dist/server/server.js +148 -52
- package/dist/server/sources-api.js +1 -1
- package/package.json +1 -1
|
@@ -2,152 +2,24 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.DefinitionStore = exports.inspectBundle = exports.DefinitionError = void 0;
|
|
4
4
|
const promises_1 = require("fs/promises");
|
|
5
|
-
const os_1 = require("os");
|
|
6
5
|
const path_1 = require("path");
|
|
7
6
|
const crypto_1 = require("crypto");
|
|
8
|
-
const yaml_1 = require("yaml");
|
|
9
|
-
const get_file_context_1 = require("../file/get-file-context");
|
|
10
|
-
const resolve_git_source_1 = require("../includes/resolve-git-source");
|
|
11
|
-
const schema_parser_1 = require("../schema/schema-parser");
|
|
12
|
-
const reference_parser_1 = require("../schema/reference-parser");
|
|
13
|
-
const work_scope_parser_1 = require("../schema/work-scope-parser");
|
|
14
|
-
const work_task_1 = require("../planner/work-task");
|
|
15
7
|
const bundle_1 = require("../remote/bundle");
|
|
16
|
-
const
|
|
17
|
-
|
|
18
|
-
}
|
|
19
|
-
exports.
|
|
20
|
-
// 40 (sha-1) or 64 (sha-256) hex digits
|
|
21
|
-
const COMMIT_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/i;
|
|
22
|
-
// A hash alone can only name what is inside the bundle, so everything a build
|
|
23
|
-
// file pulls in from elsewhere has to be pinned: a git include needs a commit
|
|
24
|
-
// and a remote that is not a path on the server.
|
|
25
|
-
function findLooseIncludes(bundle) {
|
|
26
|
-
const problems = [];
|
|
27
|
-
for (const [path, content] of Object.entries(bundle.files)) {
|
|
28
|
-
if (!/\.ya?ml$/i.test(path)) {
|
|
29
|
-
continue;
|
|
30
|
-
}
|
|
31
|
-
let document;
|
|
32
|
-
try {
|
|
33
|
-
document = (0, yaml_1.parse)(content);
|
|
34
|
-
}
|
|
35
|
-
catch {
|
|
36
|
-
continue; // a file that isn't valid yaml is reported when the bundle is parsed
|
|
37
|
-
}
|
|
38
|
-
for (const block of ['includes', 'references']) {
|
|
39
|
-
for (const [name, include] of Object.entries(document?.[block] ?? {})) {
|
|
40
|
-
const source = include;
|
|
41
|
-
if (!source || typeof source !== 'object' || source.git === undefined) {
|
|
42
|
-
continue;
|
|
43
|
-
}
|
|
44
|
-
if (typeof source.git !== 'string' || !/^(https:\/\/|ssh:\/\/|git@[\w.-]+:)/.test(source.git)) {
|
|
45
|
-
problems.push(`${path}: ${block}.${name}: git has to be an https:// or ssh:// address`);
|
|
46
|
-
}
|
|
47
|
-
if (typeof source.ref !== 'string' || !COMMIT_PATTERN.test(source.ref)) {
|
|
48
|
-
problems.push(`${path}: ${block}.${name}: pin ref to a commit, a branch or tag can move`);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
return problems;
|
|
54
|
-
}
|
|
55
|
-
// What a build file may read while it is inspected: its own files. The rest of
|
|
56
|
-
// the server is not its business (a missing file is "not there", not an error).
|
|
57
|
-
function confined(files, root) {
|
|
58
|
-
const check = (path) => {
|
|
59
|
-
if (!(0, resolve_git_source_1.isWithin)(root, (0, path_1.resolve)(root, path))) {
|
|
60
|
-
throw new DefinitionError(`${path} is outside of the bundle`);
|
|
61
|
-
}
|
|
62
|
-
};
|
|
63
|
-
const guarded = (fn) => (path) => {
|
|
64
|
-
check(path);
|
|
65
|
-
return fn.call(files, path);
|
|
66
|
-
};
|
|
67
|
-
return {
|
|
68
|
-
...files,
|
|
69
|
-
read: guarded(files.read),
|
|
70
|
-
readStream: guarded(files.readStream),
|
|
71
|
-
listFiles: guarded(files.listFiles),
|
|
72
|
-
stats: guarded(files.stats),
|
|
73
|
-
exists: async (path) => (0, resolve_git_source_1.isWithin)(root, (0, path_1.resolve)(root, path)) && files.exists(path),
|
|
74
|
-
watch: files.watch,
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
|
-
// A path of a task, written relative to the root of the bundle with forward
|
|
78
|
-
// slashes. A leading `!` (an exclusion) is kept. Nothing is returned for a path
|
|
79
|
-
// outside of the bundle.
|
|
80
|
-
function insideBundle(root, cwd, path) {
|
|
81
|
-
const negated = path.startsWith('!');
|
|
82
|
-
const absolute = (0, path_1.resolve)(cwd, negated ? path.slice(1) : path);
|
|
83
|
-
const inside = (0, path_1.relative)(root, absolute);
|
|
84
|
-
if (inside === '..' || inside.startsWith(`..${path_1.sep}`) || (0, path_1.isAbsolute)(inside)) {
|
|
85
|
-
return [];
|
|
86
|
-
}
|
|
87
|
-
const portable = inside.split(path_1.sep).join(path_1.posix.sep);
|
|
88
|
-
// a pattern keeps its glob characters, `resolve` only normalizes separators
|
|
89
|
-
return [`${negated ? '!' : ''}${portable === '' ? '.' : portable}`];
|
|
90
|
-
}
|
|
91
|
-
// Writes the bundle into a directory of its own and reads it the way a run
|
|
92
|
-
// would, so a build file that doesn't parse is refused when it is published, not
|
|
93
|
-
// when somebody wants to run it.
|
|
94
|
-
async function inspectBundle(bundle, environment) {
|
|
95
|
-
const problems = findLooseIncludes(bundle);
|
|
96
|
-
if (problems.length > 0) {
|
|
97
|
-
throw new DefinitionError(problems.join('\n'));
|
|
98
|
-
}
|
|
99
|
-
const directory = await (0, promises_1.mkdtemp)((0, path_1.join)((0, os_1.tmpdir)(), 'hammerkit-definition-'));
|
|
100
|
-
try {
|
|
101
|
-
for (const [path, content] of Object.entries(bundle.files)) {
|
|
102
|
-
await (0, promises_1.mkdir)((0, path_1.dirname)((0, path_1.join)(directory, path)), { recursive: true });
|
|
103
|
-
await (0, promises_1.writeFile)((0, path_1.join)(directory, path), content);
|
|
104
|
-
}
|
|
105
|
-
const inspected = {
|
|
106
|
-
...environment,
|
|
107
|
-
cwd: directory,
|
|
108
|
-
file: confined((0, get_file_context_1.getFileContext)(directory), directory),
|
|
109
|
-
};
|
|
110
|
-
try {
|
|
111
|
-
const { ctx, scope } = await (0, schema_parser_1.createParseContext)((0, path_1.join)(directory, bundle.entry), inspected);
|
|
112
|
-
const context = await (0, reference_parser_1.parseReferences)(ctx, scope, inspected);
|
|
113
|
-
const workTree = (0, work_scope_parser_1.getWorkContext)(context, { environmentName: null }, inspected);
|
|
114
|
-
const inputs = {};
|
|
115
|
-
for (const item of Object.values(workTree.tasks)) {
|
|
116
|
-
inputs[item.name] = {
|
|
117
|
-
// the sources a task inherits from its dependencies belong to those
|
|
118
|
-
src: item.data.src
|
|
119
|
-
.filter((source) => source.inherited === null)
|
|
120
|
-
.flatMap((source) => insideBundle(directory, item.data.cwd, source.source)),
|
|
121
|
-
mounts: (0, work_task_1.isContainerWorkTask)(item.data)
|
|
122
|
-
? item.data.mounts.flatMap((mount) => insideBundle(directory, item.data.cwd, mount.localPath))
|
|
123
|
-
: [],
|
|
124
|
-
deps: item.deps.map((dep) => dep.name).sort(),
|
|
125
|
-
needs: item.needs.map((need) => need.name).sort(),
|
|
126
|
-
};
|
|
127
|
-
}
|
|
128
|
-
return {
|
|
129
|
-
tasks: Object.keys(context.tasks).sort(),
|
|
130
|
-
services: Object.keys(context.services).sort(),
|
|
131
|
-
inputs: Object.fromEntries(Object.entries(inputs).sort(([a], [b]) => (a < b ? -1 : 1))),
|
|
132
|
-
};
|
|
133
|
-
}
|
|
134
|
-
catch (e) {
|
|
135
|
-
throw new DefinitionError((0, log_1.getErrorMessage)(e).split(directory).join('.'));
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
finally {
|
|
139
|
-
await (0, promises_1.rm)(directory, { recursive: true, force: true });
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
exports.inspectBundle = inspectBundle;
|
|
8
|
+
const plan_1 = require("./plan");
|
|
9
|
+
var plan_2 = require("./plan");
|
|
10
|
+
Object.defineProperty(exports, "DefinitionError", { enumerable: true, get: function () { return plan_2.DefinitionError; } });
|
|
11
|
+
Object.defineProperty(exports, "inspectBundle", { enumerable: true, get: function () { return plan_2.inspectBundle; } });
|
|
143
12
|
// The published build files: immutable versions, each named by its hash. They
|
|
144
13
|
// are kept in files, so a restart doesn't forget what runs were started from.
|
|
145
14
|
class DefinitionStore {
|
|
146
|
-
constructor(directory,
|
|
15
|
+
constructor(directory,
|
|
16
|
+
// the environment a build file is read with: on a server, the hosted one
|
|
17
|
+
environment, options = {}) {
|
|
147
18
|
this.directory = directory;
|
|
148
19
|
this.environment = environment;
|
|
149
|
-
this.
|
|
20
|
+
this.options = options;
|
|
150
21
|
this.versions = new Map();
|
|
22
|
+
this.now = options.now ?? Date.now;
|
|
151
23
|
}
|
|
152
24
|
async load() {
|
|
153
25
|
let names = [];
|
|
@@ -174,10 +46,43 @@ class DefinitionStore {
|
|
|
174
46
|
list(name) {
|
|
175
47
|
return this.has(name) ? [...this.versions.get(name)] : [];
|
|
176
48
|
}
|
|
177
|
-
// a hash, or `latest` for the version published last
|
|
49
|
+
// a hash, or `latest` for the version published last that is not yanked
|
|
178
50
|
find(name, hash) {
|
|
179
51
|
const versions = this.list(name);
|
|
180
|
-
return hash === 'latest' ? versions
|
|
52
|
+
return hash === 'latest' ? versions.find((v) => !v.yanked) : versions.find((v) => v.hash === hash);
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Pulls a version back (or, with `undo`, lets it go on). The version is rewritten in
|
|
56
|
+
* place, its files stay: a run that was started from it has them.
|
|
57
|
+
*/
|
|
58
|
+
async yank(name, hash, by, options = {}) {
|
|
59
|
+
const version = this.versions.get(name)?.find((v) => v.hash === hash);
|
|
60
|
+
if (!version || Boolean(version.yanked) === !options.undo) {
|
|
61
|
+
return version;
|
|
62
|
+
}
|
|
63
|
+
const bundle = await this.files(version);
|
|
64
|
+
if (!bundle) {
|
|
65
|
+
return version;
|
|
66
|
+
}
|
|
67
|
+
const next = { ...version };
|
|
68
|
+
if (options.undo) {
|
|
69
|
+
delete next.yanked;
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
next.yanked = {
|
|
73
|
+
at: new Date(this.now()).toISOString(),
|
|
74
|
+
by: { iss: by.iss, sub: by.sub },
|
|
75
|
+
...(options.reason ? { reason: options.reason } : {}),
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
const file = this.file(name, hash);
|
|
79
|
+
const temporary = `${file}.${(0, crypto_1.randomBytes)(8).toString('hex')}.tmp`;
|
|
80
|
+
await (0, promises_1.writeFile)(temporary, JSON.stringify({ version: next, bundle }), { flag: 'wx' });
|
|
81
|
+
await (0, promises_1.rename)(temporary, file);
|
|
82
|
+
// the same object, so whoever holds the version sees it
|
|
83
|
+
Object.keys(version).forEach((key) => delete version[key]);
|
|
84
|
+
Object.assign(version, next);
|
|
85
|
+
return version;
|
|
181
86
|
}
|
|
182
87
|
async files(version) {
|
|
183
88
|
try {
|
|
@@ -189,13 +94,18 @@ class DefinitionStore {
|
|
|
189
94
|
}
|
|
190
95
|
// Reads a bundle the way a run would and says what it would be as a version;
|
|
191
96
|
// `existing` is the published version with the same hash, if there is one.
|
|
192
|
-
async prepare(name, bundle, publisher
|
|
97
|
+
async prepare(name, bundle, publisher,
|
|
98
|
+
// a proposal is read before anybody trusts it: it may not make the server fetch
|
|
99
|
+
options = {}) {
|
|
193
100
|
const hash = (0, bundle_1.hashBundle)(bundle);
|
|
194
101
|
const existing = this.find(name, hash);
|
|
195
102
|
if (existing) {
|
|
196
103
|
return { version: existing, existing };
|
|
197
104
|
}
|
|
198
|
-
const { tasks, services, inputs } = await
|
|
105
|
+
const { tasks, services, inputs, plan, accounts } = await (0, plan_1.planBundle)(bundle, this.environment, {
|
|
106
|
+
policy: this.options.policy,
|
|
107
|
+
noRemote: options.noRemote,
|
|
108
|
+
});
|
|
199
109
|
return {
|
|
200
110
|
version: {
|
|
201
111
|
name,
|
|
@@ -206,6 +116,8 @@ class DefinitionStore {
|
|
|
206
116
|
tasks,
|
|
207
117
|
services,
|
|
208
118
|
inputs,
|
|
119
|
+
plan,
|
|
120
|
+
accounts,
|
|
209
121
|
},
|
|
210
122
|
};
|
|
211
123
|
}
|
|
@@ -239,7 +151,7 @@ class DefinitionStore {
|
|
|
239
151
|
}
|
|
240
152
|
file(name, hash) {
|
|
241
153
|
if (!bundle_1.HASH_PATTERN.test(hash)) {
|
|
242
|
-
throw new DefinitionError('invalid hash');
|
|
154
|
+
throw new plan_1.DefinitionError('invalid hash');
|
|
243
155
|
}
|
|
244
156
|
return (0, path_1.join)(this.directory, name, `${hash.slice('sha256:'.length)}.json`);
|
|
245
157
|
}
|
package/dist/server/executor.js
CHANGED
|
@@ -1,96 +1,87 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.HostExecutor = exports.
|
|
3
|
+
exports.HostExecutor = exports.defaultWorker = void 0;
|
|
4
|
+
const child_process_1 = require("child_process");
|
|
5
|
+
const crypto_1 = require("crypto");
|
|
6
|
+
const fs_1 = require("fs");
|
|
4
7
|
const promises_1 = require("fs/promises");
|
|
5
8
|
const path_1 = require("path");
|
|
6
|
-
const
|
|
7
|
-
const get_file_context_1 = require("../file/get-file-context");
|
|
8
|
-
const log_1 = require("../log");
|
|
9
|
+
const readline_1 = require("readline");
|
|
9
10
|
const resolve_git_source_1 = require("../includes/resolve-git-source");
|
|
10
|
-
const parse_label_arguments_1 = require("../parser/parse-label-arguments");
|
|
11
|
-
const schema_parser_1 = require("../schema/schema-parser");
|
|
12
|
-
const reference_parser_1 = require("../schema/reference-parser");
|
|
13
|
-
const work_scope_parser_1 = require("../schema/work-scope-parser");
|
|
14
|
-
const work_item_status_1 = require("../planner/work-item-status");
|
|
15
|
-
const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
|
|
16
|
-
const redact_1 = require("../utils/redact");
|
|
17
|
-
const empty_writable_1 = require("../utils/empty-writable");
|
|
18
|
-
const units_1 = require("../utils/units");
|
|
19
11
|
const git_checkout_1 = require("./git-checkout");
|
|
20
|
-
|
|
21
|
-
//
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
'
|
|
25
|
-
'
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
'
|
|
37
|
-
'
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
'
|
|
45
|
-
'
|
|
46
|
-
'LOCALAPPDATA',
|
|
47
|
-
'TEMP',
|
|
48
|
-
'TMP',
|
|
49
|
-
'ProgramData',
|
|
50
|
-
'ProgramFiles',
|
|
51
|
-
'ProgramFiles(x86)',
|
|
52
|
-
// powershell is very slow to start without it
|
|
53
|
-
'PSModulePath',
|
|
54
|
-
];
|
|
55
|
-
// one log line of a task is cut here, the registry caps how many events a run keeps
|
|
56
|
-
const MAX_LOG_TEXT = 8 * 1024;
|
|
57
|
-
function passedEnvironment(envs) {
|
|
58
|
-
const passed = {};
|
|
59
|
-
for (const name of PASSED_ENVS) {
|
|
60
|
-
if (envs[name] !== undefined) {
|
|
61
|
-
passed[name] = envs[name];
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
return passed;
|
|
12
|
+
const run_plan_1 = require("./run-plan");
|
|
13
|
+
// `.git` as a file system may read it: any case, and a name Windows reads without its
|
|
14
|
+
// trailing dots and spaces, or by its short name
|
|
15
|
+
function isGitDirectory(segment) {
|
|
16
|
+
const name = segment.toLowerCase().replace(/[. ]+$/, '');
|
|
17
|
+
return name === '.git' || name === 'git~1';
|
|
18
|
+
}
|
|
19
|
+
// A cancelled run gets this long to stop its tasks and remove its containers
|
|
20
|
+
const STOP_GRACE_MS = 15000;
|
|
21
|
+
// what is kept of the error output of a process that died without saying why
|
|
22
|
+
const MAX_STDERR = 2 * 1024;
|
|
23
|
+
/**
|
|
24
|
+
* The command that starts the process of a run: `hammerkit server-run`, from the
|
|
25
|
+
* build this module is part of. Only found next to a built `index.js`.
|
|
26
|
+
*/
|
|
27
|
+
function defaultWorker() {
|
|
28
|
+
const entry = (0, path_1.resolve)(__dirname, '..', 'index.js');
|
|
29
|
+
return (0, fs_1.existsSync)(entry) ? [process.execPath, entry, 'server-run'] : undefined;
|
|
30
|
+
}
|
|
31
|
+
exports.defaultWorker = defaultWorker;
|
|
32
|
+
// The cache a run shares with others is the one of its source: the runs of a repository
|
|
33
|
+
// share one, those of an upload only the uploader's. Somebody who can only upload can
|
|
34
|
+
// not leave an entry for a run from a reviewed repository to find.
|
|
35
|
+
function cacheNamespace(run) {
|
|
36
|
+
const key = run.source.type === 'ref' ? `ref\0${run.source.repo}` : `upload\0${run.owner.iss}\0${run.owner.sub}`;
|
|
37
|
+
return `${run.source.type}-${(0, crypto_1.createHash)('sha256').update(key).digest('hex').slice(0, 24)}`;
|
|
65
38
|
}
|
|
66
|
-
exports.passedEnvironment = passedEnvironment;
|
|
67
39
|
/**
|
|
68
|
-
* Runs the tasks of a run on this machine with the engine of `hammerkit run
|
|
40
|
+
* Runs the tasks of a run on this machine with the engine of `hammerkit run`, in a
|
|
41
|
+
* process of its own.
|
|
69
42
|
*
|
|
70
43
|
* The working directory is made for the run and removed afterwards. Its build
|
|
71
44
|
* file is the published version the run is pinned to, written over the code of the
|
|
72
|
-
* source, so the commands are the published ones whatever the source says. The
|
|
73
|
-
*
|
|
74
|
-
*
|
|
45
|
+
* source, so the commands are the published ones whatever the source says. The process
|
|
46
|
+
* starts with an environment the server builds: the variables `runs.env` lists, a home
|
|
47
|
+
* directory that is the run's alone, and a cache directory per source trust. So what
|
|
48
|
+
* the engine reads of its process (the docker host, the registry logins in the home
|
|
49
|
+
* directory, the credentials of the operator) is what the operator gave runs and
|
|
50
|
+
* nothing else. The cache is shared by the runs of the same source, a task is keyed by
|
|
51
|
+
* its definition and sources and not by where they are.
|
|
75
52
|
*/
|
|
76
53
|
class HostExecutor {
|
|
77
54
|
constructor(options) {
|
|
78
55
|
this.options = options;
|
|
79
56
|
}
|
|
80
57
|
async execute(run, context) {
|
|
81
|
-
const { definitions,
|
|
58
|
+
const { definitions, planning, repos } = this.options;
|
|
82
59
|
const version = definitions.find(run.definition.name, run.definition.hash);
|
|
83
60
|
const bundle = version ? await definitions.files(version) : undefined;
|
|
84
61
|
if (!version || !bundle) {
|
|
85
62
|
throw new Error(`the build file ${run.definition.name} at ${run.definition.hash} is no longer on this server`);
|
|
86
63
|
}
|
|
64
|
+
// pulled back after the run was accepted, while it waited for an approval or in the queue
|
|
65
|
+
if (version.yanked) {
|
|
66
|
+
throw new Error(`the build file ${run.definition.name} at ${run.definition.hash} was yanked${version.yanked.reason ? `: ${version.yanked.reason}` : ''}`);
|
|
67
|
+
}
|
|
87
68
|
if (run.source.type !== 'ref') {
|
|
88
69
|
throw new Error('runs from an upload are not supported by this server yet, use a git ref');
|
|
89
70
|
}
|
|
90
|
-
const
|
|
91
|
-
if (!
|
|
92
|
-
throw new Error(`this server
|
|
93
|
-
|
|
71
|
+
const { repo, ref, commit: pinned } = run.source;
|
|
72
|
+
if (!repos.readable(repo)) {
|
|
73
|
+
throw new Error(`this server can not read ${repo}, the operator lists it in sources`);
|
|
74
|
+
}
|
|
75
|
+
const worker = this.options.worker ?? defaultWorker();
|
|
76
|
+
if (!worker) {
|
|
77
|
+
throw new Error('this server has no command to start a run with, it needs to run from a built hammerkit');
|
|
78
|
+
}
|
|
79
|
+
// what was approved is what runs: the accounts are read again from the files with
|
|
80
|
+
// the environment of the run, and the run may use no more than it was admitted for
|
|
81
|
+
const planned = await (0, run_plan_1.planRun)(bundle, run, planning.environment, planning.policy);
|
|
82
|
+
const extra = planned.accounts.filter((account) => !run.accounts.includes(account));
|
|
83
|
+
if (extra.length > 0) {
|
|
84
|
+
throw new Error(`the run would use ${extra.join(', ')}, which it was not admitted for`);
|
|
94
85
|
}
|
|
95
86
|
const abortCtrl = new AbortController();
|
|
96
87
|
const abort = () => abortCtrl.abort();
|
|
@@ -101,138 +92,146 @@ class HostExecutor {
|
|
|
101
92
|
const root = (0, path_1.join)(this.options.directory, run.id);
|
|
102
93
|
const checkout = (0, path_1.join)(root, 'source');
|
|
103
94
|
try {
|
|
104
|
-
const
|
|
105
|
-
|
|
106
|
-
|
|
95
|
+
const credentials = await repos.credentials(repo);
|
|
96
|
+
let commit;
|
|
97
|
+
try {
|
|
98
|
+
commit = await (0, git_checkout_1.checkoutRef)(repos.url(repo), ref, checkout, credentials.env, abortCtrl.signal, pinned);
|
|
99
|
+
}
|
|
100
|
+
catch (e) {
|
|
101
|
+
throw repos.scrub(e, credentials.values);
|
|
102
|
+
}
|
|
103
|
+
context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `checked out ${repo} at ${commit}` });
|
|
107
104
|
await this.writeBundle(checkout, bundle.files);
|
|
108
|
-
|
|
105
|
+
const home = (0, path_1.join)(root, 'home');
|
|
106
|
+
await (0, promises_1.mkdir)(home, { recursive: true });
|
|
107
|
+
const cache = (0, path_1.join)(this.options.cache, cacheNamespace(run));
|
|
108
|
+
await (0, promises_1.mkdir)(cache, { recursive: true });
|
|
109
|
+
const input = {
|
|
110
|
+
entry: (0, path_1.join)(checkout, version.entry),
|
|
111
|
+
tasks: run.tasks,
|
|
112
|
+
options: run.options,
|
|
113
|
+
accounts: run.accounts,
|
|
114
|
+
catalog: planning.environment.hosted?.catalog,
|
|
115
|
+
};
|
|
116
|
+
await this.runInProcess(worker, input, checkout, this.processEnvironment(home, cache), context, abortCtrl.signal);
|
|
109
117
|
}
|
|
110
118
|
finally {
|
|
111
119
|
context.signal.removeEventListener('abort', abort);
|
|
112
120
|
await (0, promises_1.rm)(root, { recursive: true, force: true });
|
|
113
121
|
}
|
|
114
122
|
}
|
|
115
|
-
//
|
|
116
|
-
//
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
if (
|
|
121
|
-
|
|
123
|
+
// What the process of a run sees: the variables the operator listed, and a home
|
|
124
|
+
// directory and cache that are the run's. The server's own are not in it.
|
|
125
|
+
processEnvironment(home, cache) {
|
|
126
|
+
const env = {};
|
|
127
|
+
for (const [key, value] of Object.entries(this.options.planning.environment.processEnvs)) {
|
|
128
|
+
if (value !== undefined) {
|
|
129
|
+
env[key] = value;
|
|
122
130
|
}
|
|
123
|
-
await (0, promises_1.mkdir)((0, path_1.dirname)(target), { recursive: true });
|
|
124
|
-
await (0, promises_1.writeFile)(target, content);
|
|
125
131
|
}
|
|
132
|
+
env.HOME = home;
|
|
133
|
+
env.USERPROFILE = home;
|
|
134
|
+
env.APPDATA = (0, path_1.join)(home, 'AppData', 'Roaming');
|
|
135
|
+
env.LOCALAPPDATA = (0, path_1.join)(home, 'AppData', 'Local');
|
|
136
|
+
env.HAMMERKIT_DIR = cache;
|
|
137
|
+
return env;
|
|
126
138
|
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
task: message.context.name,
|
|
149
|
-
stream: message.console,
|
|
150
|
-
text: message.message.slice(0, MAX_LOG_TEXT),
|
|
151
|
-
});
|
|
139
|
+
// Starts the process, hands it the run on its standard input and relays what it reports. It
|
|
140
|
+
// is stopped with SIGTERM when the run is cancelled, so the engine removes its containers
|
|
141
|
+
// and stops its tasks, and killed if it does not stop in time.
|
|
142
|
+
runInProcess(command, input, cwd, env, context, signal) {
|
|
143
|
+
return new Promise((resolvePromise, reject) => {
|
|
144
|
+
const child = (0, child_process_1.spawn)(command[0], command.slice(1), {
|
|
145
|
+
cwd,
|
|
146
|
+
env,
|
|
147
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
148
|
+
windowsHide: true,
|
|
149
|
+
});
|
|
150
|
+
let result;
|
|
151
|
+
let stderr = '';
|
|
152
|
+
let killTimer;
|
|
153
|
+
const stop = () => {
|
|
154
|
+
child.kill('SIGTERM');
|
|
155
|
+
killTimer = setTimeout(() => child.kill('SIGKILL'), STOP_GRACE_MS);
|
|
156
|
+
killTimer.unref();
|
|
157
|
+
};
|
|
158
|
+
if (signal.aborted) {
|
|
159
|
+
stop();
|
|
152
160
|
}
|
|
153
|
-
else
|
|
154
|
-
|
|
155
|
-
type: 'log',
|
|
156
|
-
task: message.context.name,
|
|
157
|
-
stream: 'stderr',
|
|
158
|
-
text: message.message.slice(0, MAX_LOG_TEXT),
|
|
159
|
-
});
|
|
161
|
+
else {
|
|
162
|
+
signal.addEventListener('abort', stop, { once: true });
|
|
160
163
|
}
|
|
161
|
-
|
|
162
|
-
try {
|
|
163
|
-
const options = run.options;
|
|
164
|
-
const execOptions = {
|
|
165
|
-
logMode: 'live',
|
|
166
|
-
cacheDefault: options.cache ?? 'checksum',
|
|
167
|
-
cacheReadOnly: options.cacheReadOnly ?? false,
|
|
168
|
-
timeout: options.timeout ? (0, units_1.parseDuration)(options.timeout) : null,
|
|
169
|
-
skipDeps: options.skipDeps !== false,
|
|
170
|
-
};
|
|
171
|
-
// the tasks in the order they were named; by labels when none is
|
|
172
|
-
const scopes = run.tasks.length > 0
|
|
173
|
-
? run.tasks.map((task) => ({
|
|
174
|
-
name: task,
|
|
175
|
-
scope: { taskName: task, environmentName: options.env ?? null },
|
|
176
|
-
}))
|
|
177
|
-
: [
|
|
178
|
-
{
|
|
179
|
-
name: 'labels',
|
|
180
|
-
scope: {
|
|
181
|
-
filterLabels: labels(options.filter),
|
|
182
|
-
excludeLabels: labels(options.exclude),
|
|
183
|
-
environmentName: options.env ?? null,
|
|
184
|
-
},
|
|
185
|
-
},
|
|
186
|
-
];
|
|
187
|
-
for (const { name, scope } of scopes) {
|
|
188
|
-
const { ctx, scope: parsed } = await (0, schema_parser_1.createParseContext)(entry, environment);
|
|
189
|
-
const workTree = (0, work_scope_parser_1.getWorkContext)(await (0, reference_parser_1.parseReferences)(ctx, parsed, environment), scope, environment);
|
|
190
|
-
const cli = (0, cli_1.getCli)(workTree, environment);
|
|
191
|
-
if (cli.tasks().length === 0) {
|
|
192
|
-
throw new Error(`no tasks found for ${name}`);
|
|
193
|
-
}
|
|
194
|
-
const execution = cli.exec({ ...execOptions, workers: cli.defaultWorkers() });
|
|
195
|
-
const seen = new Map();
|
|
196
|
-
const watch = execution.state.on('run-events', () => reportTasks(context, execution.state.current, seen));
|
|
164
|
+
(0, readline_1.createInterface)({ input: child.stdout }).on('line', (line) => {
|
|
197
165
|
try {
|
|
198
|
-
const
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
166
|
+
const message = JSON.parse(line);
|
|
167
|
+
if (message.kind === 'event') {
|
|
168
|
+
context.emit(message.event);
|
|
169
|
+
}
|
|
170
|
+
else if (message.kind === 'result') {
|
|
171
|
+
result = message;
|
|
202
172
|
}
|
|
203
173
|
}
|
|
204
|
-
|
|
205
|
-
|
|
174
|
+
catch {
|
|
175
|
+
// not ours: whatever else the process prints is not an event of the run
|
|
206
176
|
}
|
|
177
|
+
});
|
|
178
|
+
child.stderr.on('data', (chunk) => {
|
|
179
|
+
stderr = (stderr + chunk.toString('utf8')).slice(-MAX_STDERR);
|
|
180
|
+
});
|
|
181
|
+
child.stdin.on('error', () => undefined);
|
|
182
|
+
child.stdin.end(JSON.stringify(input));
|
|
183
|
+
child.on('error', (e) => {
|
|
184
|
+
signal.removeEventListener('abort', stop);
|
|
185
|
+
clearTimeout(killTimer);
|
|
186
|
+
reject(new Error(`the process of the run could not start: ${e.message}`));
|
|
187
|
+
});
|
|
188
|
+
child.on('close', (code, killed) => {
|
|
189
|
+
signal.removeEventListener('abort', stop);
|
|
190
|
+
clearTimeout(killTimer);
|
|
191
|
+
if (result?.ok) {
|
|
192
|
+
resolvePromise();
|
|
193
|
+
}
|
|
194
|
+
else if (result) {
|
|
195
|
+
reject(new Error(result.error ?? 'the run failed'));
|
|
196
|
+
}
|
|
197
|
+
else if (signal.aborted) {
|
|
198
|
+
reject(new Error('the run was cancelled'));
|
|
199
|
+
}
|
|
200
|
+
else {
|
|
201
|
+
const reason = stderr.trim().split('\n').pop() ?? '';
|
|
202
|
+
reject(new Error(`the process of the run stopped (${killed ? `signal ${killed}` : `code ${code}`})${reason ? `: ${reason}` : ''}`));
|
|
203
|
+
}
|
|
204
|
+
});
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
// The published files go over the checkout; nothing may land outside of it or in
|
|
208
|
+
// its git directory. The checkout is somebody else's: it may hold a symbolic link where
|
|
209
|
+
// a directory or a file of the bundle goes, which would carry the write out of it, so
|
|
210
|
+
// none is followed. `.git` is compared the way a case-insensitive or Windows file system
|
|
211
|
+
// would read it.
|
|
212
|
+
async writeBundle(directory, files) {
|
|
213
|
+
for (const [path, content] of Object.entries(files)) {
|
|
214
|
+
const target = (0, path_1.resolve)(directory, path);
|
|
215
|
+
const refused = () => new Error(`the published file ${JSON.stringify(path)} can not be written into the source`);
|
|
216
|
+
if (!(0, resolve_git_source_1.isWithin)(directory, target) || path.split(/[\\/]/).some((part) => isGitDirectory(part))) {
|
|
217
|
+
throw refused();
|
|
207
218
|
}
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
219
|
+
const parts = (0, path_1.relative)(directory, target).split(path_1.sep);
|
|
220
|
+
let current = directory;
|
|
221
|
+
for (const [index, part] of parts.entries()) {
|
|
222
|
+
current = (0, path_1.join)(current, part);
|
|
223
|
+
const stats = await (0, promises_1.lstat)(current).catch(() => undefined);
|
|
224
|
+
if (!stats) {
|
|
225
|
+
break;
|
|
226
|
+
}
|
|
227
|
+
if (stats.isSymbolicLink() || (index < parts.length - 1 && !stats.isDirectory())) {
|
|
228
|
+
throw refused();
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
await (0, promises_1.mkdir)((0, path_1.dirname)(target), { recursive: true });
|
|
232
|
+
await (0, promises_1.writeFile)(target, content);
|
|
211
233
|
}
|
|
212
234
|
}
|
|
213
235
|
}
|
|
214
236
|
exports.HostExecutor = HostExecutor;
|
|
215
|
-
function labels(values) {
|
|
216
|
-
return values && values.length > 0 ? (0, parse_label_arguments_1.parseLabelArguments)(values) : undefined;
|
|
217
|
-
}
|
|
218
|
-
// A task event when a task has moved on since the last look.
|
|
219
|
-
function reportTasks(context, workTree, seen) {
|
|
220
|
-
for (const task of (0, plan_work_tasks_1.iterateWorkTasks)(workTree)) {
|
|
221
|
-
const state = task.state.current;
|
|
222
|
-
let event = null;
|
|
223
|
-
if (state.type === 'running') {
|
|
224
|
-
event = 'running';
|
|
225
|
-
}
|
|
226
|
-
else if (state.type === 'completed') {
|
|
227
|
-
event = state.skipped ? null : state.cached ? 'cached' : 'succeeded';
|
|
228
|
-
}
|
|
229
|
-
else if (state.type === 'error' || state.type === 'crash') {
|
|
230
|
-
event = 'failed';
|
|
231
|
-
}
|
|
232
|
-
if (event && seen.get(task.name) !== event) {
|
|
233
|
-
seen.set(task.name, event);
|
|
234
|
-
context.emit({ type: 'task', task: task.name, state: event });
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
237
|
//# sourceMappingURL=executor.js.map
|