@terascope/scripts 2.20.0 → 2.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/helpers/bump/index.js +1 -1
- package/dist/src/helpers/bump/index.js.map +1 -1
- package/dist/src/helpers/certs.d.ts +28 -0
- package/dist/src/helpers/certs.d.ts.map +1 -0
- package/dist/src/helpers/certs.js +267 -0
- package/dist/src/helpers/certs.js.map +1 -0
- package/dist/src/helpers/doc-builder/index.js +1 -1
- package/dist/src/helpers/doc-builder/index.js.map +1 -1
- package/dist/src/helpers/docker.d.ts +78 -0
- package/dist/src/helpers/docker.d.ts.map +1 -0
- package/dist/src/helpers/docker.js +395 -0
- package/dist/src/helpers/docker.js.map +1 -0
- package/dist/src/helpers/exec.d.ts +19 -0
- package/dist/src/helpers/exec.d.ts.map +1 -0
- package/dist/src/helpers/exec.js +81 -0
- package/dist/src/helpers/exec.js.map +1 -0
- package/dist/src/helpers/git.d.ts +4 -0
- package/dist/src/helpers/git.d.ts.map +1 -0
- package/dist/src/helpers/git.js +47 -0
- package/dist/src/helpers/git.js.map +1 -0
- package/dist/src/helpers/helm.d.ts +25 -0
- package/dist/src/helpers/helm.d.ts.map +1 -0
- package/dist/src/helpers/helm.js +297 -0
- package/dist/src/helpers/helm.js.map +1 -0
- package/dist/src/helpers/images/index.js +1 -1
- package/dist/src/helpers/images/index.js.map +1 -1
- package/dist/src/helpers/k8s-env/index.d.ts.map +1 -1
- package/dist/src/helpers/k8s-env/index.js +5 -2
- package/dist/src/helpers/k8s-env/index.js.map +1 -1
- package/dist/src/helpers/kubernetes.d.ts +10 -0
- package/dist/src/helpers/kubernetes.d.ts.map +1 -0
- package/dist/src/helpers/kubernetes.js +160 -0
- package/dist/src/helpers/kubernetes.js.map +1 -0
- package/dist/src/helpers/misc.d.ts +9 -5
- package/dist/src/helpers/misc.d.ts.map +1 -1
- package/dist/src/helpers/misc.js +66 -9
- package/dist/src/helpers/misc.js.map +1 -1
- package/dist/src/helpers/package-manager.d.ts +18 -0
- package/dist/src/helpers/package-manager.d.ts.map +1 -0
- package/dist/src/helpers/package-manager.js +152 -0
- package/dist/src/helpers/package-manager.js.map +1 -0
- package/dist/src/helpers/packages.js +1 -1
- package/dist/src/helpers/packages.js.map +1 -1
- package/dist/src/helpers/publish/index.d.ts.map +1 -1
- package/dist/src/helpers/publish/index.js +2 -1
- package/dist/src/helpers/publish/index.js.map +1 -1
- package/dist/src/helpers/publish/utils.d.ts.map +1 -1
- package/dist/src/helpers/publish/utils.js +2 -1
- package/dist/src/helpers/publish/utils.js.map +1 -1
- package/dist/src/helpers/sync/utils.js +1 -1
- package/dist/src/helpers/sync/utils.js.map +1 -1
- package/dist/src/helpers/test-runner/index.d.ts.map +1 -1
- package/dist/src/helpers/test-runner/index.js +5 -1
- package/dist/src/helpers/test-runner/index.js.map +1 -1
- package/dist/src/helpers/test-runner/services.d.ts.map +1 -1
- package/dist/src/helpers/test-runner/services.js +11 -5
- package/dist/src/helpers/test-runner/services.js.map +1 -1
- package/dist/src/helpers/test-runner/utils.d.ts +1 -1
- package/dist/src/helpers/test-runner/utils.d.ts.map +1 -1
- package/dist/src/helpers/test-runner/utils.js +3 -2
- package/dist/src/helpers/test-runner/utils.js.map +1 -1
- package/dist/src/index.d.ts +7 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +7 -1
- package/dist/src/index.js.map +1 -1
- package/package.json +1 -1
- package/dist/src/helpers/scripts.d.ts +0 -174
- package/dist/src/helpers/scripts.d.ts.map +0 -1
- package/dist/src/helpers/scripts.js +0 -1404
- package/dist/src/helpers/scripts.js.map +0 -1
|
@@ -1,1404 +0,0 @@
|
|
|
1
|
-
import fs from 'node:fs';
|
|
2
|
-
import os from 'node:os';
|
|
3
|
-
import ms from 'ms';
|
|
4
|
-
import path from 'node:path';
|
|
5
|
-
import { X509Certificate } from 'node:crypto';
|
|
6
|
-
import { execa, execaCommand } from 'execa';
|
|
7
|
-
import fse from 'fs-extra';
|
|
8
|
-
import { load } from 'js-yaml';
|
|
9
|
-
import got from 'got';
|
|
10
|
-
import { parseDocument } from 'yaml';
|
|
11
|
-
import { debugLogger, isString, get, pWhile, pDelay, TSError, isCI, } from '@terascope/core-utils';
|
|
12
|
-
import { Service } from '@terascope/types';
|
|
13
|
-
import { getRootDir, getRootInfo, getPackageManager } from './misc.js';
|
|
14
|
-
import signale from './signale.js';
|
|
15
|
-
import config from './config.js';
|
|
16
|
-
import { getVolumesFromDockerfile } from './kind.js';
|
|
17
|
-
import { TestFrameworks } from './test-runner/interfaces.js';
|
|
18
|
-
const logger = debugLogger('ts-scripts:cmd');
|
|
19
|
-
export const TEST_CONFIGS = 'test-configs';
|
|
20
|
-
function _exec(opts) {
|
|
21
|
-
let subprocess;
|
|
22
|
-
const options = {
|
|
23
|
-
cwd: opts.cwd || getRootDir(),
|
|
24
|
-
env: opts.env,
|
|
25
|
-
preferLocal: true,
|
|
26
|
-
detached: opts.detached,
|
|
27
|
-
timeout: opts.timeout,
|
|
28
|
-
stdio: opts.stdio,
|
|
29
|
-
};
|
|
30
|
-
logger.debug('executing command', opts);
|
|
31
|
-
if (opts.args && opts.args.length) {
|
|
32
|
-
subprocess = execa(opts.cmd, opts.args, options);
|
|
33
|
-
}
|
|
34
|
-
else {
|
|
35
|
-
subprocess = execa(opts.cmd, options);
|
|
36
|
-
}
|
|
37
|
-
if (!subprocess) {
|
|
38
|
-
throw new Error(`Failed to execution ${opts.cmd}`);
|
|
39
|
-
}
|
|
40
|
-
if (!opts.stdio) {
|
|
41
|
-
if (!subprocess.stderr || !subprocess.stdout) {
|
|
42
|
-
throw new Error(`Command ${opts.cmd} failed, stderr or stdout is not available`);
|
|
43
|
-
}
|
|
44
|
-
subprocess.stderr.pipe(process.stderr);
|
|
45
|
-
}
|
|
46
|
-
return subprocess;
|
|
47
|
-
}
|
|
48
|
-
export async function exec(opts, log = true) {
|
|
49
|
-
try {
|
|
50
|
-
const env = { FORCE_COLOR: '0', ...opts.env };
|
|
51
|
-
const _opts = { ...opts };
|
|
52
|
-
_opts.env = env;
|
|
53
|
-
const subprocess = _exec(_opts);
|
|
54
|
-
const { stdout } = await subprocess;
|
|
55
|
-
if (typeof stdout !== 'string') {
|
|
56
|
-
throw new Error('exec() requires ExecOpts that result in a stdout string. See the execa docs for details.');
|
|
57
|
-
}
|
|
58
|
-
const result = stdout.trim();
|
|
59
|
-
logger.debug(`exec result: ${opts.cmd} ${(opts.args || []).join(' ')}`, log && result);
|
|
60
|
-
return result;
|
|
61
|
-
}
|
|
62
|
-
catch (err) {
|
|
63
|
-
if (!err.command) {
|
|
64
|
-
throw err;
|
|
65
|
-
}
|
|
66
|
-
process.exitCode = err.exitCode || 1;
|
|
67
|
-
throw new Error(err.message);
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
export async function fork(opts) {
|
|
71
|
-
try {
|
|
72
|
-
const env = {
|
|
73
|
-
FORCE_COLOR: config.FORCE_COLOR,
|
|
74
|
-
...opts.env
|
|
75
|
-
};
|
|
76
|
-
const _opts = { stdio: 'inherit', ...opts };
|
|
77
|
-
_opts.env = env;
|
|
78
|
-
await _exec(_opts);
|
|
79
|
-
}
|
|
80
|
-
catch (err) {
|
|
81
|
-
if (!err.command) {
|
|
82
|
-
throw err;
|
|
83
|
-
}
|
|
84
|
-
process.exitCode = err.exitCode || 1;
|
|
85
|
-
throw new Error(err.message);
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
export async function runTSScript(cmd, args) {
|
|
89
|
-
const scriptName = process.argv[1];
|
|
90
|
-
return fork({
|
|
91
|
-
cmd: scriptName,
|
|
92
|
-
args: [cmd, ...args],
|
|
93
|
-
});
|
|
94
|
-
}
|
|
95
|
-
export async function build(pkgInfo) {
|
|
96
|
-
if (pkgInfo) {
|
|
97
|
-
const distDir = path.join(pkgInfo.dir, 'dist');
|
|
98
|
-
if (fse.existsSync(distDir)) {
|
|
99
|
-
await fse.emptyDir(distDir);
|
|
100
|
-
}
|
|
101
|
-
await packageMngrRun('build', [], pkgInfo.dir);
|
|
102
|
-
return;
|
|
103
|
-
}
|
|
104
|
-
await packageMngrRun('build');
|
|
105
|
-
}
|
|
106
|
-
export async function setup() {
|
|
107
|
-
await packageMngrRun('setup');
|
|
108
|
-
}
|
|
109
|
-
export async function packageMngrRun(script, args, cwd, env, log) {
|
|
110
|
-
const dir = cwd || getRootDir();
|
|
111
|
-
const pkgJSON = await fse.readJSON(path.join(dir, 'package.json'));
|
|
112
|
-
const hasScript = Boolean(get(pkgJSON, ['scripts', script]));
|
|
113
|
-
if (!hasScript)
|
|
114
|
-
return;
|
|
115
|
-
const pm = getPackageManager();
|
|
116
|
-
const _args = ['run', script, ...(args ?? [])];
|
|
117
|
-
if (log) {
|
|
118
|
-
signale.info(`running ${pm} ${_args.join(' ')}...`);
|
|
119
|
-
}
|
|
120
|
-
await fork({
|
|
121
|
-
cmd: pm, args: _args, cwd: dir, env
|
|
122
|
-
});
|
|
123
|
-
}
|
|
124
|
-
export async function runTestFramework(cwd, argsMap, env, extraArgs, debug, attachJestDebugger, framework = TestFrameworks.jest, frameworkConfig) {
|
|
125
|
-
const pm = getPackageManager();
|
|
126
|
-
// When running jest in yarn3 PnP with ESM we must call 'yarn jest <...args>'
|
|
127
|
-
// to prevent module not found errors. Therefore we will call fork with the yarn/pnpm
|
|
128
|
-
// command and set jest to the first argument.
|
|
129
|
-
const frameworkArgs = {
|
|
130
|
-
jest: ['jest'],
|
|
131
|
-
playwright: ['playwright', 'test']
|
|
132
|
-
};
|
|
133
|
-
let args = frameworkArgs[framework];
|
|
134
|
-
// Set with ATTACH_JEST_DEBUGGER env variable
|
|
135
|
-
// Does not work with repos with pnp
|
|
136
|
-
if (attachJestDebugger && framework === TestFrameworks.jest) {
|
|
137
|
-
const nodeLinkerConfig = await getNodeLinkerConfig();
|
|
138
|
-
if (nodeLinkerConfig === 'node-modules') {
|
|
139
|
-
// Grab jest bin file
|
|
140
|
-
const jestBinCall = await execa(pm, ['bin', 'jest'], {
|
|
141
|
-
cwd: getRootDir()
|
|
142
|
-
});
|
|
143
|
-
if (jestBinCall.stderr.length) {
|
|
144
|
-
throw new Error(`Unable to find jest bin directory when calling "${pm} bin jest": ${jestBinCall.stderr}`);
|
|
145
|
-
}
|
|
146
|
-
const jestBinDir = jestBinCall.stdout;
|
|
147
|
-
args = [
|
|
148
|
-
'node',
|
|
149
|
-
'--inspect-brk=9230',
|
|
150
|
-
'--experimental-vm-modules',
|
|
151
|
-
jestBinDir
|
|
152
|
-
];
|
|
153
|
-
}
|
|
154
|
-
else {
|
|
155
|
-
signale.warn(`Projects with ${nodeLinkerConfig} are not compatible with `
|
|
156
|
-
+ `ATTACH_JEST_DEBUGGER env config and cannot be used. `
|
|
157
|
-
+ 'Only node-modules configuration is valid');
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
args.push(...mapToArgs(argsMap));
|
|
161
|
-
if (extraArgs) {
|
|
162
|
-
extraArgs.forEach((extraArg) => {
|
|
163
|
-
if (extraArg.startsWith('-') && args.includes(extraArg)) {
|
|
164
|
-
if (debug) {
|
|
165
|
-
logger.debug(`* skipping duplicate jest arg ${extraArg}`);
|
|
166
|
-
}
|
|
167
|
-
return;
|
|
168
|
-
}
|
|
169
|
-
args.push(extraArg);
|
|
170
|
-
});
|
|
171
|
-
}
|
|
172
|
-
if (frameworkConfig) {
|
|
173
|
-
args.push(`--config`);
|
|
174
|
-
args.push(frameworkConfig);
|
|
175
|
-
}
|
|
176
|
-
let logArgs = args;
|
|
177
|
-
if (framework === 'jest') {
|
|
178
|
-
logArgs = [...args];
|
|
179
|
-
const configIdx = args.findIndex((el) => el === '--config');
|
|
180
|
-
if (configIdx > -1 && args[configIdx + 1]?.startsWith('{')) {
|
|
181
|
-
logArgs[configIdx + 1] = '<STRINGIFIED_CONFIG>';
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
signale.info(`executing ${framework}: ${logArgs.join(' ')}`);
|
|
185
|
-
await fork({
|
|
186
|
-
cmd: pm,
|
|
187
|
-
cwd,
|
|
188
|
-
args,
|
|
189
|
-
env,
|
|
190
|
-
});
|
|
191
|
-
}
|
|
192
|
-
async function getNodeLinkerConfig() {
|
|
193
|
-
const pm = getPackageManager();
|
|
194
|
-
// pnpm always uses node-modules (with symlinks)
|
|
195
|
-
if (pm === 'pnpm') {
|
|
196
|
-
return 'node-modules';
|
|
197
|
-
}
|
|
198
|
-
try {
|
|
199
|
-
const { stdout: nodeLinkerconfig, stderr } = await execa('yarn', ['config', 'get', 'nodeLinker'], {
|
|
200
|
-
cwd: getRootDir()
|
|
201
|
-
});
|
|
202
|
-
// If info is printed in stderr, there must have been an issue
|
|
203
|
-
if (stderr.length) {
|
|
204
|
-
throw new Error(stderr);
|
|
205
|
-
}
|
|
206
|
-
return nodeLinkerconfig;
|
|
207
|
-
}
|
|
208
|
-
catch (err) {
|
|
209
|
-
throw new Error(`Error trying to grab yarn nodeLinker config from the project: ${err.message}`);
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
export async function dockerPull(image, timeout = 0) {
|
|
213
|
-
try {
|
|
214
|
-
await exec({
|
|
215
|
-
cmd: 'docker',
|
|
216
|
-
args: ['pull', image],
|
|
217
|
-
timeout,
|
|
218
|
-
});
|
|
219
|
-
}
|
|
220
|
-
catch (err) {
|
|
221
|
-
process.exitCode = 0;
|
|
222
|
-
throw err;
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
export async function dockerStop(name) {
|
|
226
|
-
await exec({
|
|
227
|
-
cmd: 'docker',
|
|
228
|
-
args: ['stop', name],
|
|
229
|
-
});
|
|
230
|
-
}
|
|
231
|
-
export async function dockerTag(from, to) {
|
|
232
|
-
signale.pending(`Tagging image ${from} as ${to}`);
|
|
233
|
-
await exec({
|
|
234
|
-
cmd: 'docker',
|
|
235
|
-
args: ['tag', from, to],
|
|
236
|
-
});
|
|
237
|
-
signale.success(`Image ${from} re-tagged as ${to}`);
|
|
238
|
-
}
|
|
239
|
-
export async function getNodeVersionFromImage(image) {
|
|
240
|
-
try {
|
|
241
|
-
const { stdout } = await execa('docker', ['run', image, 'node', '-v']);
|
|
242
|
-
return stdout;
|
|
243
|
-
}
|
|
244
|
-
catch (err) {
|
|
245
|
-
throw new Error(`Unable to get node version from image due to Error: ${err}`);
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
export async function getContainerInfo(name) {
|
|
249
|
-
const result = await exec({
|
|
250
|
-
cmd: 'docker',
|
|
251
|
-
args: ['ps', '--format={{json .}}', `--filter=name=${name}`],
|
|
252
|
-
});
|
|
253
|
-
if (!result)
|
|
254
|
-
return null;
|
|
255
|
-
return JSON.parse(result);
|
|
256
|
-
}
|
|
257
|
-
export async function dockerNetworkExists(name) {
|
|
258
|
-
const subprocess = await execaCommand(`docker network ls --format='{{json .Name}}' | grep '"${name}"'`, { reject: false, shell: true });
|
|
259
|
-
return subprocess.exitCode ? subprocess.exitCode > 0 : false;
|
|
260
|
-
}
|
|
261
|
-
export async function remoteDockerImageExists(image) {
|
|
262
|
-
try {
|
|
263
|
-
await dockerPull(image, ms('30s'));
|
|
264
|
-
return true;
|
|
265
|
-
}
|
|
266
|
-
catch (err) {
|
|
267
|
-
return false;
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
export async function dockerRun(opt, tag, ignoreMount, debug) {
|
|
271
|
-
const args = ['run', '--rm'];
|
|
272
|
-
if (!opt.image) {
|
|
273
|
-
throw new Error('Missing required image option');
|
|
274
|
-
}
|
|
275
|
-
if (!opt.name) {
|
|
276
|
-
throw new Error('Missing required name option');
|
|
277
|
-
}
|
|
278
|
-
if (opt.mount && !ignoreMount) {
|
|
279
|
-
for (const mount of opt.mount) {
|
|
280
|
-
args.push('--mount', mount);
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
if (opt.ports && opt.ports.length) {
|
|
284
|
-
opt.ports.forEach((port) => {
|
|
285
|
-
if (isString(port)) {
|
|
286
|
-
args.push('--publish', port);
|
|
287
|
-
}
|
|
288
|
-
else {
|
|
289
|
-
args.push('--publish', `${port}:${port}`);
|
|
290
|
-
}
|
|
291
|
-
});
|
|
292
|
-
}
|
|
293
|
-
if (opt.env) {
|
|
294
|
-
Object.entries(opt.env).forEach(([key, val]) => {
|
|
295
|
-
args.push('--env', `${key}=${val}`);
|
|
296
|
-
});
|
|
297
|
-
}
|
|
298
|
-
if (opt.tmpfs && opt.tmpfs.length) {
|
|
299
|
-
args.push('--tmpfs', opt.tmpfs.join(','));
|
|
300
|
-
}
|
|
301
|
-
if (opt.network) {
|
|
302
|
-
const exists = await dockerNetworkExists(opt.network);
|
|
303
|
-
if (!exists) {
|
|
304
|
-
throw new Error(`Docker network ${opt.network} does not exist`);
|
|
305
|
-
}
|
|
306
|
-
args.push('--network', opt.network);
|
|
307
|
-
}
|
|
308
|
-
args.push('--name', opt.name);
|
|
309
|
-
args.push(`${opt.image}:${tag ?? 'latest'}`);
|
|
310
|
-
if (opt.args) {
|
|
311
|
-
args.push(...opt.args);
|
|
312
|
-
}
|
|
313
|
-
let error;
|
|
314
|
-
let stderr;
|
|
315
|
-
let done = true;
|
|
316
|
-
if (debug) {
|
|
317
|
-
signale.debug(`executing: docker ${args.join(' ')}`);
|
|
318
|
-
}
|
|
319
|
-
const subprocess = execa('docker', args);
|
|
320
|
-
if (!subprocess || !subprocess.stderr) {
|
|
321
|
-
throw new Error('Failed to execute docker run');
|
|
322
|
-
}
|
|
323
|
-
(async () => {
|
|
324
|
-
done = false;
|
|
325
|
-
try {
|
|
326
|
-
const result = await subprocess;
|
|
327
|
-
if (result.exitCode && result.exitCode > 0) {
|
|
328
|
-
stderr = result.all;
|
|
329
|
-
error = new Error(`${result.command} failed`);
|
|
330
|
-
}
|
|
331
|
-
}
|
|
332
|
-
catch (err) {
|
|
333
|
-
if (err.failed) {
|
|
334
|
-
error = err.stderr;
|
|
335
|
-
}
|
|
336
|
-
error = err.stack;
|
|
337
|
-
stderr = err.all;
|
|
338
|
-
}
|
|
339
|
-
finally {
|
|
340
|
-
done = true;
|
|
341
|
-
}
|
|
342
|
-
})();
|
|
343
|
-
const upFor = ms('3s');
|
|
344
|
-
await pWhile(() => dockerContainerReady(opt.name, upFor, error), {
|
|
345
|
-
name: `Docker container up for 2m (${opt.name})`,
|
|
346
|
-
timeoutMs: ms('2m')
|
|
347
|
-
});
|
|
348
|
-
if (error) {
|
|
349
|
-
if (stderr) {
|
|
350
|
-
process.stderr.write(stderr);
|
|
351
|
-
}
|
|
352
|
-
throw error;
|
|
353
|
-
}
|
|
354
|
-
if (done) {
|
|
355
|
-
throw new Error('Service ended early');
|
|
356
|
-
}
|
|
357
|
-
return () => {
|
|
358
|
-
if (error) {
|
|
359
|
-
if (stderr) {
|
|
360
|
-
process.stderr.write(stderr);
|
|
361
|
-
}
|
|
362
|
-
signale.error(error);
|
|
363
|
-
}
|
|
364
|
-
if (done && !subprocess.killed)
|
|
365
|
-
return;
|
|
366
|
-
subprocess.kill();
|
|
367
|
-
};
|
|
368
|
-
}
|
|
369
|
-
export async function dockerContainerReady(name, upFor, error) {
|
|
370
|
-
if (error)
|
|
371
|
-
throw new TSError(error);
|
|
372
|
-
try {
|
|
373
|
-
const result = await exec({
|
|
374
|
-
cmd: 'docker',
|
|
375
|
-
args: [
|
|
376
|
-
'ps', '--format', '"{{json .Status}}"', '--filter', `name=${name}`
|
|
377
|
-
]
|
|
378
|
-
});
|
|
379
|
-
const timeup = ms(result.replace(/[(Up)\s"]+|/ig, ''));
|
|
380
|
-
if (!timeup)
|
|
381
|
-
return false;
|
|
382
|
-
return timeup >= upFor;
|
|
383
|
-
}
|
|
384
|
-
catch (err) {
|
|
385
|
-
await pDelay(1000);
|
|
386
|
-
return false;
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
export async function dockerBuild(tag, cacheFrom, target, buildArgs, dockerFileName, dockerFilePath) {
|
|
390
|
-
const cacheFromArgs = [];
|
|
391
|
-
cacheFrom?.forEach((image) => {
|
|
392
|
-
cacheFromArgs.push('--cache-from', image);
|
|
393
|
-
});
|
|
394
|
-
const targetArgs = target ? ['--target', target] : [];
|
|
395
|
-
const buildsArgs = buildArgs
|
|
396
|
-
? ['--build-arg', ...buildArgs.join(',--build-arg,').split(',')]
|
|
397
|
-
: [];
|
|
398
|
-
const fileName = dockerFileName ? ['-f', dockerFileName] : [];
|
|
399
|
-
const filePath = dockerFilePath ?? '.';
|
|
400
|
-
await fork({
|
|
401
|
-
cmd: 'docker',
|
|
402
|
-
args: ['build', ...cacheFromArgs, ...targetArgs, ...buildsArgs, '--tag', tag, ...fileName, filePath],
|
|
403
|
-
});
|
|
404
|
-
}
|
|
405
|
-
export async function dockerPush(image) {
|
|
406
|
-
const subprocess = await execaCommand(`docker push ${image}`, { reject: false });
|
|
407
|
-
if (subprocess.exitCode !== 0) {
|
|
408
|
-
throw new Error(`Unable to push docker image ${image}, ${subprocess.stderr}`);
|
|
409
|
-
}
|
|
410
|
-
}
|
|
411
|
-
export async function dockerExec(containerName, command) {
|
|
412
|
-
const args = ['exec', containerName, ...command];
|
|
413
|
-
signale.debug(`executing: docker ${args.join(' ')}`);
|
|
414
|
-
const subprocess = await execa('docker', args, { stdio: 'pipe' });
|
|
415
|
-
if (subprocess.exitCode !== 0) {
|
|
416
|
-
throw new Error(`Docker exec to container ${containerName} failed, ${subprocess.stderr}`);
|
|
417
|
-
}
|
|
418
|
-
}
|
|
419
|
-
async function dockerImageRm(image) {
|
|
420
|
-
const subprocess = await execaCommand(`docker image rm ${image}`, { reject: false });
|
|
421
|
-
if (subprocess.exitCode !== 0) {
|
|
422
|
-
throw new Error(`Unable to remove docker image ${image}, ${subprocess.stderr}`);
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
/**
|
|
426
|
-
* Unzips and loads a Docker image from a Docker cache
|
|
427
|
-
* If successful and skipDelete is false the image will be deleted from the cache
|
|
428
|
-
* @param {string} imageName Name of the image to load
|
|
429
|
-
* @param {boolean} skipDelete Skip removal of docker image from cache
|
|
430
|
-
* @returns {Promise<boolean>} Whether or not the image loaded successfully
|
|
431
|
-
*/
|
|
432
|
-
export async function loadThenDeleteImageFromCache(imageName, skipDelete = false) {
|
|
433
|
-
signale.time(`unzip and load ${imageName}`);
|
|
434
|
-
const fileName = imageName.trim().replace(/[/:]/g, '_');
|
|
435
|
-
const filePath = path.join(config.DOCKER_CACHE_PATH, `${fileName}.tar.gz`);
|
|
436
|
-
if (!fs.existsSync(filePath)) {
|
|
437
|
-
signale.error(`No file found at ${filePath}. Have you restored the cache?`);
|
|
438
|
-
return false;
|
|
439
|
-
}
|
|
440
|
-
const result = await execaCommand(`gunzip -c ${filePath} | docker load`, { shell: true });
|
|
441
|
-
signale.info('Result: ', result);
|
|
442
|
-
if (result.exitCode !== 0) {
|
|
443
|
-
signale.error(`Error loading ${filePath} to docker`);
|
|
444
|
-
return false;
|
|
445
|
-
}
|
|
446
|
-
if (!skipDelete) {
|
|
447
|
-
signale.info(`Deleting ${imageName} from docker image cache.`);
|
|
448
|
-
fs.rmSync(filePath);
|
|
449
|
-
}
|
|
450
|
-
signale.timeEnd(`unzip and load ${imageName}`);
|
|
451
|
-
return true;
|
|
452
|
-
}
|
|
453
|
-
export async function deleteDockerImageCache() {
|
|
454
|
-
signale.info(`Deleting Docker image cache at ${config.DOCKER_CACHE_PATH}`);
|
|
455
|
-
fse.removeSync(config.DOCKER_CACHE_PATH);
|
|
456
|
-
}
|
|
457
|
-
export async function pgrep(name) {
|
|
458
|
-
const result = await exec({ cmd: 'ps', args: ['aux'] }, false);
|
|
459
|
-
if (!result) {
|
|
460
|
-
throw new Error('Invalid result from ps aux');
|
|
461
|
-
}
|
|
462
|
-
const found = result.split('\n').find((line) => {
|
|
463
|
-
if (!line)
|
|
464
|
-
return false;
|
|
465
|
-
return line.toLowerCase().includes(name.toLowerCase());
|
|
466
|
-
});
|
|
467
|
-
if (found) {
|
|
468
|
-
logger.trace('found process', found);
|
|
469
|
-
return found;
|
|
470
|
-
}
|
|
471
|
-
return '';
|
|
472
|
-
}
|
|
473
|
-
/**
|
|
474
|
-
* Save a docker image as a tar.gz to a local directory.
|
|
475
|
-
* Then remove the image from docker
|
|
476
|
-
* @param {string} imageName Name of image to pull and save
|
|
477
|
-
* @param {string} imageSavePath Location where image will be saved and compressed.
|
|
478
|
-
* @returns void
|
|
479
|
-
*/
|
|
480
|
-
export async function saveAndZip(imageName, imageSavePath) {
|
|
481
|
-
signale.info(`Saving Docker image: ${imageName}`);
|
|
482
|
-
const fileName = imageName.replace(/[/:]/g, '_');
|
|
483
|
-
const filePath = path.join(imageSavePath, `${fileName}.tar`);
|
|
484
|
-
const command = `docker save ${imageName} | gzip > ${filePath}.gz`;
|
|
485
|
-
await execaCommand(command, { shell: true });
|
|
486
|
-
await dockerImageRm(imageName);
|
|
487
|
-
}
|
|
488
|
-
export async function getCommitHash() {
|
|
489
|
-
if (process.env.GIT_COMMIT_HASH)
|
|
490
|
-
return process.env.GIT_COMMIT_HASH;
|
|
491
|
-
if (config.SKIP_GIT_COMMANDS) {
|
|
492
|
-
throw new Error('Unable to determine git commit hash when env.SKIP_GIT_COMMANDS is set, set env.GIT_COMMIT_HASH to fix this');
|
|
493
|
-
}
|
|
494
|
-
return exec({ cmd: 'git', args: ['rev-parse', '--short', 'HEAD'] });
|
|
495
|
-
}
|
|
496
|
-
export async function gitDiff(files = []) {
|
|
497
|
-
if (config.SKIP_GIT_COMMANDS)
|
|
498
|
-
return;
|
|
499
|
-
try {
|
|
500
|
-
await fork({ cmd: 'git', args: ['diff', ...files] });
|
|
501
|
-
}
|
|
502
|
-
catch (e) {
|
|
503
|
-
process.exitCode = 0;
|
|
504
|
-
logger.warn(e);
|
|
505
|
-
}
|
|
506
|
-
}
|
|
507
|
-
export async function getChangedFiles(...files) {
|
|
508
|
-
if (config.SKIP_GIT_COMMANDS)
|
|
509
|
-
return [];
|
|
510
|
-
try {
|
|
511
|
-
const result = await exec({
|
|
512
|
-
cmd: 'git', args: ['diff', '--name-only', ...files]
|
|
513
|
-
});
|
|
514
|
-
return result
|
|
515
|
-
.split('\n')
|
|
516
|
-
.map((str) => str.trim())
|
|
517
|
-
.filter(Boolean);
|
|
518
|
-
}
|
|
519
|
-
catch (e) {
|
|
520
|
-
// if there error includes ENOENT
|
|
521
|
-
// we can just log the error and continue on
|
|
522
|
-
if (e.toString().includes('ENOENT')) {
|
|
523
|
-
process.exitCode = 0;
|
|
524
|
-
logger.warn(e);
|
|
525
|
-
return [];
|
|
526
|
-
}
|
|
527
|
-
throw e;
|
|
528
|
-
}
|
|
529
|
-
}
|
|
530
|
-
export function mapToArgs(input) {
|
|
531
|
-
const args = [];
|
|
532
|
-
for (const [key, value] of Object.entries(input)) {
|
|
533
|
-
const vals = Array.isArray(value) ? value : [value];
|
|
534
|
-
if (key.length > 1) {
|
|
535
|
-
args.push(`--${key}`, ...vals);
|
|
536
|
-
}
|
|
537
|
-
else {
|
|
538
|
-
args.push(`-${key}`, ...vals);
|
|
539
|
-
}
|
|
540
|
-
}
|
|
541
|
-
return args.filter((str) => str != null && str !== '');
|
|
542
|
-
}
|
|
543
|
-
/**
|
|
544
|
-
* Publish package to NPM using yarn (versions 2, 3, 4) or pnpm
|
|
545
|
-
*/
|
|
546
|
-
export async function packageMngrPublish(pkgInfo, tag = 'latest') {
|
|
547
|
-
const pm = getPackageManager();
|
|
548
|
-
const args = pm === 'pnpm'
|
|
549
|
-
? ['publish', '--tag', tag, '--no-git-checks']
|
|
550
|
-
: ['npm', 'publish', '--tag', tag];
|
|
551
|
-
await fork({
|
|
552
|
-
cmd: pm,
|
|
553
|
-
args,
|
|
554
|
-
cwd: pkgInfo.dir,
|
|
555
|
-
env: {
|
|
556
|
-
NODE_ENV: 'production'
|
|
557
|
-
}
|
|
558
|
-
});
|
|
559
|
-
}
|
|
560
|
-
export async function isKindInstalled() {
|
|
561
|
-
try {
|
|
562
|
-
const subprocess = await execaCommand('command -v kind');
|
|
563
|
-
return !!subprocess.stdout;
|
|
564
|
-
}
|
|
565
|
-
catch (err) {
|
|
566
|
-
return false;
|
|
567
|
-
}
|
|
568
|
-
}
|
|
569
|
-
export async function isKubectlInstalled() {
|
|
570
|
-
try {
|
|
571
|
-
const subprocess = await execaCommand('command -v kubectl');
|
|
572
|
-
return !!subprocess.stdout;
|
|
573
|
-
}
|
|
574
|
-
catch (err) {
|
|
575
|
-
return false;
|
|
576
|
-
}
|
|
577
|
-
}
|
|
578
|
-
export async function isHelmInstalled() {
|
|
579
|
-
try {
|
|
580
|
-
const subprocess = await execaCommand('command -v helm');
|
|
581
|
-
return !!subprocess.stdout;
|
|
582
|
-
}
|
|
583
|
-
catch (err) {
|
|
584
|
-
return false;
|
|
585
|
-
}
|
|
586
|
-
}
|
|
587
|
-
export async function isHelmfileInstalled() {
|
|
588
|
-
try {
|
|
589
|
-
const subprocess = await execaCommand('command -v helmfile');
|
|
590
|
-
return !!subprocess.stdout;
|
|
591
|
-
}
|
|
592
|
-
catch (err) {
|
|
593
|
-
return false;
|
|
594
|
-
}
|
|
595
|
-
}
|
|
596
|
-
function waitForKafkaRunning(name, timeoutMs = 12000) {
|
|
597
|
-
const endAt = Date.now() + timeoutMs;
|
|
598
|
-
const _waitForKafkaRunning = async () => {
|
|
599
|
-
if (Date.now() > endAt) {
|
|
600
|
-
if (logger.level() <= 20) {
|
|
601
|
-
try {
|
|
602
|
-
const errorSearchCommand = await execaCommand(`kubectl -n services-dev1 logs -l app.kubernetes.io/name=${name}`);
|
|
603
|
-
logger.debug('Kafka pod logs:');
|
|
604
|
-
logger.debug(errorSearchCommand.stdout);
|
|
605
|
-
}
|
|
606
|
-
catch (err) {
|
|
607
|
-
logger.error(err, 'Failure to retrieve kafka pod logs');
|
|
608
|
-
const describePodCommand = await execaCommand(`kubectl -n services-dev1 describe pods -l app.kubernetes.io/name=${name}`);
|
|
609
|
-
logger.debug('Describe kafka pod:');
|
|
610
|
-
logger.debug(describePodCommand.stdout);
|
|
611
|
-
}
|
|
612
|
-
}
|
|
613
|
-
throw new Error(`Failure to communicate with kafka after ${timeoutMs}ms`);
|
|
614
|
-
}
|
|
615
|
-
let kafkaRunning = false;
|
|
616
|
-
try {
|
|
617
|
-
const kubectlResponse = await execaCommand(`kubectl -n services-dev1 get pods -l app.kubernetes.io/name=${name} -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"`);
|
|
618
|
-
const kafkaReady = kubectlResponse.stdout;
|
|
619
|
-
logger.debug('Kafka ready: ', kafkaReady);
|
|
620
|
-
if (kafkaReady === '"true"') {
|
|
621
|
-
kafkaRunning = true;
|
|
622
|
-
}
|
|
623
|
-
}
|
|
624
|
-
catch (err) {
|
|
625
|
-
await pDelay(3000);
|
|
626
|
-
return _waitForKafkaRunning();
|
|
627
|
-
}
|
|
628
|
-
if (kafkaRunning) {
|
|
629
|
-
return;
|
|
630
|
-
}
|
|
631
|
-
await pDelay(3000);
|
|
632
|
-
return _waitForKafkaRunning();
|
|
633
|
-
};
|
|
634
|
-
return _waitForKafkaRunning();
|
|
635
|
-
}
|
|
636
|
-
export async function setAlias(tsPort) {
|
|
637
|
-
try {
|
|
638
|
-
const subprocess1 = await execaCommand('earl aliases remove k8s-e2e 2> /dev/null || true', { shell: true });
|
|
639
|
-
logger.debug(subprocess1.stdout);
|
|
640
|
-
if (subprocess1.stderr) {
|
|
641
|
-
throw new Error(subprocess1.stderr);
|
|
642
|
-
}
|
|
643
|
-
const subprocess2 = await execaCommand(`earl aliases add k8s-e2e http://${config.HOST_IP}:${tsPort}`);
|
|
644
|
-
logger.debug(subprocess2.stdout);
|
|
645
|
-
if (subprocess2.stderr) {
|
|
646
|
-
throw new Error(subprocess2.stderr);
|
|
647
|
-
}
|
|
648
|
-
}
|
|
649
|
-
catch (err) {
|
|
650
|
-
throw new Error(`Failed to set alias: ${err}`);
|
|
651
|
-
}
|
|
652
|
-
}
|
|
653
|
-
export async function showState(tsPort, isTeardown = false) {
|
|
654
|
-
try {
|
|
655
|
-
if (isTeardown) {
|
|
656
|
-
logger.debug('=== k8s cluster state ===');
|
|
657
|
-
const clusterState = await execaCommand('kubectl get deployments,po,svc --all-namespaces --show-labels -o wide');
|
|
658
|
-
logger.debug(clusterState.stdout);
|
|
659
|
-
logger.debug('=== docker stats ===');
|
|
660
|
-
const dockerStats = await execaCommand('docker stats --no-stream');
|
|
661
|
-
logger.debug(dockerStats.stdout);
|
|
662
|
-
logger.debug('=== opensearch2 pod description ===');
|
|
663
|
-
const searchHost = await determineSearchHost();
|
|
664
|
-
const os2Desc = await execaCommand(`kubectl -n services-dev1 describe pod ${searchHost}-cluster-master-0`);
|
|
665
|
-
logger.debug(os2Desc.stdout);
|
|
666
|
-
// TODO: consider adding describe for kafka and minio pods
|
|
667
|
-
}
|
|
668
|
-
const subprocess = await execaCommand('kubectl get deployments,po,svc --all-namespaces --show-labels -o wide');
|
|
669
|
-
logger.debug(subprocess.stdout);
|
|
670
|
-
logger.debug(await showESIndices());
|
|
671
|
-
logger.debug(await showAssets(tsPort));
|
|
672
|
-
}
|
|
673
|
-
catch (err) {
|
|
674
|
-
signale.error(`Failed to get k8s resources: ${err}`);
|
|
675
|
-
}
|
|
676
|
-
}
|
|
677
|
-
async function showESIndices() {
|
|
678
|
-
const subprocess = await execaCommand(`curl -k ${config.SEARCH_TEST_HOST}/_cat/indices?v`);
|
|
679
|
-
return subprocess.stdout;
|
|
680
|
-
}
|
|
681
|
-
async function showAssets(tsPort) {
|
|
682
|
-
try {
|
|
683
|
-
const subprocess = await execaCommand(`curl ${config.HOST_IP}:${tsPort}/v1/assets`);
|
|
684
|
-
return subprocess.stdout;
|
|
685
|
-
}
|
|
686
|
-
catch (err) {
|
|
687
|
-
return err;
|
|
688
|
-
}
|
|
689
|
-
}
|
|
690
|
-
export async function logTCPPorts(service) {
|
|
691
|
-
try {
|
|
692
|
-
const command = 'netstat -an | grep \'^tcp\' | awk \'{print $4}\' | tr ".:" " " | awk \'{print $NF}\' | sort -n | uniq | tr "\n" " "';
|
|
693
|
-
const subprocess = await execaCommand(command, { shell: true, reject: false });
|
|
694
|
-
const { stdout, stderr } = subprocess;
|
|
695
|
-
if (stderr) {
|
|
696
|
-
throw new Error(stderr);
|
|
697
|
-
}
|
|
698
|
-
signale.info(`TCP Ports currently in use when starting ${service}:\n ${stdout}`);
|
|
699
|
-
}
|
|
700
|
-
catch (err) {
|
|
701
|
-
signale.error(`Execa command failed trying to log ports: ${err}`);
|
|
702
|
-
}
|
|
703
|
-
}
|
|
704
|
-
export async function deletePersistentVolumeClaim(searchHost) {
|
|
705
|
-
try {
|
|
706
|
-
const label = searchHost.includes('opensearch') ? `app.kubernetes.io/instance=${searchHost}` : `app=${searchHost}-master`;
|
|
707
|
-
const subprocess = await execaCommand(`kubectl delete -n services-dev1 pvc -l ${label}`);
|
|
708
|
-
logger.debug(`kubectl delete pvc: ${subprocess.stdout}`);
|
|
709
|
-
}
|
|
710
|
-
catch (err) {
|
|
711
|
-
throw new TSError(`Failed to delete persistent volume claim:\n${err}`);
|
|
712
|
-
}
|
|
713
|
-
}
|
|
714
|
-
export async function helmfileDestroy(selector) {
|
|
715
|
-
const helmfilePath = path.join(getRootDir(), 'packages/scripts/helm/helmfile.yaml.gotmpl');
|
|
716
|
-
try {
|
|
717
|
-
const subprocess = await execaCommand(`helmfile destroy -f ${helmfilePath} --selector app=${selector}`);
|
|
718
|
-
logger.debug(`helmfile destroy:\n${subprocess.stdout}`);
|
|
719
|
-
}
|
|
720
|
-
catch (err) {
|
|
721
|
-
logger.info(err);
|
|
722
|
-
}
|
|
723
|
-
}
|
|
724
|
-
export async function helmfileCommand(command, clusteringType, devMode = false, logs = false, e2e = true) {
|
|
725
|
-
const helmfilePath = path.join(getRootDir(), 'packages/scripts/helm/helmfile.yaml.gotmpl');
|
|
726
|
-
const { valuesPath, valuesDir } = generateHelmValuesFromServices(clusteringType, devMode, logs, e2e);
|
|
727
|
-
let subprocess;
|
|
728
|
-
try {
|
|
729
|
-
subprocess = await execaCommand(`helmfile --state-values-file ${valuesPath} ${command} -f ${helmfilePath}`);
|
|
730
|
-
}
|
|
731
|
-
catch (err) {
|
|
732
|
-
// TSError truncates to 3000 characters which is an issue here
|
|
733
|
-
throw new Error(`Helmfile ${command} command failed:\n${err}`);
|
|
734
|
-
}
|
|
735
|
-
finally {
|
|
736
|
-
fs.rmSync(valuesDir, { recursive: true, force: true });
|
|
737
|
-
}
|
|
738
|
-
logger.debug(`helmfile ${command}:\n${subprocess.stdout}`);
|
|
739
|
-
}
|
|
740
|
-
export async function launchTerasliceWithHelmfile(clusteringType, devMode = false, logs = false, debug = false, e2e = true) {
|
|
741
|
-
if (debug && !isCI) {
|
|
742
|
-
await helmfileCommand('diff', clusteringType, devMode, logs, e2e);
|
|
743
|
-
}
|
|
744
|
-
await helmfileCommand('sync', clusteringType, devMode, logs, e2e);
|
|
745
|
-
if (config.ENV_SERVICES.includes(Service.Kafka)) {
|
|
746
|
-
await waitForKafkaRunning('kafka');
|
|
747
|
-
}
|
|
748
|
-
}
|
|
749
|
-
export async function launchTerasliceWithCustomHelmfile(configFilePath, debug = false, selector) {
|
|
750
|
-
let diffProcess;
|
|
751
|
-
let syncProcess;
|
|
752
|
-
const diffSelector = selector && selector.diff ? `-l group!=skipDiff,${selector.diff}` : '-l group!=skipDiff';
|
|
753
|
-
const syncSelector = selector && selector.sync ? `-l ${selector.sync}` : '';
|
|
754
|
-
const helmfilePath = path.join(getRootDir(), 'packages/scripts/helm/helmfile.yaml.gotmpl');
|
|
755
|
-
try {
|
|
756
|
-
if (debug && !isCI) {
|
|
757
|
-
// We want to exclude certain charts from the diff command because
|
|
758
|
-
// they may require crds that aren't installed
|
|
759
|
-
diffProcess = await execaCommand(`helmfile ${diffSelector} --state-values-file ${configFilePath} diff -f ${helmfilePath}`);
|
|
760
|
-
logger.debug(`helmfile diff:\n${diffProcess.stdout}`);
|
|
761
|
-
}
|
|
762
|
-
syncProcess = await execaCommand(`helmfile ${syncSelector} --state-values-file ${configFilePath} sync -f ${helmfilePath}`);
|
|
763
|
-
logger.debug(`helmfile sync:\n${syncProcess.stdout}`);
|
|
764
|
-
}
|
|
765
|
-
catch (err) {
|
|
766
|
-
// TSError truncates to 3000 characters which is an issue here
|
|
767
|
-
throw new Error(`Helmfile command failed:\n${err}`);
|
|
768
|
-
}
|
|
769
|
-
}
|
|
770
|
-
export async function determineSearchHost() {
|
|
771
|
-
const possible = ['opensearch1', 'opensearch2', 'opensearch3'];
|
|
772
|
-
const subprocess = await execaCommand('helm list -n services-dev1 -o json');
|
|
773
|
-
logger.debug(`helmfile list:\n${subprocess.stdout}`);
|
|
774
|
-
const serviceList = JSON.parse(subprocess.stdout);
|
|
775
|
-
const filtered = serviceList.filter((svc) => possible.includes(svc.name));
|
|
776
|
-
if (filtered.length > 1) {
|
|
777
|
-
throw new TSError('Multiple Possible Search Hosts Detected. Cannot reset store.');
|
|
778
|
-
}
|
|
779
|
-
if (filtered.length === 0) {
|
|
780
|
-
throw new TSError('No Search Host Detected. Cannot reset store.');
|
|
781
|
-
}
|
|
782
|
-
return filtered[0].name;
|
|
783
|
-
}
|
|
784
|
-
// Helper function for reading the contents of a certificate by providing its path
|
|
785
|
-
function readCertFromPath(certPath) {
|
|
786
|
-
if (!fs.existsSync(certPath)) {
|
|
787
|
-
throw new TSError(`Unable to find cert at: ${certPath}`);
|
|
788
|
-
}
|
|
789
|
-
return fs.readFileSync(certPath, 'utf8');
|
|
790
|
-
}
|
|
791
|
-
/**
|
|
792
|
-
* Extracts the admin distinguished name (DN) from a certificate.
|
|
793
|
-
*
|
|
794
|
-
* This function is designed specifically for mkcert generated certificates.
|
|
795
|
-
* It reads the certificate file (`opensearch-cert.pem`), extracts the `O`
|
|
796
|
-
* and `OU` fields from the subject, and formats them
|
|
797
|
-
* in the required order (`OU` first, `O` second) for OpenSearch authentication.
|
|
798
|
-
*
|
|
799
|
-
* @returns {string} The formatted (DN) string in the format:
|
|
800
|
-
* `"OU=example, O=example"`
|
|
801
|
-
* @throws {Error} If the certificate file is missing or invalid.
|
|
802
|
-
*
|
|
803
|
-
* @example
|
|
804
|
-
* ```ts
|
|
805
|
-
* const adminDn = getAdminDnFromCert();
|
|
806
|
-
* console.log(adminDn);
|
|
807
|
-
* // Output: "OU=anon@anon-MBP (Anon User),O=mkcert development certificate"
|
|
808
|
-
* ```
|
|
809
|
-
*/
|
|
810
|
-
export function getAdminDnFromCert() {
|
|
811
|
-
let ca;
|
|
812
|
-
let organization;
|
|
813
|
-
let organizationalUnit;
|
|
814
|
-
try {
|
|
815
|
-
ca = readCertFromPath(path.join(config.CERT_PATH, 'opensearch-cert.pem'));
|
|
816
|
-
}
|
|
817
|
-
catch (err) {
|
|
818
|
-
throw new TSError(`Failed to read certificate file (opensearch-cert.pem).\n${err}`);
|
|
819
|
-
}
|
|
820
|
-
try {
|
|
821
|
-
const rootCA = new X509Certificate(ca);
|
|
822
|
-
// This splits the OU and O in two separate parts
|
|
823
|
-
const subjectParts = rootCA.subject.split('\n');
|
|
824
|
-
// Loop through the parts and assign based on prefix
|
|
825
|
-
// We don't want to assume the order that these are returned
|
|
826
|
-
for (const part of subjectParts) {
|
|
827
|
-
if (part.startsWith('OU=')) {
|
|
828
|
-
organizationalUnit = part;
|
|
829
|
-
}
|
|
830
|
-
else if (part.startsWith('O=')) {
|
|
831
|
-
organization = part;
|
|
832
|
-
}
|
|
833
|
-
}
|
|
834
|
-
}
|
|
835
|
-
catch (err) {
|
|
836
|
-
throw new TSError(`Failed to parse openSearch certificate. Make sure it's a valid X.509 certificate.\n${err}`);
|
|
837
|
-
}
|
|
838
|
-
if (!organizationalUnit || !organization) {
|
|
839
|
-
throw new TSError(`Certificate is missing required fields. Expected both 'OU' and 'O' fields.`);
|
|
840
|
-
}
|
|
841
|
-
// Return with specific format that opensearch expects
|
|
842
|
-
return `${organizationalUnit},${organization}`;
|
|
843
|
-
}
|
|
844
|
-
/**
|
|
845
|
-
* Generates a temporary `values.yaml` file based on the ts-scripts command configuration.
|
|
846
|
-
* This file is used to configure Helmfile when launching the k8sEnv or test environment.
|
|
847
|
-
*
|
|
848
|
-
* The function:
|
|
849
|
-
* - Loads a base `values.yaml` template from `packages/scripts/helm/values.yaml`.
|
|
850
|
-
* - Enables services specified in `ENV_SERVICES`, setting their versions when needed
|
|
851
|
-
* - Configures OpenSearch to align with versioning conventions.
|
|
852
|
-
* - Handles OpenSearch, Minio and Kafka SSL settings if encryption is enabled.
|
|
853
|
-
* - Adds extraVolumes, extraVolumeMounts and env values if running in dev mode.
|
|
854
|
-
* - Generates a temporary directory to store the modified `values.yaml`.
|
|
855
|
-
*
|
|
856
|
-
* @param { 'kubernetesV2' } clusteringType - backend cluster manager type
|
|
857
|
-
* @param { boolean } devMode - Mount local teraslice to k8s resources for faster development.
|
|
858
|
-
* @param { boolean } logs - Copy teraslice and service logs from k8s pods to local filesystem.
|
|
859
|
-
* @param { boolean } e2e - Specify e2e tests or k8s env mode.
|
|
860
|
-
* @returns An object containing:
|
|
861
|
-
* - `valuesPath` - Path to the generated `values.yaml` file.
|
|
862
|
-
* - `valuesDir` - Path to the temporary directory containing the file.
|
|
863
|
-
*/
|
|
864
|
-
function generateHelmValuesFromServices(clusteringType, devMode, logs, e2e) {
|
|
865
|
-
// Grab default values from the packages/scripts/helm/values.yaml
|
|
866
|
-
const helmfileValuesPath = path.join(getRootDir(), 'packages/scripts/helm/values.yaml');
|
|
867
|
-
const values = parseDocument(fs.readFileSync(helmfileValuesPath, 'utf8'));
|
|
868
|
-
// Map services to versions used for the image tag
|
|
869
|
-
const versionMap = {
|
|
870
|
-
[Service.Opensearch]: config.OPENSEARCH_VERSION,
|
|
871
|
-
[Service.Kafka]: config.KAFKA_VERSION,
|
|
872
|
-
[Service.Minio]: config.MINIO_VERSION,
|
|
873
|
-
[Service.RabbitMQ]: config.RABBITMQ_VERSION,
|
|
874
|
-
[Service.RestrainedOpensearch]: config.OPENSEARCH_VERSION,
|
|
875
|
-
[Service.Utility]: config.UTILITY_SVC_VERSION,
|
|
876
|
-
[Service.Valkey]: config.VALKEY_VERSION,
|
|
877
|
-
[Service.Teraslice]: config.TERASLICE_DOCKER_IMAGE,
|
|
878
|
-
};
|
|
879
|
-
let stateCluster;
|
|
880
|
-
let caCert;
|
|
881
|
-
// disable chart's OS2 default
|
|
882
|
-
values.setIn(['opensearch2', 'enabled'], false);
|
|
883
|
-
// Iterate over each service we want to start and enable them in the
|
|
884
|
-
// helmfile.
|
|
885
|
-
config.ENV_SERVICES.forEach((service) => {
|
|
886
|
-
// "serviceString" represents the literal service name string
|
|
887
|
-
// in the "values.yaml"
|
|
888
|
-
let serviceString = service;
|
|
889
|
-
const version = versionMap[service];
|
|
890
|
-
if (service === Service.Opensearch) {
|
|
891
|
-
const major = config.OPENSEARCH_VERSION.charAt(0);
|
|
892
|
-
serviceString += major;
|
|
893
|
-
// This assumes there is only one search service enabled. If both ES and OS services
|
|
894
|
-
// are present the state cluster will be set to elasticsearch below.
|
|
895
|
-
stateCluster = serviceString;
|
|
896
|
-
if (config.ENCRYPT_OPENSEARCH) {
|
|
897
|
-
if (major === '1') {
|
|
898
|
-
throw new TSError('Encrypted Opensearch version 1 is not enabled. Please use OS2 or OS3.');
|
|
899
|
-
}
|
|
900
|
-
if (!caCert) {
|
|
901
|
-
caCert = readCertFromPath(path.join(config.CERT_PATH, 'CAs/rootCA.pem')).replace(/\n/g, '\\n');
|
|
902
|
-
}
|
|
903
|
-
const admin_dn = getAdminDnFromCert();
|
|
904
|
-
values.setIn([serviceString, 'ssl', 'enabled'], true);
|
|
905
|
-
values.setIn([serviceString, 'ssl', 'caCert'], caCert);
|
|
906
|
-
values.setIn([serviceString, 'ssl', 'admin_dn'], admin_dn);
|
|
907
|
-
}
|
|
908
|
-
}
|
|
909
|
-
if (service === Service.Kafka) {
|
|
910
|
-
if (config.ENCRYPT_KAFKA) {
|
|
911
|
-
if (!caCert) {
|
|
912
|
-
caCert = readCertFromPath(path.join(config.CERT_PATH, 'CAs/rootCA.pem')).replace(/\n/g, '\\n');
|
|
913
|
-
}
|
|
914
|
-
values.setIn(['kafka', 'ssl', 'enabled'], true);
|
|
915
|
-
values.setIn(['kafka', 'ssl', 'caCert'], caCert);
|
|
916
|
-
}
|
|
917
|
-
// Listener settings go into configurationOverrides (dotted Kafka key format) so the
|
|
918
|
-
// chart's brokerConfigs helper merges them with internal defaults without creating
|
|
919
|
-
// duplicate env vars. Values come from config.ts which sets k8s-appropriate defaults
|
|
920
|
-
// based on TEST_PLATFORM and ENCRYPT_KAFKA. If you add new listener settings, add
|
|
921
|
-
// them to config.ts and include them in this object.
|
|
922
|
-
const kafkaK8sConfigOverrides = {
|
|
923
|
-
'advertised.listeners': config.KAFKA_ADVERTISED_LISTENERS,
|
|
924
|
-
'listener.security.protocol.map': config.KAFKA_LISTENER_SECURITY_PROTOCOL_MAP,
|
|
925
|
-
listeners: config.KAFKA_LISTENERS,
|
|
926
|
-
};
|
|
927
|
-
if (config.ENCRYPT_KAFKA) {
|
|
928
|
-
// PLAINTEXT is the inter-broker listener in the SSL setup: brokers talk to each
|
|
929
|
-
// other on the internal PLAINTEXT listener while clients connect via TLS.
|
|
930
|
-
kafkaK8sConfigOverrides['inter.broker.listener.name'] = 'PLAINTEXT';
|
|
931
|
-
kafkaK8sConfigOverrides['security.protocol'] = config.KAFKA_SECURITY_PROTOCOL.toLowerCase();
|
|
932
|
-
}
|
|
933
|
-
values.setIn(['kafka', 'configurationOverrides'], kafkaK8sConfigOverrides);
|
|
934
|
-
// Only behavioral vars belong in envOverrides — the chart does not generate these
|
|
935
|
-
// internally so there is no duplicate risk.
|
|
936
|
-
// If you add a new Kafka env var to config.ts and services.ts, add it here too.
|
|
937
|
-
values.setIn(['kafka', 'envOverrides', 'KAFKA_AUTO_CREATE_TOPICS_ENABLE'], config.KAFKA_AUTO_CREATE_TOPICS_ENABLE);
|
|
938
|
-
values.setIn(['kafka', 'envOverrides', 'KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR'], config.KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR);
|
|
939
|
-
values.setIn(['kafka', 'envOverrides', 'KAFKA_TRANSACTION_STATE_LOG_MIN_ISR'], config.KAFKA_TRANSACTION_STATE_LOG_MIN_ISR);
|
|
940
|
-
values.setIn(['kafka', 'envOverrides', 'KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS'], config.KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS);
|
|
941
|
-
}
|
|
942
|
-
if (service === Service.Minio) {
|
|
943
|
-
if (config.ENCRYPT_MINIO) {
|
|
944
|
-
if (!caCert) {
|
|
945
|
-
caCert = readCertFromPath(path.join(config.CERT_PATH, 'CAs/rootCA.pem')).replace(/\n/g, '\\n');
|
|
946
|
-
}
|
|
947
|
-
const publicCert = readCertFromPath(path.join(config.CERT_PATH, 'public.crt')).replace(/\n/g, '\\n');
|
|
948
|
-
const privateKey = readCertFromPath(path.join(config.CERT_PATH, 'private.key')).replace(/\n/g, '\\n');
|
|
949
|
-
values.setIn(['minio', 'tls', 'enabled'], true);
|
|
950
|
-
values.setIn(['minio', 'tls', 'caCert'], caCert);
|
|
951
|
-
values.setIn(['minio', 'tls', 'publicCert'], publicCert);
|
|
952
|
-
values.setIn(['minio', 'tls', 'privateKey'], privateKey);
|
|
953
|
-
values.setIn(['minio', 'tls', 'certSecret'], 'tls-ssl-minio');
|
|
954
|
-
}
|
|
955
|
-
}
|
|
956
|
-
if (service === Service.Valkey) {
|
|
957
|
-
if (config.ENCRYPT_VALKEY) {
|
|
958
|
-
throw new Error('Valkey encryption not supported');
|
|
959
|
-
}
|
|
960
|
-
}
|
|
961
|
-
values.setIn([serviceString, 'enabled'], true);
|
|
962
|
-
values.setIn([serviceString, 'version'], version);
|
|
963
|
-
});
|
|
964
|
-
// If no search service specified then default to OS2
|
|
965
|
-
if (!stateCluster) {
|
|
966
|
-
stateCluster = 'opensearch2';
|
|
967
|
-
values.setIn(['opensearch2', 'enabled'], true);
|
|
968
|
-
}
|
|
969
|
-
values.setIn(['teraslice', 'stateCluster'], stateCluster);
|
|
970
|
-
values.setIn(['teraslice', 'image', 'tag'], `e2e-nodev${config.NODE_VERSION}`);
|
|
971
|
-
values.setIn(['teraslice', 'asset_storage_connection_type'], config.ASSET_STORAGE_CONNECTION_TYPE);
|
|
972
|
-
values.setIn(['teraslice', 'asset_storage_connection'], config.ASSET_STORAGE_CONNECTION);
|
|
973
|
-
values.setIn(['teraslice', 'cluster_manager_type'], clusteringType);
|
|
974
|
-
values.setIn(['teraslice', 'name'], config.CLUSTER_NAME);
|
|
975
|
-
values.setIn(['teraslice', 'assets_directory'], e2e ? '/app/e2e-assets' : '/app/assets');
|
|
976
|
-
values.setIn(['teraslice', 'e2e'], e2e);
|
|
977
|
-
if (devMode) {
|
|
978
|
-
// Dev image runs as root and writes/compiles into the bind-mounted source at
|
|
979
|
-
// runtime, so the chart's hardened securityContext won't work. Override the
|
|
980
|
-
// blocking fields back to permissive (explicit values, since Helm deep-merges
|
|
981
|
-
// and an empty {} would leave the hardened defaults in place).
|
|
982
|
-
values.setIn(['teraslice', 'securityContext'], {
|
|
983
|
-
runAsUser: 0,
|
|
984
|
-
runAsNonRoot: false,
|
|
985
|
-
readOnlyRootFilesystem: false
|
|
986
|
-
});
|
|
987
|
-
const dockerfileMounts = getVolumesFromDockerfile(false, logger);
|
|
988
|
-
// Shared node_modules volume: lives on the Kind node
|
|
989
|
-
// mounted into all pods so only the master runs pnpm install once.
|
|
990
|
-
dockerfileMounts.volumes.push({
|
|
991
|
-
name: 'dev-node-modules',
|
|
992
|
-
hostPath: { path: '/dev-node-modules', type: 'DirectoryOrCreate' }
|
|
993
|
-
});
|
|
994
|
-
dockerfileMounts.volumeMounts.push({
|
|
995
|
-
name: 'dev-node-modules',
|
|
996
|
-
mountPath: '/app/source/node_modules'
|
|
997
|
-
});
|
|
998
|
-
values.setIn(['teraslice', 'extraVolumeMounts'], dockerfileMounts.volumeMounts);
|
|
999
|
-
values.setIn(['teraslice', 'extraVolumes'], dockerfileMounts.volumes);
|
|
1000
|
-
/// Pass in env so master passes volumes to ex's and workers
|
|
1001
|
-
values.setIn(['teraslice', 'env'], {
|
|
1002
|
-
MOUNT_LOCAL_TERASLICE: Buffer.from(JSON.stringify(dockerfileMounts)).toString('base64')
|
|
1003
|
-
});
|
|
1004
|
-
}
|
|
1005
|
-
if (logs) {
|
|
1006
|
-
values.setIn(['stern', 'enabled'], true);
|
|
1007
|
-
const arch = os.arch() === 'x64' ? 'amd64' : 'arm64';
|
|
1008
|
-
const sternVersion = '1.30.0';
|
|
1009
|
-
values.setIn(['stern', 'downloadUrl'], `https://github.com/stern/stern/releases/download/v${sternVersion}/stern_${sternVersion}_linux_${arch}.tar.gz`);
|
|
1010
|
-
}
|
|
1011
|
-
signale.debug('helmfile command values: ', JSON.stringify(values));
|
|
1012
|
-
// Write the values to a temporary file
|
|
1013
|
-
const valuesDir = fs.mkdtempSync(path.join(os.tmpdir(), 'generated-yaml'));
|
|
1014
|
-
const valuesPath = path.join(valuesDir, 'values.yaml');
|
|
1015
|
-
fs.writeFileSync(valuesPath, values.toString(), 'utf8');
|
|
1016
|
-
return { valuesPath, valuesDir };
|
|
1017
|
-
}
|
|
1018
|
-
/**
|
|
1019
|
-
* Gets the current version of the Teraslice Helm chart from `Chart.yaml`.
|
|
1020
|
-
*
|
|
1021
|
-
* @throws {Error} If the `Chart.yaml` file cannot be read
|
|
1022
|
-
* @returns {Promise<string>} Resolves with the Helm chart version as a string
|
|
1023
|
-
*/
|
|
1024
|
-
export async function getCurrentHelmChartVersion() {
|
|
1025
|
-
const chartYamlPath = path.join(getRootDir(), '/helm/teraslice/Chart.yaml');
|
|
1026
|
-
const chartYAML = await load(fs.readFileSync(chartYamlPath, 'utf8'));
|
|
1027
|
-
return chartYAML.version;
|
|
1028
|
-
}
|
|
1029
|
-
function getTerasliceVersion() {
|
|
1030
|
-
const rootPackageInfo = getRootInfo();
|
|
1031
|
-
return rootPackageInfo.version;
|
|
1032
|
-
}
|
|
1033
|
-
/**
|
|
1034
|
-
* Extracts the base Docker image information from the top-level Dockerfile
|
|
1035
|
-
*
|
|
1036
|
-
* This function parses the Dockerfile to determine the base image name,
|
|
1037
|
-
* node version, registry, and repository.
|
|
1038
|
-
*
|
|
1039
|
-
* @throws {TSError} If the Dockerfile cannot be read or the base image format is unexpected.
|
|
1040
|
-
* @returns {{
|
|
1041
|
-
* name: string;
|
|
1042
|
-
* tag: string;
|
|
1043
|
-
* registry: string;
|
|
1044
|
-
* repo: string;
|
|
1045
|
-
* }} An object containing:
|
|
1046
|
-
* - `name: Full base image name
|
|
1047
|
-
* - `tag`: Node version used in the image
|
|
1048
|
-
* - `registry`: Docker registry (defaults to `docker.io` if not specified)
|
|
1049
|
-
* - `repo`: Repository name including organization
|
|
1050
|
-
*/
|
|
1051
|
-
export function getDockerBaseImageInfo() {
|
|
1052
|
-
try {
|
|
1053
|
-
const dockerFilePath = path.join(getRootDir(), 'Dockerfile');
|
|
1054
|
-
const dockerfileContent = fs.readFileSync(dockerFilePath, 'utf8');
|
|
1055
|
-
// Grab the "ARG NODE_VERSION" line in the Dockerfile
|
|
1056
|
-
const nodeVersionDefault = dockerfileContent.match(/^ARG NODE_VERSION=(\d+)/m);
|
|
1057
|
-
if (nodeVersionDefault) {
|
|
1058
|
-
return {
|
|
1059
|
-
name: `node`,
|
|
1060
|
-
tag: `${nodeVersionDefault[1]}-alpine`,
|
|
1061
|
-
registry: 'docker.io',
|
|
1062
|
-
repo: 'library'
|
|
1063
|
-
};
|
|
1064
|
-
}
|
|
1065
|
-
else {
|
|
1066
|
-
throw new TSError('Failed to parse Dockerfile for base image.');
|
|
1067
|
-
}
|
|
1068
|
-
}
|
|
1069
|
-
catch (err) {
|
|
1070
|
-
throw new TSError(`Failed to read top-level Dockerfile to get base image.\n${err}`);
|
|
1071
|
-
}
|
|
1072
|
-
}
|
|
1073
|
-
/**
|
|
1074
|
-
* Retrieves the Node.js version from the base Docker image specified in the Teraslice `Dockerfile`.
|
|
1075
|
-
*
|
|
1076
|
-
* This function:
|
|
1077
|
-
* - Extracts the base image details from the `Dockerfile`
|
|
1078
|
-
* - Authenticates with the container registry to retrieve a token
|
|
1079
|
-
* - Fetches the image manifest and configuration
|
|
1080
|
-
* - Extracts the `node_version` label from the image config
|
|
1081
|
-
*
|
|
1082
|
-
* @throws {TSError} If any request to the registry fails or expected data is missing.
|
|
1083
|
-
* @returns {Promise<string>} Resolves with the Node.js version string.
|
|
1084
|
-
*/
|
|
1085
|
-
export async function grabCurrentTSNodeVersion() {
|
|
1086
|
-
// Extract base image details from the Dockerfile
|
|
1087
|
-
const baseImage = getDockerBaseImageInfo();
|
|
1088
|
-
let token;
|
|
1089
|
-
// Request authentication token for accessing image manifests
|
|
1090
|
-
try {
|
|
1091
|
-
const authUrl = 'https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/node:pull';
|
|
1092
|
-
const authResponse = await got(authUrl);
|
|
1093
|
-
token = JSON.parse(authResponse.body).token;
|
|
1094
|
-
}
|
|
1095
|
-
catch (err) {
|
|
1096
|
-
throw new TSError(`Unable to retrieve token from ${baseImage.registry} for repo ${baseImage.repo}:\n${err}`);
|
|
1097
|
-
}
|
|
1098
|
-
// Grab the manifest list to find the right architecture digest
|
|
1099
|
-
let manifestDigest;
|
|
1100
|
-
try {
|
|
1101
|
-
const manifestUrl = `https://registry-1.docker.io/v2/library/node/manifests/${baseImage.tag}`;
|
|
1102
|
-
const response = await got(manifestUrl, {
|
|
1103
|
-
headers: {
|
|
1104
|
-
Authorization: `Bearer ${token}`,
|
|
1105
|
-
},
|
|
1106
|
-
responseType: 'json'
|
|
1107
|
-
});
|
|
1108
|
-
const body = response.body;
|
|
1109
|
-
const amd64Manifest = body.manifests.find((manifest) => manifest.platform.architecture === 'amd64');
|
|
1110
|
-
if (!amd64Manifest) {
|
|
1111
|
-
throw new TSError(`No amd64 manifest found for ${baseImage.repo}:${baseImage.tag}`);
|
|
1112
|
-
}
|
|
1113
|
-
manifestDigest = amd64Manifest.digest;
|
|
1114
|
-
}
|
|
1115
|
-
catch (err) {
|
|
1116
|
-
throw new TSError(`Unable to retrieve image manifest list from ${baseImage.registry} for ${baseImage.repo}:${baseImage.tag}:\n${err}`);
|
|
1117
|
-
}
|
|
1118
|
-
let configManifest;
|
|
1119
|
-
try {
|
|
1120
|
-
const manifestUrl = `https://registry-1.docker.io/v2/library/node/manifests/${manifestDigest}`;
|
|
1121
|
-
const response = await got(manifestUrl, {
|
|
1122
|
-
headers: {
|
|
1123
|
-
Authorization: `Bearer ${token}`,
|
|
1124
|
-
},
|
|
1125
|
-
responseType: 'json'
|
|
1126
|
-
});
|
|
1127
|
-
const manifestList = response.body;
|
|
1128
|
-
configManifest = manifestList.config.digest;
|
|
1129
|
-
if (!configManifest) {
|
|
1130
|
-
throw new TSError(`No config digest found for ${baseImage.repo}:${baseImage.tag}`);
|
|
1131
|
-
}
|
|
1132
|
-
}
|
|
1133
|
-
catch (err) {
|
|
1134
|
-
throw new TSError(`Unable to retrieve image manifest list from ${baseImage.registry} for ${baseImage.repo}:${baseImage.tag}:\n${err}`);
|
|
1135
|
-
}
|
|
1136
|
-
// Retrieve the image configuration and extract the Node.js version label
|
|
1137
|
-
try {
|
|
1138
|
-
const configUrl = `https://registry-1.docker.io/v2/library/node/blobs/${configManifest}`;
|
|
1139
|
-
const response = await got(configUrl, {
|
|
1140
|
-
headers: {
|
|
1141
|
-
Authorization: `Bearer ${token}`,
|
|
1142
|
-
},
|
|
1143
|
-
responseType: 'json'
|
|
1144
|
-
});
|
|
1145
|
-
const imageConfig = response.body;
|
|
1146
|
-
const nodeVersion = imageConfig.config?.Env?.find((envVar) => envVar.startsWith('NODE_VERSION='))?.split('=')[1];
|
|
1147
|
-
if (!nodeVersion) {
|
|
1148
|
-
throw new TSError(`Node version label missing in config for ${baseImage.repo}:${baseImage.tag}`);
|
|
1149
|
-
}
|
|
1150
|
-
return nodeVersion;
|
|
1151
|
-
}
|
|
1152
|
-
catch (err) {
|
|
1153
|
-
throw new TSError(`Unable to grab image config from ${baseImage.registry} for ${baseImage.repo}:${baseImage.tag}:\n${err}`);
|
|
1154
|
-
}
|
|
1155
|
-
}
|
|
1156
|
-
/**
|
|
1157
|
-
* Updates the Teraslice Helm chart YAML files (`Chart.yaml` and `values.yaml`)
|
|
1158
|
-
* with the new chart version
|
|
1159
|
-
*
|
|
1160
|
-
* @param {string | null} newChartVersion - The new version to set in `Chart.yaml`.
|
|
1161
|
-
* - If `null`, the function does not update the chart version.
|
|
1162
|
-
* @throws {TSError} If the function fails to read or write YAML files.
|
|
1163
|
-
* @returns {Promise<void>} Resolves when the Helm chart files have been successfully updated
|
|
1164
|
-
*/
|
|
1165
|
-
export async function updateHelmChart(newChartVersion) {
|
|
1166
|
-
const currentNodeVersion = await grabCurrentTSNodeVersion();
|
|
1167
|
-
const rootDir = getRootDir();
|
|
1168
|
-
const chartYamlPath = path.join(rootDir, 'helm/teraslice/Chart.yaml');
|
|
1169
|
-
const valuesYamlPath = path.join(rootDir, 'helm/teraslice/values.yaml');
|
|
1170
|
-
try {
|
|
1171
|
-
// Read YAML files and parse them into objects
|
|
1172
|
-
const chartFileContent = fs.readFileSync(chartYamlPath, 'utf8');
|
|
1173
|
-
const valuesFileContent = fs.readFileSync(valuesYamlPath, 'utf8');
|
|
1174
|
-
const chartDoc = parseDocument(chartFileContent);
|
|
1175
|
-
const valuesDoc = parseDocument(valuesFileContent);
|
|
1176
|
-
// Update specific values for the chart
|
|
1177
|
-
if (newChartVersion) {
|
|
1178
|
-
chartDoc.set('version', newChartVersion);
|
|
1179
|
-
}
|
|
1180
|
-
chartDoc.set('appVersion', `v${getTerasliceVersion()}`);
|
|
1181
|
-
valuesDoc.setIn(['image', 'nodeVersion'], `v${currentNodeVersion}`);
|
|
1182
|
-
// Write the updated YAML back to the files
|
|
1183
|
-
fs.writeFileSync(chartYamlPath, chartDoc.toString(), 'utf8');
|
|
1184
|
-
fs.writeFileSync(valuesYamlPath, valuesDoc.toString(), 'utf8');
|
|
1185
|
-
}
|
|
1186
|
-
catch (err) {
|
|
1187
|
-
throw new TSError(`Unable to read or write Helm chart YAML files:\n${err}`);
|
|
1188
|
-
}
|
|
1189
|
-
}
|
|
1190
|
-
/**
|
|
1191
|
-
* Writes the Opensearch internal_users.yml file with demo user credentials
|
|
1192
|
-
* to the specified certificate directory. This file is required by Opensearch
|
|
1193
|
-
* security configuration to define local user accounts.
|
|
1194
|
-
*
|
|
1195
|
-
* @param certDir - Absolute path to the directory where the file will be written
|
|
1196
|
-
*/
|
|
1197
|
-
function createInternalUsersFile(certDir) {
|
|
1198
|
-
const content = `# This is the internal user database
|
|
1199
|
-
# The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh
|
|
1200
|
-
|
|
1201
|
-
_meta:
|
|
1202
|
-
type: "internalusers"
|
|
1203
|
-
config_version: 2
|
|
1204
|
-
|
|
1205
|
-
# Define your internal users here
|
|
1206
|
-
|
|
1207
|
-
## Demo users
|
|
1208
|
-
# This hash password is passwordsufhbivbU123%$
|
|
1209
|
-
admin:
|
|
1210
|
-
hash: "$2y$12$Z234bambHnVJMAXiccuMluNgGhNNdOFIY6pFT2/lk3ZC.RDoDIFme"
|
|
1211
|
-
reserved: true
|
|
1212
|
-
backend_roles:
|
|
1213
|
-
- "admin"
|
|
1214
|
-
description: "Demo admin user"
|
|
1215
|
-
anomalyadmin:
|
|
1216
|
-
hash: "$2y$12$TRwAAJgnNo67w3rVUz4FIeLx9Dy/llB79zf9I15CKJ9vkM4ZzAd3."
|
|
1217
|
-
reserved: false
|
|
1218
|
-
opendistro_security_roles:
|
|
1219
|
-
- "anomaly_full_access"
|
|
1220
|
-
description: "Demo anomaly admin user, using internal role"
|
|
1221
|
-
kibanaserver:
|
|
1222
|
-
hash: "$2a$12$4AcgAt3xwOWadA5s5blL6ev39OXDNhmOesEoo33eZtrq2N0YrU3H."
|
|
1223
|
-
reserved: true
|
|
1224
|
-
description: "Demo OpenSearch Dashboards user"
|
|
1225
|
-
kibanaro:
|
|
1226
|
-
hash: "$2a$12$JJSXNfTowz7Uu5ttXfeYpeYE0arACvcwlPBStB1F.MI7f0U9Z4DGC"
|
|
1227
|
-
reserved: false
|
|
1228
|
-
backend_roles:
|
|
1229
|
-
- "kibanauser"
|
|
1230
|
-
- "readall"
|
|
1231
|
-
attributes:
|
|
1232
|
-
attribute1: "value1"
|
|
1233
|
-
attribute2: "value2"
|
|
1234
|
-
attribute3: "value3"
|
|
1235
|
-
description: "Demo OpenSearch Dashboards read only user, using external role mapping"
|
|
1236
|
-
logstash:
|
|
1237
|
-
hash: "$2a$12$u1ShR4l4uBS3Uv59Pa2y5.1uQuZBrZtmNfqB3iM/.jL0XoV9sghS2"
|
|
1238
|
-
reserved: false
|
|
1239
|
-
backend_roles:
|
|
1240
|
-
- "logstash"
|
|
1241
|
-
description: "Demo logstash user, using external role mapping"
|
|
1242
|
-
readall:
|
|
1243
|
-
hash: "$2a$12$ae4ycwzwvLtZxwZ82RmiEunBbIPiAmGZduBAjKN0TXdwQFtCwARz2"
|
|
1244
|
-
reserved: false
|
|
1245
|
-
backend_roles:
|
|
1246
|
-
- "readall"
|
|
1247
|
-
description: "Demo readall user, using external role mapping"
|
|
1248
|
-
snapshotrestore:
|
|
1249
|
-
hash: "$2y$12$DpwmetHKwgYnorbgdvORCenv4NAK8cPUg8AI6pxLCuWf/ALc0.v7W"
|
|
1250
|
-
reserved: false
|
|
1251
|
-
backend_roles:
|
|
1252
|
-
- "snapshotrestore"
|
|
1253
|
-
description: "Demo snapshotrestore user, using external role mapping"
|
|
1254
|
-
`;
|
|
1255
|
-
fs.writeFileSync(path.join(certDir, 'internal_users.yml'), content, 'utf8');
|
|
1256
|
-
}
|
|
1257
|
-
/**
|
|
1258
|
-
* Generates TLS certs using mkcert and organizes the output files
|
|
1259
|
-
* into the layout expected by each service format.
|
|
1260
|
-
*
|
|
1261
|
-
* mkcert creates a locally-trusted CA and issues certificates signed by it.
|
|
1262
|
-
* This function handles the full lifecycle:
|
|
1263
|
-
* 1. Runs mkcert to produce a key + certificate for the given DNS names
|
|
1264
|
-
* 2. Copies the root CA certificate into a CAs/ subdirectory
|
|
1265
|
-
* 3. Renames/copies files into the structure each service expects
|
|
1266
|
-
* 4. Cleans up the raw mkcert output files
|
|
1267
|
-
*
|
|
1268
|
-
* Supported formats:
|
|
1269
|
-
* - `minio` → private.key, public.crt
|
|
1270
|
-
* - `opensearch` → opensearch-key.pem, opensearch-cert.pem, internal_users.yml
|
|
1271
|
-
* - `kafka` → kafka-keypair.pem (key + cert concatenated)
|
|
1272
|
-
*
|
|
1273
|
-
* @param formats - List of service formats to produce (e.g. ['minio', 'opensearch'])
|
|
1274
|
-
* @param dirPath - Absolute path to the output directory (recreated if it already exists)
|
|
1275
|
-
* @param dnsNames - DNS names / IPs the certificate should be valid for
|
|
1276
|
-
* @throws {Error} If mkcert is not installed, dirPath is relative, or cert files cannot be found
|
|
1277
|
-
*/
|
|
1278
|
-
async function generateCerts(formats, dirPath, dnsNames) {
|
|
1279
|
-
// mkcert must be installed
|
|
1280
|
-
try {
|
|
1281
|
-
await execa('mkcert', ['--version']);
|
|
1282
|
-
}
|
|
1283
|
-
catch {
|
|
1284
|
-
throw new Error('mkcert is not installed. Please install mkcert and try again.');
|
|
1285
|
-
}
|
|
1286
|
-
if (!path.isAbsolute(dirPath)) {
|
|
1287
|
-
throw new Error('dirPath must be an absolute path');
|
|
1288
|
-
}
|
|
1289
|
-
if (dnsNames.length === 0) {
|
|
1290
|
-
throw new Error('At least one DNS name is required');
|
|
1291
|
-
}
|
|
1292
|
-
// Always start with a clean dir in the case there is stuff here
|
|
1293
|
-
if (fse.existsSync(dirPath)) {
|
|
1294
|
-
await fse.remove(dirPath);
|
|
1295
|
-
}
|
|
1296
|
-
await fse.mkdirp(dirPath);
|
|
1297
|
-
// Run mkcert in the output directory — it writes the key and cert files there
|
|
1298
|
-
await execa('mkcert', ['--client', ...dnsNames], { cwd: dirPath });
|
|
1299
|
-
// mkcert stores its root CA in a system dir. copy it into our cert dir
|
|
1300
|
-
const { stdout: caRoot } = await execa('mkcert', ['-CAROOT']);
|
|
1301
|
-
fse.copySync(path.join(caRoot.trim(), 'rootCA.pem'), path.join(dirPath, 'rootCA.pem'));
|
|
1302
|
-
// Move the root CA into a CAs/ subdirectory. Keeps it organized
|
|
1303
|
-
await fse.mkdirp(path.join(dirPath, 'CAs'));
|
|
1304
|
-
fse.moveSync(path.join(dirPath, 'rootCA.pem'), path.join(dirPath, 'CAs', 'rootCA.pem'));
|
|
1305
|
-
// mkcert names its output files based on the DNS names, so we locate them by pattern
|
|
1306
|
-
const files = fs.readdirSync(dirPath);
|
|
1307
|
-
const privateKeyName = files.find((f) => f.toLowerCase().includes('key.pem'));
|
|
1308
|
-
const publicCertName = files.find((f) => f.toLowerCase().endsWith('.pem') && !f.toLowerCase().includes('key.pem'));
|
|
1309
|
-
if (!privateKeyName || !publicCertName) {
|
|
1310
|
-
throw new Error(`Could not locate key.pem or public cert in ${dirPath}`);
|
|
1311
|
-
}
|
|
1312
|
-
const privateKeyPath = path.join(dirPath, privateKeyName);
|
|
1313
|
-
const publicCertPath = path.join(dirPath, publicCertName);
|
|
1314
|
-
// Copy/rename files into the layout each service expects. Each service like it in
|
|
1315
|
-
// a specific format.
|
|
1316
|
-
for (const format of formats) {
|
|
1317
|
-
switch (format) {
|
|
1318
|
-
case 'minio':
|
|
1319
|
-
// https://min.io/docs/minio/linux/operations/network-encryption.html
|
|
1320
|
-
fse.copySync(privateKeyPath, path.join(dirPath, 'private.key'));
|
|
1321
|
-
fse.copySync(publicCertPath, path.join(dirPath, 'public.crt'));
|
|
1322
|
-
break;
|
|
1323
|
-
case 'opensearch':
|
|
1324
|
-
// https://opensearch.org/docs/latest/security/configuration/tls/#x509-pem-certificates-and-pkcs-8-keys
|
|
1325
|
-
fse.copySync(privateKeyPath, path.join(dirPath, 'opensearch-key.pem'));
|
|
1326
|
-
fse.copySync(publicCertPath, path.join(dirPath, 'opensearch-cert.pem'));
|
|
1327
|
-
// Opensearch also needs a user database file alongside the certs
|
|
1328
|
-
createInternalUsersFile(dirPath);
|
|
1329
|
-
break;
|
|
1330
|
-
case 'kafka': {
|
|
1331
|
-
// https://kafka.apache.org/42/security/encryption-and-authentication-using-ssl/
|
|
1332
|
-
// Kafka expects key and cert in a single PEM file
|
|
1333
|
-
const keyContent = fs.readFileSync(privateKeyPath, 'utf8');
|
|
1334
|
-
const certContent = fs.readFileSync(publicCertPath, 'utf8');
|
|
1335
|
-
fs.writeFileSync(path.join(dirPath, 'kafka-keypair.pem'), keyContent + certContent, 'utf8');
|
|
1336
|
-
break;
|
|
1337
|
-
}
|
|
1338
|
-
default:
|
|
1339
|
-
signale.warn(`Unknown format '${format}' ignored.`);
|
|
1340
|
-
}
|
|
1341
|
-
}
|
|
1342
|
-
// Remove the original mkcert files now that we've copied and renamed them
|
|
1343
|
-
if (formats.length > 0) {
|
|
1344
|
-
fs.unlinkSync(privateKeyPath);
|
|
1345
|
-
fs.unlinkSync(publicCertPath);
|
|
1346
|
-
}
|
|
1347
|
-
// Make sure all cert files are readable
|
|
1348
|
-
await execa('chmod', ['-R', 'a+rX', dirPath]);
|
|
1349
|
-
}
|
|
1350
|
-
/**
|
|
1351
|
-
* Generates CA certificates for encrypted services in the test environment if needed
|
|
1352
|
-
*
|
|
1353
|
-
* @throws {Error} If certificate generation fails.
|
|
1354
|
-
*/
|
|
1355
|
-
export async function generateTestCaCerts() {
|
|
1356
|
-
const encryptedServices = [];
|
|
1357
|
-
const hostNames = ['localhost'];
|
|
1358
|
-
if (config.ENCRYPT_OPENSEARCH) {
|
|
1359
|
-
encryptedServices.push('opensearch');
|
|
1360
|
-
hostNames.push('opensearch2.services-dev1', 'opensearch3.services-dev1', 'opensearch', config.OPENSEARCH_HOSTNAME);
|
|
1361
|
-
}
|
|
1362
|
-
if (config.ENCRYPT_MINIO) {
|
|
1363
|
-
encryptedServices.push('minio');
|
|
1364
|
-
hostNames.push('minio.services-dev1', 'minio', config.MINIO_HOSTNAME);
|
|
1365
|
-
}
|
|
1366
|
-
if (config.ENCRYPT_KAFKA) {
|
|
1367
|
-
encryptedServices.push('kafka');
|
|
1368
|
-
hostNames.push('kafka-headless.services-dev1.svc.cluster.local', 'kafka-headless.services-dev1', 'kafka-headless', 'kafka', config.KAFKA_HOSTNAME);
|
|
1369
|
-
}
|
|
1370
|
-
if (encryptedServices.length > 0) {
|
|
1371
|
-
// Formats the encrypted service list to print with the user feedback
|
|
1372
|
-
const serviceList = encryptedServices.length === 1
|
|
1373
|
-
? encryptedServices[0]
|
|
1374
|
-
: encryptedServices.length === 2
|
|
1375
|
-
? encryptedServices.join(' and ')
|
|
1376
|
-
: `${encryptedServices.slice(0, -1).join(', ')} and ${encryptedServices[encryptedServices.length - 1]}`;
|
|
1377
|
-
try {
|
|
1378
|
-
signale.pending(`Generating new ca-certificates for ${serviceList}...`);
|
|
1379
|
-
signale.debug('Generating certs: ', { formats: encryptedServices, dirPath: config.CERT_PATH, dnsNames: hostNames });
|
|
1380
|
-
await generateCerts(encryptedServices, config.CERT_PATH, hostNames);
|
|
1381
|
-
signale.success(`Created certs in ${config.CERT_PATH}`);
|
|
1382
|
-
}
|
|
1383
|
-
catch (err) {
|
|
1384
|
-
throw new Error(`Error generating ca-certificates for ${serviceList}: ${err.message}`);
|
|
1385
|
-
}
|
|
1386
|
-
}
|
|
1387
|
-
}
|
|
1388
|
-
export async function getConfigValueFromCustomYaml(configFilePath, valuePath) {
|
|
1389
|
-
const customConfig = load(fs.readFileSync(configFilePath, 'utf8'));
|
|
1390
|
-
const value = get(customConfig, valuePath, undefined);
|
|
1391
|
-
return value;
|
|
1392
|
-
}
|
|
1393
|
-
export async function setConfigValuesForCustomYaml(configFilePath, valuePath, valueToSet) {
|
|
1394
|
-
try {
|
|
1395
|
-
const customConfig = parseDocument(fs.readFileSync(configFilePath, 'utf8'));
|
|
1396
|
-
const splitPath = valuePath.split('.');
|
|
1397
|
-
customConfig.setIn(splitPath, valueToSet);
|
|
1398
|
-
fs.writeFileSync(configFilePath, customConfig.toString(), 'utf8');
|
|
1399
|
-
}
|
|
1400
|
-
catch (err) {
|
|
1401
|
-
throw new Error(`Failed to set ${valuePath} to ${valueToSet} in config file ${configFilePath}. Reason: ${err.message}`);
|
|
1402
|
-
}
|
|
1403
|
-
}
|
|
1404
|
-
//# sourceMappingURL=scripts.js.map
|