@terascope/scripts 2.19.2 → 2.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/src/helpers/bump/index.js +1 -1
  2. package/dist/src/helpers/bump/index.js.map +1 -1
  3. package/dist/src/helpers/certs.d.ts +28 -0
  4. package/dist/src/helpers/certs.d.ts.map +1 -0
  5. package/dist/src/helpers/certs.js +267 -0
  6. package/dist/src/helpers/certs.js.map +1 -0
  7. package/dist/src/helpers/doc-builder/index.js +1 -1
  8. package/dist/src/helpers/doc-builder/index.js.map +1 -1
  9. package/dist/src/helpers/docker.d.ts +78 -0
  10. package/dist/src/helpers/docker.d.ts.map +1 -0
  11. package/dist/src/helpers/docker.js +395 -0
  12. package/dist/src/helpers/docker.js.map +1 -0
  13. package/dist/src/helpers/exec.d.ts +19 -0
  14. package/dist/src/helpers/exec.d.ts.map +1 -0
  15. package/dist/src/helpers/exec.js +81 -0
  16. package/dist/src/helpers/exec.js.map +1 -0
  17. package/dist/src/helpers/git.d.ts +4 -0
  18. package/dist/src/helpers/git.d.ts.map +1 -0
  19. package/dist/src/helpers/git.js +47 -0
  20. package/dist/src/helpers/git.js.map +1 -0
  21. package/dist/src/helpers/helm.d.ts +25 -0
  22. package/dist/src/helpers/helm.d.ts.map +1 -0
  23. package/dist/src/helpers/helm.js +297 -0
  24. package/dist/src/helpers/helm.js.map +1 -0
  25. package/dist/src/helpers/images/index.js +1 -1
  26. package/dist/src/helpers/images/index.js.map +1 -1
  27. package/dist/src/helpers/k8s-env/index.d.ts.map +1 -1
  28. package/dist/src/helpers/k8s-env/index.js +5 -2
  29. package/dist/src/helpers/k8s-env/index.js.map +1 -1
  30. package/dist/src/helpers/kubernetes.d.ts +10 -0
  31. package/dist/src/helpers/kubernetes.d.ts.map +1 -0
  32. package/dist/src/helpers/kubernetes.js +160 -0
  33. package/dist/src/helpers/kubernetes.js.map +1 -0
  34. package/dist/src/helpers/misc.d.ts +9 -5
  35. package/dist/src/helpers/misc.d.ts.map +1 -1
  36. package/dist/src/helpers/misc.js +66 -9
  37. package/dist/src/helpers/misc.js.map +1 -1
  38. package/dist/src/helpers/package-manager.d.ts +18 -0
  39. package/dist/src/helpers/package-manager.d.ts.map +1 -0
  40. package/dist/src/helpers/package-manager.js +152 -0
  41. package/dist/src/helpers/package-manager.js.map +1 -0
  42. package/dist/src/helpers/packages.js +1 -1
  43. package/dist/src/helpers/packages.js.map +1 -1
  44. package/dist/src/helpers/publish/index.d.ts.map +1 -1
  45. package/dist/src/helpers/publish/index.js +2 -1
  46. package/dist/src/helpers/publish/index.js.map +1 -1
  47. package/dist/src/helpers/publish/utils.d.ts.map +1 -1
  48. package/dist/src/helpers/publish/utils.js +2 -1
  49. package/dist/src/helpers/publish/utils.js.map +1 -1
  50. package/dist/src/helpers/sync/utils.js +1 -1
  51. package/dist/src/helpers/sync/utils.js.map +1 -1
  52. package/dist/src/helpers/test-runner/index.d.ts.map +1 -1
  53. package/dist/src/helpers/test-runner/index.js +5 -1
  54. package/dist/src/helpers/test-runner/index.js.map +1 -1
  55. package/dist/src/helpers/test-runner/services.d.ts.map +1 -1
  56. package/dist/src/helpers/test-runner/services.js +11 -5
  57. package/dist/src/helpers/test-runner/services.js.map +1 -1
  58. package/dist/src/helpers/test-runner/utils.d.ts +1 -1
  59. package/dist/src/helpers/test-runner/utils.d.ts.map +1 -1
  60. package/dist/src/helpers/test-runner/utils.js +3 -2
  61. package/dist/src/helpers/test-runner/utils.js.map +1 -1
  62. package/dist/src/index.d.ts +7 -1
  63. package/dist/src/index.d.ts.map +1 -1
  64. package/dist/src/index.js +7 -1
  65. package/dist/src/index.js.map +1 -1
  66. package/package.json +6 -6
  67. package/dist/src/helpers/scripts.d.ts +0 -174
  68. package/dist/src/helpers/scripts.d.ts.map +0 -1
  69. package/dist/src/helpers/scripts.js +0 -1404
  70. package/dist/src/helpers/scripts.js.map +0 -1
@@ -1,1404 +0,0 @@
1
- import fs from 'node:fs';
2
- import os from 'node:os';
3
- import ms from 'ms';
4
- import path from 'node:path';
5
- import { X509Certificate } from 'node:crypto';
6
- import { execa, execaCommand } from 'execa';
7
- import fse from 'fs-extra';
8
- import { load } from 'js-yaml';
9
- import got from 'got';
10
- import { parseDocument } from 'yaml';
11
- import { debugLogger, isString, get, pWhile, pDelay, TSError, isCI, } from '@terascope/core-utils';
12
- import { Service } from '@terascope/types';
13
- import { getRootDir, getRootInfo, getPackageManager } from './misc.js';
14
- import signale from './signale.js';
15
- import config from './config.js';
16
- import { getVolumesFromDockerfile } from './kind.js';
17
- import { TestFrameworks } from './test-runner/interfaces.js';
18
- const logger = debugLogger('ts-scripts:cmd');
19
- export const TEST_CONFIGS = 'test-configs';
20
- function _exec(opts) {
21
- let subprocess;
22
- const options = {
23
- cwd: opts.cwd || getRootDir(),
24
- env: opts.env,
25
- preferLocal: true,
26
- detached: opts.detached,
27
- timeout: opts.timeout,
28
- stdio: opts.stdio,
29
- };
30
- logger.debug('executing command', opts);
31
- if (opts.args && opts.args.length) {
32
- subprocess = execa(opts.cmd, opts.args, options);
33
- }
34
- else {
35
- subprocess = execa(opts.cmd, options);
36
- }
37
- if (!subprocess) {
38
- throw new Error(`Failed to execution ${opts.cmd}`);
39
- }
40
- if (!opts.stdio) {
41
- if (!subprocess.stderr || !subprocess.stdout) {
42
- throw new Error(`Command ${opts.cmd} failed, stderr or stdout is not available`);
43
- }
44
- subprocess.stderr.pipe(process.stderr);
45
- }
46
- return subprocess;
47
- }
48
- export async function exec(opts, log = true) {
49
- try {
50
- const env = { FORCE_COLOR: '0', ...opts.env };
51
- const _opts = { ...opts };
52
- _opts.env = env;
53
- const subprocess = _exec(_opts);
54
- const { stdout } = await subprocess;
55
- if (typeof stdout !== 'string') {
56
- throw new Error('exec() requires ExecOpts that result in a stdout string. See the execa docs for details.');
57
- }
58
- const result = stdout.trim();
59
- logger.debug(`exec result: ${opts.cmd} ${(opts.args || []).join(' ')}`, log && result);
60
- return result;
61
- }
62
- catch (err) {
63
- if (!err.command) {
64
- throw err;
65
- }
66
- process.exitCode = err.exitCode || 1;
67
- throw new Error(err.message);
68
- }
69
- }
70
- export async function fork(opts) {
71
- try {
72
- const env = {
73
- FORCE_COLOR: config.FORCE_COLOR,
74
- ...opts.env
75
- };
76
- const _opts = { stdio: 'inherit', ...opts };
77
- _opts.env = env;
78
- await _exec(_opts);
79
- }
80
- catch (err) {
81
- if (!err.command) {
82
- throw err;
83
- }
84
- process.exitCode = err.exitCode || 1;
85
- throw new Error(err.message);
86
- }
87
- }
88
- export async function runTSScript(cmd, args) {
89
- const scriptName = process.argv[1];
90
- return fork({
91
- cmd: scriptName,
92
- args: [cmd, ...args],
93
- });
94
- }
95
- export async function build(pkgInfo) {
96
- if (pkgInfo) {
97
- const distDir = path.join(pkgInfo.dir, 'dist');
98
- if (fse.existsSync(distDir)) {
99
- await fse.emptyDir(distDir);
100
- }
101
- await packageMngrRun('build', [], pkgInfo.dir);
102
- return;
103
- }
104
- await packageMngrRun('build');
105
- }
106
- export async function setup() {
107
- await packageMngrRun('setup');
108
- }
109
- export async function packageMngrRun(script, args, cwd, env, log) {
110
- const dir = cwd || getRootDir();
111
- const pkgJSON = await fse.readJSON(path.join(dir, 'package.json'));
112
- const hasScript = Boolean(get(pkgJSON, ['scripts', script]));
113
- if (!hasScript)
114
- return;
115
- const pm = getPackageManager();
116
- const _args = ['run', script, ...(args ?? [])];
117
- if (log) {
118
- signale.info(`running ${pm} ${_args.join(' ')}...`);
119
- }
120
- await fork({
121
- cmd: pm, args: _args, cwd: dir, env
122
- });
123
- }
124
- export async function runTestFramework(cwd, argsMap, env, extraArgs, debug, attachJestDebugger, framework = TestFrameworks.jest, frameworkConfig) {
125
- const pm = getPackageManager();
126
- // When running jest in yarn3 PnP with ESM we must call 'yarn jest <...args>'
127
- // to prevent module not found errors. Therefore we will call fork with the yarn/pnpm
128
- // command and set jest to the first argument.
129
- const frameworkArgs = {
130
- jest: ['jest'],
131
- playwright: ['playwright', 'test']
132
- };
133
- let args = frameworkArgs[framework];
134
- // Set with ATTACH_JEST_DEBUGGER env variable
135
- // Does not work with repos with pnp
136
- if (attachJestDebugger && framework === TestFrameworks.jest) {
137
- const nodeLinkerConfig = await getNodeLinkerConfig();
138
- if (nodeLinkerConfig === 'node-modules') {
139
- // Grab jest bin file
140
- const jestBinCall = await execa(pm, ['bin', 'jest'], {
141
- cwd: getRootDir()
142
- });
143
- if (jestBinCall.stderr.length) {
144
- throw new Error(`Unable to find jest bin directory when calling "${pm} bin jest": ${jestBinCall.stderr}`);
145
- }
146
- const jestBinDir = jestBinCall.stdout;
147
- args = [
148
- 'node',
149
- '--inspect-brk=9230',
150
- '--experimental-vm-modules',
151
- jestBinDir
152
- ];
153
- }
154
- else {
155
- signale.warn(`Projects with ${nodeLinkerConfig} are not compatible with `
156
- + `ATTACH_JEST_DEBUGGER env config and cannot be used. `
157
- + 'Only node-modules configuration is valid');
158
- }
159
- }
160
- args.push(...mapToArgs(argsMap));
161
- if (extraArgs) {
162
- extraArgs.forEach((extraArg) => {
163
- if (extraArg.startsWith('-') && args.includes(extraArg)) {
164
- if (debug) {
165
- logger.debug(`* skipping duplicate jest arg ${extraArg}`);
166
- }
167
- return;
168
- }
169
- args.push(extraArg);
170
- });
171
- }
172
- if (frameworkConfig) {
173
- args.push(`--config`);
174
- args.push(frameworkConfig);
175
- }
176
- let logArgs = args;
177
- if (framework === 'jest') {
178
- logArgs = [...args];
179
- const configIdx = args.findIndex((el) => el === '--config');
180
- if (configIdx > -1 && args[configIdx + 1]?.startsWith('{')) {
181
- logArgs[configIdx + 1] = '<STRINGIFIED_CONFIG>';
182
- }
183
- }
184
- signale.info(`executing ${framework}: ${logArgs.join(' ')}`);
185
- await fork({
186
- cmd: pm,
187
- cwd,
188
- args,
189
- env,
190
- });
191
- }
192
- async function getNodeLinkerConfig() {
193
- const pm = getPackageManager();
194
- // pnpm always uses node-modules (with symlinks)
195
- if (pm === 'pnpm') {
196
- return 'node-modules';
197
- }
198
- try {
199
- const { stdout: nodeLinkerconfig, stderr } = await execa('yarn', ['config', 'get', 'nodeLinker'], {
200
- cwd: getRootDir()
201
- });
202
- // If info is printed in stderr, there must have been an issue
203
- if (stderr.length) {
204
- throw new Error(stderr);
205
- }
206
- return nodeLinkerconfig;
207
- }
208
- catch (err) {
209
- throw new Error(`Error trying to grab yarn nodeLinker config from the project: ${err.message}`);
210
- }
211
- }
212
- export async function dockerPull(image, timeout = 0) {
213
- try {
214
- await exec({
215
- cmd: 'docker',
216
- args: ['pull', image],
217
- timeout,
218
- });
219
- }
220
- catch (err) {
221
- process.exitCode = 0;
222
- throw err;
223
- }
224
- }
225
- export async function dockerStop(name) {
226
- await exec({
227
- cmd: 'docker',
228
- args: ['stop', name],
229
- });
230
- }
231
- export async function dockerTag(from, to) {
232
- signale.pending(`Tagging image ${from} as ${to}`);
233
- await exec({
234
- cmd: 'docker',
235
- args: ['tag', from, to],
236
- });
237
- signale.success(`Image ${from} re-tagged as ${to}`);
238
- }
239
- export async function getNodeVersionFromImage(image) {
240
- try {
241
- const { stdout } = await execa('docker', ['run', image, 'node', '-v']);
242
- return stdout;
243
- }
244
- catch (err) {
245
- throw new Error(`Unable to get node version from image due to Error: ${err}`);
246
- }
247
- }
248
- export async function getContainerInfo(name) {
249
- const result = await exec({
250
- cmd: 'docker',
251
- args: ['ps', '--format={{json .}}', `--filter=name=${name}`],
252
- });
253
- if (!result)
254
- return null;
255
- return JSON.parse(result);
256
- }
257
- export async function dockerNetworkExists(name) {
258
- const subprocess = await execaCommand(`docker network ls --format='{{json .Name}}' | grep '"${name}"'`, { reject: false, shell: true });
259
- return subprocess.exitCode ? subprocess.exitCode > 0 : false;
260
- }
261
- export async function remoteDockerImageExists(image) {
262
- try {
263
- await dockerPull(image, ms('30s'));
264
- return true;
265
- }
266
- catch (err) {
267
- return false;
268
- }
269
- }
270
- export async function dockerRun(opt, tag, ignoreMount, debug) {
271
- const args = ['run', '--rm'];
272
- if (!opt.image) {
273
- throw new Error('Missing required image option');
274
- }
275
- if (!opt.name) {
276
- throw new Error('Missing required name option');
277
- }
278
- if (opt.mount && !ignoreMount) {
279
- for (const mount of opt.mount) {
280
- args.push('--mount', mount);
281
- }
282
- }
283
- if (opt.ports && opt.ports.length) {
284
- opt.ports.forEach((port) => {
285
- if (isString(port)) {
286
- args.push('--publish', port);
287
- }
288
- else {
289
- args.push('--publish', `${port}:${port}`);
290
- }
291
- });
292
- }
293
- if (opt.env) {
294
- Object.entries(opt.env).forEach(([key, val]) => {
295
- args.push('--env', `${key}=${val}`);
296
- });
297
- }
298
- if (opt.tmpfs && opt.tmpfs.length) {
299
- args.push('--tmpfs', opt.tmpfs.join(','));
300
- }
301
- if (opt.network) {
302
- const exists = await dockerNetworkExists(opt.network);
303
- if (!exists) {
304
- throw new Error(`Docker network ${opt.network} does not exist`);
305
- }
306
- args.push('--network', opt.network);
307
- }
308
- args.push('--name', opt.name);
309
- args.push(`${opt.image}:${tag ?? 'latest'}`);
310
- if (opt.args) {
311
- args.push(...opt.args);
312
- }
313
- let error;
314
- let stderr;
315
- let done = true;
316
- if (debug) {
317
- signale.debug(`executing: docker ${args.join(' ')}`);
318
- }
319
- const subprocess = execa('docker', args);
320
- if (!subprocess || !subprocess.stderr) {
321
- throw new Error('Failed to execute docker run');
322
- }
323
- (async () => {
324
- done = false;
325
- try {
326
- const result = await subprocess;
327
- if (result.exitCode && result.exitCode > 0) {
328
- stderr = result.all;
329
- error = new Error(`${result.command} failed`);
330
- }
331
- }
332
- catch (err) {
333
- if (err.failed) {
334
- error = err.stderr;
335
- }
336
- error = err.stack;
337
- stderr = err.all;
338
- }
339
- finally {
340
- done = true;
341
- }
342
- })();
343
- const upFor = ms('3s');
344
- await pWhile(() => dockerContainerReady(opt.name, upFor, error), {
345
- name: `Docker container up for 2m (${opt.name})`,
346
- timeoutMs: ms('2m')
347
- });
348
- if (error) {
349
- if (stderr) {
350
- process.stderr.write(stderr);
351
- }
352
- throw error;
353
- }
354
- if (done) {
355
- throw new Error('Service ended early');
356
- }
357
- return () => {
358
- if (error) {
359
- if (stderr) {
360
- process.stderr.write(stderr);
361
- }
362
- signale.error(error);
363
- }
364
- if (done && !subprocess.killed)
365
- return;
366
- subprocess.kill();
367
- };
368
- }
369
- export async function dockerContainerReady(name, upFor, error) {
370
- if (error)
371
- throw new TSError(error);
372
- try {
373
- const result = await exec({
374
- cmd: 'docker',
375
- args: [
376
- 'ps', '--format', '"{{json .Status}}"', '--filter', `name=${name}`
377
- ]
378
- });
379
- const timeup = ms(result.replace(/[(Up)\s"]+|/ig, ''));
380
- if (!timeup)
381
- return false;
382
- return timeup >= upFor;
383
- }
384
- catch (err) {
385
- await pDelay(1000);
386
- return false;
387
- }
388
- }
389
- export async function dockerBuild(tag, cacheFrom, target, buildArgs, dockerFileName, dockerFilePath) {
390
- const cacheFromArgs = [];
391
- cacheFrom?.forEach((image) => {
392
- cacheFromArgs.push('--cache-from', image);
393
- });
394
- const targetArgs = target ? ['--target', target] : [];
395
- const buildsArgs = buildArgs
396
- ? ['--build-arg', ...buildArgs.join(',--build-arg,').split(',')]
397
- : [];
398
- const fileName = dockerFileName ? ['-f', dockerFileName] : [];
399
- const filePath = dockerFilePath ?? '.';
400
- await fork({
401
- cmd: 'docker',
402
- args: ['build', ...cacheFromArgs, ...targetArgs, ...buildsArgs, '--tag', tag, ...fileName, filePath],
403
- });
404
- }
405
- export async function dockerPush(image) {
406
- const subprocess = await execaCommand(`docker push ${image}`, { reject: false });
407
- if (subprocess.exitCode !== 0) {
408
- throw new Error(`Unable to push docker image ${image}, ${subprocess.stderr}`);
409
- }
410
- }
411
- export async function dockerExec(containerName, command) {
412
- const args = ['exec', containerName, ...command];
413
- signale.debug(`executing: docker ${args.join(' ')}`);
414
- const subprocess = await execa('docker', args, { stdio: 'pipe' });
415
- if (subprocess.exitCode !== 0) {
416
- throw new Error(`Docker exec to container ${containerName} failed, ${subprocess.stderr}`);
417
- }
418
- }
419
- async function dockerImageRm(image) {
420
- const subprocess = await execaCommand(`docker image rm ${image}`, { reject: false });
421
- if (subprocess.exitCode !== 0) {
422
- throw new Error(`Unable to remove docker image ${image}, ${subprocess.stderr}`);
423
- }
424
- }
425
- /**
426
- * Unzips and loads a Docker image from a Docker cache
427
- * If successful and skipDelete is false the image will be deleted from the cache
428
- * @param {string} imageName Name of the image to load
429
- * @param {boolean} skipDelete Skip removal of docker image from cache
430
- * @returns {Promise<boolean>} Whether or not the image loaded successfully
431
- */
432
- export async function loadThenDeleteImageFromCache(imageName, skipDelete = false) {
433
- signale.time(`unzip and load ${imageName}`);
434
- const fileName = imageName.trim().replace(/[/:]/g, '_');
435
- const filePath = path.join(config.DOCKER_CACHE_PATH, `${fileName}.tar.gz`);
436
- if (!fs.existsSync(filePath)) {
437
- signale.error(`No file found at ${filePath}. Have you restored the cache?`);
438
- return false;
439
- }
440
- const result = await execaCommand(`gunzip -c ${filePath} | docker load`, { shell: true });
441
- signale.info('Result: ', result);
442
- if (result.exitCode !== 0) {
443
- signale.error(`Error loading ${filePath} to docker`);
444
- return false;
445
- }
446
- if (!skipDelete) {
447
- signale.info(`Deleting ${imageName} from docker image cache.`);
448
- fs.rmSync(filePath);
449
- }
450
- signale.timeEnd(`unzip and load ${imageName}`);
451
- return true;
452
- }
453
- export async function deleteDockerImageCache() {
454
- signale.info(`Deleting Docker image cache at ${config.DOCKER_CACHE_PATH}`);
455
- fse.removeSync(config.DOCKER_CACHE_PATH);
456
- }
457
- export async function pgrep(name) {
458
- const result = await exec({ cmd: 'ps', args: ['aux'] }, false);
459
- if (!result) {
460
- throw new Error('Invalid result from ps aux');
461
- }
462
- const found = result.split('\n').find((line) => {
463
- if (!line)
464
- return false;
465
- return line.toLowerCase().includes(name.toLowerCase());
466
- });
467
- if (found) {
468
- logger.trace('found process', found);
469
- return found;
470
- }
471
- return '';
472
- }
473
- /**
474
- * Save a docker image as a tar.gz to a local directory.
475
- * Then remove the image from docker
476
- * @param {string} imageName Name of image to pull and save
477
- * @param {string} imageSavePath Location where image will be saved and compressed.
478
- * @returns void
479
- */
480
- export async function saveAndZip(imageName, imageSavePath) {
481
- signale.info(`Saving Docker image: ${imageName}`);
482
- const fileName = imageName.replace(/[/:]/g, '_');
483
- const filePath = path.join(imageSavePath, `${fileName}.tar`);
484
- const command = `docker save ${imageName} | gzip > ${filePath}.gz`;
485
- await execaCommand(command, { shell: true });
486
- await dockerImageRm(imageName);
487
- }
488
- export async function getCommitHash() {
489
- if (process.env.GIT_COMMIT_HASH)
490
- return process.env.GIT_COMMIT_HASH;
491
- if (config.SKIP_GIT_COMMANDS) {
492
- throw new Error('Unable to determine git commit hash when env.SKIP_GIT_COMMANDS is set, set env.GIT_COMMIT_HASH to fix this');
493
- }
494
- return exec({ cmd: 'git', args: ['rev-parse', '--short', 'HEAD'] });
495
- }
496
- export async function gitDiff(files = []) {
497
- if (config.SKIP_GIT_COMMANDS)
498
- return;
499
- try {
500
- await fork({ cmd: 'git', args: ['diff', ...files] });
501
- }
502
- catch (e) {
503
- process.exitCode = 0;
504
- logger.warn(e);
505
- }
506
- }
507
- export async function getChangedFiles(...files) {
508
- if (config.SKIP_GIT_COMMANDS)
509
- return [];
510
- try {
511
- const result = await exec({
512
- cmd: 'git', args: ['diff', '--name-only', ...files]
513
- });
514
- return result
515
- .split('\n')
516
- .map((str) => str.trim())
517
- .filter(Boolean);
518
- }
519
- catch (e) {
520
- // if there error includes ENOENT
521
- // we can just log the error and continue on
522
- if (e.toString().includes('ENOENT')) {
523
- process.exitCode = 0;
524
- logger.warn(e);
525
- return [];
526
- }
527
- throw e;
528
- }
529
- }
530
- export function mapToArgs(input) {
531
- const args = [];
532
- for (const [key, value] of Object.entries(input)) {
533
- const vals = Array.isArray(value) ? value : [value];
534
- if (key.length > 1) {
535
- args.push(`--${key}`, ...vals);
536
- }
537
- else {
538
- args.push(`-${key}`, ...vals);
539
- }
540
- }
541
- return args.filter((str) => str != null && str !== '');
542
- }
543
- /**
544
- * Publish package to NPM using yarn (versions 2, 3, 4) or pnpm
545
- */
546
- export async function packageMngrPublish(pkgInfo, tag = 'latest') {
547
- const pm = getPackageManager();
548
- const args = pm === 'pnpm'
549
- ? ['publish', '--tag', tag, '--no-git-checks']
550
- : ['npm', 'publish', '--tag', tag];
551
- await fork({
552
- cmd: pm,
553
- args,
554
- cwd: pkgInfo.dir,
555
- env: {
556
- NODE_ENV: 'production'
557
- }
558
- });
559
- }
560
- export async function isKindInstalled() {
561
- try {
562
- const subprocess = await execaCommand('command -v kind');
563
- return !!subprocess.stdout;
564
- }
565
- catch (err) {
566
- return false;
567
- }
568
- }
569
- export async function isKubectlInstalled() {
570
- try {
571
- const subprocess = await execaCommand('command -v kubectl');
572
- return !!subprocess.stdout;
573
- }
574
- catch (err) {
575
- return false;
576
- }
577
- }
578
- export async function isHelmInstalled() {
579
- try {
580
- const subprocess = await execaCommand('command -v helm');
581
- return !!subprocess.stdout;
582
- }
583
- catch (err) {
584
- return false;
585
- }
586
- }
587
- export async function isHelmfileInstalled() {
588
- try {
589
- const subprocess = await execaCommand('command -v helmfile');
590
- return !!subprocess.stdout;
591
- }
592
- catch (err) {
593
- return false;
594
- }
595
- }
596
- function waitForKafkaRunning(name, timeoutMs = 12000) {
597
- const endAt = Date.now() + timeoutMs;
598
- const _waitForKafkaRunning = async () => {
599
- if (Date.now() > endAt) {
600
- if (logger.level() <= 20) {
601
- try {
602
- const errorSearchCommand = await execaCommand(`kubectl -n services-dev1 logs -l app.kubernetes.io/name=${name}`);
603
- logger.debug('Kafka pod logs:');
604
- logger.debug(errorSearchCommand.stdout);
605
- }
606
- catch (err) {
607
- logger.error(err, 'Failure to retrieve kafka pod logs');
608
- const describePodCommand = await execaCommand(`kubectl -n services-dev1 describe pods -l app.kubernetes.io/name=${name}`);
609
- logger.debug('Describe kafka pod:');
610
- logger.debug(describePodCommand.stdout);
611
- }
612
- }
613
- throw new Error(`Failure to communicate with kafka after ${timeoutMs}ms`);
614
- }
615
- let kafkaRunning = false;
616
- try {
617
- const kubectlResponse = await execaCommand(`kubectl -n services-dev1 get pods -l app.kubernetes.io/name=${name} -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"`);
618
- const kafkaReady = kubectlResponse.stdout;
619
- logger.debug('Kafka ready: ', kafkaReady);
620
- if (kafkaReady === '"true"') {
621
- kafkaRunning = true;
622
- }
623
- }
624
- catch (err) {
625
- await pDelay(3000);
626
- return _waitForKafkaRunning();
627
- }
628
- if (kafkaRunning) {
629
- return;
630
- }
631
- await pDelay(3000);
632
- return _waitForKafkaRunning();
633
- };
634
- return _waitForKafkaRunning();
635
- }
636
- export async function setAlias(tsPort) {
637
- try {
638
- const subprocess1 = await execaCommand('earl aliases remove k8s-e2e 2> /dev/null || true', { shell: true });
639
- logger.debug(subprocess1.stdout);
640
- if (subprocess1.stderr) {
641
- throw new Error(subprocess1.stderr);
642
- }
643
- const subprocess2 = await execaCommand(`earl aliases add k8s-e2e http://${config.HOST_IP}:${tsPort}`);
644
- logger.debug(subprocess2.stdout);
645
- if (subprocess2.stderr) {
646
- throw new Error(subprocess2.stderr);
647
- }
648
- }
649
- catch (err) {
650
- throw new Error(`Failed to set alias: ${err}`);
651
- }
652
- }
653
- export async function showState(tsPort, isTeardown = false) {
654
- try {
655
- if (isTeardown) {
656
- logger.debug('=== k8s cluster state ===');
657
- const clusterState = await execaCommand('kubectl get deployments,po,svc --all-namespaces --show-labels -o wide');
658
- logger.debug(clusterState.stdout);
659
- logger.debug('=== docker stats ===');
660
- const dockerStats = await execaCommand('docker stats --no-stream');
661
- logger.debug(dockerStats.stdout);
662
- logger.debug('=== opensearch2 pod description ===');
663
- const searchHost = await determineSearchHost();
664
- const os2Desc = await execaCommand(`kubectl -n services-dev1 describe pod ${searchHost}-cluster-master-0`);
665
- logger.debug(os2Desc.stdout);
666
- // TODO: consider adding describe for kafka and minio pods
667
- }
668
- const subprocess = await execaCommand('kubectl get deployments,po,svc --all-namespaces --show-labels -o wide');
669
- logger.debug(subprocess.stdout);
670
- logger.debug(await showESIndices());
671
- logger.debug(await showAssets(tsPort));
672
- }
673
- catch (err) {
674
- signale.error(`Failed to get k8s resources: ${err}`);
675
- }
676
- }
677
- async function showESIndices() {
678
- const subprocess = await execaCommand(`curl -k ${config.SEARCH_TEST_HOST}/_cat/indices?v`);
679
- return subprocess.stdout;
680
- }
681
- async function showAssets(tsPort) {
682
- try {
683
- const subprocess = await execaCommand(`curl ${config.HOST_IP}:${tsPort}/v1/assets`);
684
- return subprocess.stdout;
685
- }
686
- catch (err) {
687
- return err;
688
- }
689
- }
690
- export async function logTCPPorts(service) {
691
- try {
692
- const command = 'netstat -an | grep \'^tcp\' | awk \'{print $4}\' | tr ".:" " " | awk \'{print $NF}\' | sort -n | uniq | tr "\n" " "';
693
- const subprocess = await execaCommand(command, { shell: true, reject: false });
694
- const { stdout, stderr } = subprocess;
695
- if (stderr) {
696
- throw new Error(stderr);
697
- }
698
- signale.info(`TCP Ports currently in use when starting ${service}:\n ${stdout}`);
699
- }
700
- catch (err) {
701
- signale.error(`Execa command failed trying to log ports: ${err}`);
702
- }
703
- }
704
- export async function deletePersistentVolumeClaim(searchHost) {
705
- try {
706
- const label = searchHost.includes('opensearch') ? `app.kubernetes.io/instance=${searchHost}` : `app=${searchHost}-master`;
707
- const subprocess = await execaCommand(`kubectl delete -n services-dev1 pvc -l ${label}`);
708
- logger.debug(`kubectl delete pvc: ${subprocess.stdout}`);
709
- }
710
- catch (err) {
711
- throw new TSError(`Failed to delete persistent volume claim:\n${err}`);
712
- }
713
- }
714
- export async function helmfileDestroy(selector) {
715
- const helmfilePath = path.join(getRootDir(), 'packages/scripts/helm/helmfile.yaml.gotmpl');
716
- try {
717
- const subprocess = await execaCommand(`helmfile destroy -f ${helmfilePath} --selector app=${selector}`);
718
- logger.debug(`helmfile destroy:\n${subprocess.stdout}`);
719
- }
720
- catch (err) {
721
- logger.info(err);
722
- }
723
- }
724
- export async function helmfileCommand(command, clusteringType, devMode = false, logs = false, e2e = true) {
725
- const helmfilePath = path.join(getRootDir(), 'packages/scripts/helm/helmfile.yaml.gotmpl');
726
- const { valuesPath, valuesDir } = generateHelmValuesFromServices(clusteringType, devMode, logs, e2e);
727
- let subprocess;
728
- try {
729
- subprocess = await execaCommand(`helmfile --state-values-file ${valuesPath} ${command} -f ${helmfilePath}`);
730
- }
731
- catch (err) {
732
- // TSError truncates to 3000 characters which is an issue here
733
- throw new Error(`Helmfile ${command} command failed:\n${err}`);
734
- }
735
- finally {
736
- fs.rmSync(valuesDir, { recursive: true, force: true });
737
- }
738
- logger.debug(`helmfile ${command}:\n${subprocess.stdout}`);
739
- }
740
- export async function launchTerasliceWithHelmfile(clusteringType, devMode = false, logs = false, debug = false, e2e = true) {
741
- if (debug && !isCI) {
742
- await helmfileCommand('diff', clusteringType, devMode, logs, e2e);
743
- }
744
- await helmfileCommand('sync', clusteringType, devMode, logs, e2e);
745
- if (config.ENV_SERVICES.includes(Service.Kafka)) {
746
- await waitForKafkaRunning('kafka');
747
- }
748
- }
749
- export async function launchTerasliceWithCustomHelmfile(configFilePath, debug = false, selector) {
750
- let diffProcess;
751
- let syncProcess;
752
- const diffSelector = selector && selector.diff ? `-l group!=skipDiff,${selector.diff}` : '-l group!=skipDiff';
753
- const syncSelector = selector && selector.sync ? `-l ${selector.sync}` : '';
754
- const helmfilePath = path.join(getRootDir(), 'packages/scripts/helm/helmfile.yaml.gotmpl');
755
- try {
756
- if (debug && !isCI) {
757
- // We want to exclude certain charts from the diff command because
758
- // they may require crds that aren't installed
759
- diffProcess = await execaCommand(`helmfile ${diffSelector} --state-values-file ${configFilePath} diff -f ${helmfilePath}`);
760
- logger.debug(`helmfile diff:\n${diffProcess.stdout}`);
761
- }
762
- syncProcess = await execaCommand(`helmfile ${syncSelector} --state-values-file ${configFilePath} sync -f ${helmfilePath}`);
763
- logger.debug(`helmfile sync:\n${syncProcess.stdout}`);
764
- }
765
- catch (err) {
766
- // TSError truncates to 3000 characters which is an issue here
767
- throw new Error(`Helmfile command failed:\n${err}`);
768
- }
769
- }
770
- export async function determineSearchHost() {
771
- const possible = ['opensearch1', 'opensearch2', 'opensearch3'];
772
- const subprocess = await execaCommand('helm list -n services-dev1 -o json');
773
- logger.debug(`helmfile list:\n${subprocess.stdout}`);
774
- const serviceList = JSON.parse(subprocess.stdout);
775
- const filtered = serviceList.filter((svc) => possible.includes(svc.name));
776
- if (filtered.length > 1) {
777
- throw new TSError('Multiple Possible Search Hosts Detected. Cannot reset store.');
778
- }
779
- if (filtered.length === 0) {
780
- throw new TSError('No Search Host Detected. Cannot reset store.');
781
- }
782
- return filtered[0].name;
783
- }
784
- // Helper function for reading the contents of a certificate by providing its path
785
- function readCertFromPath(certPath) {
786
- if (!fs.existsSync(certPath)) {
787
- throw new TSError(`Unable to find cert at: ${certPath}`);
788
- }
789
- return fs.readFileSync(certPath, 'utf8');
790
- }
791
- /**
792
- * Extracts the admin distinguished name (DN) from a certificate.
793
- *
794
- * This function is designed specifically for mkcert generated certificates.
795
- * It reads the certificate file (`opensearch-cert.pem`), extracts the `O`
796
- * and `OU` fields from the subject, and formats them
797
- * in the required order (`OU` first, `O` second) for OpenSearch authentication.
798
- *
799
- * @returns {string} The formatted (DN) string in the format:
800
- * `"OU=example, O=example"`
801
- * @throws {Error} If the certificate file is missing or invalid.
802
- *
803
- * @example
804
- * ```ts
805
- * const adminDn = getAdminDnFromCert();
806
- * console.log(adminDn);
807
- * // Output: "OU=anon@anon-MBP (Anon User),O=mkcert development certificate"
808
- * ```
809
- */
810
- export function getAdminDnFromCert() {
811
- let ca;
812
- let organization;
813
- let organizationalUnit;
814
- try {
815
- ca = readCertFromPath(path.join(config.CERT_PATH, 'opensearch-cert.pem'));
816
- }
817
- catch (err) {
818
- throw new TSError(`Failed to read certificate file (opensearch-cert.pem).\n${err}`);
819
- }
820
- try {
821
- const rootCA = new X509Certificate(ca);
822
- // This splits the OU and O in two separate parts
823
- const subjectParts = rootCA.subject.split('\n');
824
- // Loop through the parts and assign based on prefix
825
- // We don't want to assume the order that these are returned
826
- for (const part of subjectParts) {
827
- if (part.startsWith('OU=')) {
828
- organizationalUnit = part;
829
- }
830
- else if (part.startsWith('O=')) {
831
- organization = part;
832
- }
833
- }
834
- }
835
- catch (err) {
836
- throw new TSError(`Failed to parse openSearch certificate. Make sure it's a valid X.509 certificate.\n${err}`);
837
- }
838
- if (!organizationalUnit || !organization) {
839
- throw new TSError(`Certificate is missing required fields. Expected both 'OU' and 'O' fields.`);
840
- }
841
- // Return with specific format that opensearch expects
842
- return `${organizationalUnit},${organization}`;
843
- }
844
- /**
845
- * Generates a temporary `values.yaml` file based on the ts-scripts command configuration.
846
- * This file is used to configure Helmfile when launching the k8sEnv or test environment.
847
- *
848
- * The function:
849
- * - Loads a base `values.yaml` template from `packages/scripts/helm/values.yaml`.
850
- * - Enables services specified in `ENV_SERVICES`, setting their versions when needed
851
- * - Configures OpenSearch to align with versioning conventions.
852
- * - Handles OpenSearch, Minio and Kafka SSL settings if encryption is enabled.
853
- * - Adds extraVolumes, extraVolumeMounts and env values if running in dev mode.
854
- * - Generates a temporary directory to store the modified `values.yaml`.
855
- *
856
- * @param { 'kubernetesV2' } clusteringType - backend cluster manager type
857
- * @param { boolean } devMode - Mount local teraslice to k8s resources for faster development.
858
- * @param { boolean } logs - Copy teraslice and service logs from k8s pods to local filesystem.
859
- * @param { boolean } e2e - Specify e2e tests or k8s env mode.
860
- * @returns An object containing:
861
- * - `valuesPath` - Path to the generated `values.yaml` file.
862
- * - `valuesDir` - Path to the temporary directory containing the file.
863
- */
864
- function generateHelmValuesFromServices(clusteringType, devMode, logs, e2e) {
865
- // Grab default values from the packages/scripts/helm/values.yaml
866
- const helmfileValuesPath = path.join(getRootDir(), 'packages/scripts/helm/values.yaml');
867
- const values = parseDocument(fs.readFileSync(helmfileValuesPath, 'utf8'));
868
- // Map services to versions used for the image tag
869
- const versionMap = {
870
- [Service.Opensearch]: config.OPENSEARCH_VERSION,
871
- [Service.Kafka]: config.KAFKA_VERSION,
872
- [Service.Minio]: config.MINIO_VERSION,
873
- [Service.RabbitMQ]: config.RABBITMQ_VERSION,
874
- [Service.RestrainedOpensearch]: config.OPENSEARCH_VERSION,
875
- [Service.Utility]: config.UTILITY_SVC_VERSION,
876
- [Service.Valkey]: config.VALKEY_VERSION,
877
- [Service.Teraslice]: config.TERASLICE_DOCKER_IMAGE,
878
- };
879
- let stateCluster;
880
- let caCert;
881
- // disable chart's OS2 default
882
- values.setIn(['opensearch2', 'enabled'], false);
883
- // Iterate over each service we want to start and enable them in the
884
- // helmfile.
885
- config.ENV_SERVICES.forEach((service) => {
886
- // "serviceString" represents the literal service name string
887
- // in the "values.yaml"
888
- let serviceString = service;
889
- const version = versionMap[service];
890
- if (service === Service.Opensearch) {
891
- const major = config.OPENSEARCH_VERSION.charAt(0);
892
- serviceString += major;
893
- // This assumes there is only one search service enabled. If both ES and OS services
894
- // are present the state cluster will be set to elasticsearch below.
895
- stateCluster = serviceString;
896
- if (config.ENCRYPT_OPENSEARCH) {
897
- if (major === '1') {
898
- throw new TSError('Encrypted Opensearch version 1 is not enabled. Please use OS2 or OS3.');
899
- }
900
- if (!caCert) {
901
- caCert = readCertFromPath(path.join(config.CERT_PATH, 'CAs/rootCA.pem')).replace(/\n/g, '\\n');
902
- }
903
- const admin_dn = getAdminDnFromCert();
904
- values.setIn([serviceString, 'ssl', 'enabled'], true);
905
- values.setIn([serviceString, 'ssl', 'caCert'], caCert);
906
- values.setIn([serviceString, 'ssl', 'admin_dn'], admin_dn);
907
- }
908
- }
909
- if (service === Service.Kafka) {
910
- if (config.ENCRYPT_KAFKA) {
911
- if (!caCert) {
912
- caCert = readCertFromPath(path.join(config.CERT_PATH, 'CAs/rootCA.pem')).replace(/\n/g, '\\n');
913
- }
914
- values.setIn(['kafka', 'ssl', 'enabled'], true);
915
- values.setIn(['kafka', 'ssl', 'caCert'], caCert);
916
- }
917
- // Listener settings go into configurationOverrides (dotted Kafka key format) so the
918
- // chart's brokerConfigs helper merges them with internal defaults without creating
919
- // duplicate env vars. Values come from config.ts which sets k8s-appropriate defaults
920
- // based on TEST_PLATFORM and ENCRYPT_KAFKA. If you add new listener settings, add
921
- // them to config.ts and include them in this object.
922
- const kafkaK8sConfigOverrides = {
923
- 'advertised.listeners': config.KAFKA_ADVERTISED_LISTENERS,
924
- 'listener.security.protocol.map': config.KAFKA_LISTENER_SECURITY_PROTOCOL_MAP,
925
- listeners: config.KAFKA_LISTENERS,
926
- };
927
- if (config.ENCRYPT_KAFKA) {
928
- // PLAINTEXT is the inter-broker listener in the SSL setup: brokers talk to each
929
- // other on the internal PLAINTEXT listener while clients connect via TLS.
930
- kafkaK8sConfigOverrides['inter.broker.listener.name'] = 'PLAINTEXT';
931
- kafkaK8sConfigOverrides['security.protocol'] = config.KAFKA_SECURITY_PROTOCOL.toLowerCase();
932
- }
933
- values.setIn(['kafka', 'configurationOverrides'], kafkaK8sConfigOverrides);
934
- // Only behavioral vars belong in envOverrides — the chart does not generate these
935
- // internally so there is no duplicate risk.
936
- // If you add a new Kafka env var to config.ts and services.ts, add it here too.
937
- values.setIn(['kafka', 'envOverrides', 'KAFKA_AUTO_CREATE_TOPICS_ENABLE'], config.KAFKA_AUTO_CREATE_TOPICS_ENABLE);
938
- values.setIn(['kafka', 'envOverrides', 'KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR'], config.KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR);
939
- values.setIn(['kafka', 'envOverrides', 'KAFKA_TRANSACTION_STATE_LOG_MIN_ISR'], config.KAFKA_TRANSACTION_STATE_LOG_MIN_ISR);
940
- values.setIn(['kafka', 'envOverrides', 'KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS'], config.KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS);
941
- }
942
- if (service === Service.Minio) {
943
- if (config.ENCRYPT_MINIO) {
944
- if (!caCert) {
945
- caCert = readCertFromPath(path.join(config.CERT_PATH, 'CAs/rootCA.pem')).replace(/\n/g, '\\n');
946
- }
947
- const publicCert = readCertFromPath(path.join(config.CERT_PATH, 'public.crt')).replace(/\n/g, '\\n');
948
- const privateKey = readCertFromPath(path.join(config.CERT_PATH, 'private.key')).replace(/\n/g, '\\n');
949
- values.setIn(['minio', 'tls', 'enabled'], true);
950
- values.setIn(['minio', 'tls', 'caCert'], caCert);
951
- values.setIn(['minio', 'tls', 'publicCert'], publicCert);
952
- values.setIn(['minio', 'tls', 'privateKey'], privateKey);
953
- values.setIn(['minio', 'tls', 'certSecret'], 'tls-ssl-minio');
954
- }
955
- }
956
- if (service === Service.Valkey) {
957
- if (config.ENCRYPT_VALKEY) {
958
- throw new Error('Valkey encryption not supported');
959
- }
960
- }
961
- values.setIn([serviceString, 'enabled'], true);
962
- values.setIn([serviceString, 'version'], version);
963
- });
964
- // If no search service specified then default to OS2
965
- if (!stateCluster) {
966
- stateCluster = 'opensearch2';
967
- values.setIn(['opensearch2', 'enabled'], true);
968
- }
969
- values.setIn(['teraslice', 'stateCluster'], stateCluster);
970
- values.setIn(['teraslice', 'image', 'tag'], `e2e-nodev${config.NODE_VERSION}`);
971
- values.setIn(['teraslice', 'asset_storage_connection_type'], config.ASSET_STORAGE_CONNECTION_TYPE);
972
- values.setIn(['teraslice', 'asset_storage_connection'], config.ASSET_STORAGE_CONNECTION);
973
- values.setIn(['teraslice', 'cluster_manager_type'], clusteringType);
974
- values.setIn(['teraslice', 'name'], config.CLUSTER_NAME);
975
- values.setIn(['teraslice', 'assets_directory'], e2e ? '/app/e2e-assets' : '/app/assets');
976
- values.setIn(['teraslice', 'e2e'], e2e);
977
- if (devMode) {
978
- // Dev image runs as root and writes/compiles into the bind-mounted source at
979
- // runtime, so the chart's hardened securityContext won't work. Override the
980
- // blocking fields back to permissive (explicit values, since Helm deep-merges
981
- // and an empty {} would leave the hardened defaults in place).
982
- values.setIn(['teraslice', 'securityContext'], {
983
- runAsUser: 0,
984
- runAsNonRoot: false,
985
- readOnlyRootFilesystem: false
986
- });
987
- const dockerfileMounts = getVolumesFromDockerfile(false, logger);
988
- // Shared node_modules volume: lives on the Kind node
989
- // mounted into all pods so only the master runs pnpm install once.
990
- dockerfileMounts.volumes.push({
991
- name: 'dev-node-modules',
992
- hostPath: { path: '/dev-node-modules', type: 'DirectoryOrCreate' }
993
- });
994
- dockerfileMounts.volumeMounts.push({
995
- name: 'dev-node-modules',
996
- mountPath: '/app/source/node_modules'
997
- });
998
- values.setIn(['teraslice', 'extraVolumeMounts'], dockerfileMounts.volumeMounts);
999
- values.setIn(['teraslice', 'extraVolumes'], dockerfileMounts.volumes);
1000
- /// Pass in env so master passes volumes to ex's and workers
1001
- values.setIn(['teraslice', 'env'], {
1002
- MOUNT_LOCAL_TERASLICE: Buffer.from(JSON.stringify(dockerfileMounts)).toString('base64')
1003
- });
1004
- }
1005
- if (logs) {
1006
- values.setIn(['stern', 'enabled'], true);
1007
- const arch = os.arch() === 'x64' ? 'amd64' : 'arm64';
1008
- const sternVersion = '1.30.0';
1009
- values.setIn(['stern', 'downloadUrl'], `https://github.com/stern/stern/releases/download/v${sternVersion}/stern_${sternVersion}_linux_${arch}.tar.gz`);
1010
- }
1011
- signale.debug('helmfile command values: ', JSON.stringify(values));
1012
- // Write the values to a temporary file
1013
- const valuesDir = fs.mkdtempSync(path.join(os.tmpdir(), 'generated-yaml'));
1014
- const valuesPath = path.join(valuesDir, 'values.yaml');
1015
- fs.writeFileSync(valuesPath, values.toString(), 'utf8');
1016
- return { valuesPath, valuesDir };
1017
- }
1018
- /**
1019
- * Gets the current version of the Teraslice Helm chart from `Chart.yaml`.
1020
- *
1021
- * @throws {Error} If the `Chart.yaml` file cannot be read
1022
- * @returns {Promise<string>} Resolves with the Helm chart version as a string
1023
- */
1024
- export async function getCurrentHelmChartVersion() {
1025
- const chartYamlPath = path.join(getRootDir(), '/helm/teraslice/Chart.yaml');
1026
- const chartYAML = await load(fs.readFileSync(chartYamlPath, 'utf8'));
1027
- return chartYAML.version;
1028
- }
1029
- function getTerasliceVersion() {
1030
- const rootPackageInfo = getRootInfo();
1031
- return rootPackageInfo.version;
1032
- }
1033
- /**
1034
- * Extracts the base Docker image information from the top-level Dockerfile
1035
- *
1036
- * This function parses the Dockerfile to determine the base image name,
1037
- * node version, registry, and repository.
1038
- *
1039
- * @throws {TSError} If the Dockerfile cannot be read or the base image format is unexpected.
1040
- * @returns {{
1041
- * name: string;
1042
- * tag: string;
1043
- * registry: string;
1044
- * repo: string;
1045
- * }} An object containing:
1046
- * - `name: Full base image name
1047
- * - `tag`: Node version used in the image
1048
- * - `registry`: Docker registry (defaults to `docker.io` if not specified)
1049
- * - `repo`: Repository name including organization
1050
- */
1051
- export function getDockerBaseImageInfo() {
1052
- try {
1053
- const dockerFilePath = path.join(getRootDir(), 'Dockerfile');
1054
- const dockerfileContent = fs.readFileSync(dockerFilePath, 'utf8');
1055
- // Grab the "ARG NODE_VERSION" line in the Dockerfile
1056
- const nodeVersionDefault = dockerfileContent.match(/^ARG NODE_VERSION=(\d+)/m);
1057
- if (nodeVersionDefault) {
1058
- return {
1059
- name: `node`,
1060
- tag: `${nodeVersionDefault[1]}-alpine`,
1061
- registry: 'docker.io',
1062
- repo: 'library'
1063
- };
1064
- }
1065
- else {
1066
- throw new TSError('Failed to parse Dockerfile for base image.');
1067
- }
1068
- }
1069
- catch (err) {
1070
- throw new TSError(`Failed to read top-level Dockerfile to get base image.\n${err}`);
1071
- }
1072
- }
1073
- /**
1074
- * Retrieves the Node.js version from the base Docker image specified in the Teraslice `Dockerfile`.
1075
- *
1076
- * This function:
1077
- * - Extracts the base image details from the `Dockerfile`
1078
- * - Authenticates with the container registry to retrieve a token
1079
- * - Fetches the image manifest and configuration
1080
- * - Extracts the `node_version` label from the image config
1081
- *
1082
- * @throws {TSError} If any request to the registry fails or expected data is missing.
1083
- * @returns {Promise<string>} Resolves with the Node.js version string.
1084
- */
1085
- export async function grabCurrentTSNodeVersion() {
1086
- // Extract base image details from the Dockerfile
1087
- const baseImage = getDockerBaseImageInfo();
1088
- let token;
1089
- // Request authentication token for accessing image manifests
1090
- try {
1091
- const authUrl = 'https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/node:pull';
1092
- const authResponse = await got(authUrl);
1093
- token = JSON.parse(authResponse.body).token;
1094
- }
1095
- catch (err) {
1096
- throw new TSError(`Unable to retrieve token from ${baseImage.registry} for repo ${baseImage.repo}:\n${err}`);
1097
- }
1098
- // Grab the manifest list to find the right architecture digest
1099
- let manifestDigest;
1100
- try {
1101
- const manifestUrl = `https://registry-1.docker.io/v2/library/node/manifests/${baseImage.tag}`;
1102
- const response = await got(manifestUrl, {
1103
- headers: {
1104
- Authorization: `Bearer ${token}`,
1105
- },
1106
- responseType: 'json'
1107
- });
1108
- const body = response.body;
1109
- const amd64Manifest = body.manifests.find((manifest) => manifest.platform.architecture === 'amd64');
1110
- if (!amd64Manifest) {
1111
- throw new TSError(`No amd64 manifest found for ${baseImage.repo}:${baseImage.tag}`);
1112
- }
1113
- manifestDigest = amd64Manifest.digest;
1114
- }
1115
- catch (err) {
1116
- throw new TSError(`Unable to retrieve image manifest list from ${baseImage.registry} for ${baseImage.repo}:${baseImage.tag}:\n${err}`);
1117
- }
1118
- let configManifest;
1119
- try {
1120
- const manifestUrl = `https://registry-1.docker.io/v2/library/node/manifests/${manifestDigest}`;
1121
- const response = await got(manifestUrl, {
1122
- headers: {
1123
- Authorization: `Bearer ${token}`,
1124
- },
1125
- responseType: 'json'
1126
- });
1127
- const manifestList = response.body;
1128
- configManifest = manifestList.config.digest;
1129
- if (!configManifest) {
1130
- throw new TSError(`No config digest found for ${baseImage.repo}:${baseImage.tag}`);
1131
- }
1132
- }
1133
- catch (err) {
1134
- throw new TSError(`Unable to retrieve image manifest list from ${baseImage.registry} for ${baseImage.repo}:${baseImage.tag}:\n${err}`);
1135
- }
1136
- // Retrieve the image configuration and extract the Node.js version label
1137
- try {
1138
- const configUrl = `https://registry-1.docker.io/v2/library/node/blobs/${configManifest}`;
1139
- const response = await got(configUrl, {
1140
- headers: {
1141
- Authorization: `Bearer ${token}`,
1142
- },
1143
- responseType: 'json'
1144
- });
1145
- const imageConfig = response.body;
1146
- const nodeVersion = imageConfig.config?.Env?.find((envVar) => envVar.startsWith('NODE_VERSION='))?.split('=')[1];
1147
- if (!nodeVersion) {
1148
- throw new TSError(`Node version label missing in config for ${baseImage.repo}:${baseImage.tag}`);
1149
- }
1150
- return nodeVersion;
1151
- }
1152
- catch (err) {
1153
- throw new TSError(`Unable to grab image config from ${baseImage.registry} for ${baseImage.repo}:${baseImage.tag}:\n${err}`);
1154
- }
1155
- }
1156
- /**
1157
- * Updates the Teraslice Helm chart YAML files (`Chart.yaml` and `values.yaml`)
1158
- * with the new chart version
1159
- *
1160
- * @param {string | null} newChartVersion - The new version to set in `Chart.yaml`.
1161
- * - If `null`, the function does not update the chart version.
1162
- * @throws {TSError} If the function fails to read or write YAML files.
1163
- * @returns {Promise<void>} Resolves when the Helm chart files have been successfully updated
1164
- */
1165
- export async function updateHelmChart(newChartVersion) {
1166
- const currentNodeVersion = await grabCurrentTSNodeVersion();
1167
- const rootDir = getRootDir();
1168
- const chartYamlPath = path.join(rootDir, 'helm/teraslice/Chart.yaml');
1169
- const valuesYamlPath = path.join(rootDir, 'helm/teraslice/values.yaml');
1170
- try {
1171
- // Read YAML files and parse them into objects
1172
- const chartFileContent = fs.readFileSync(chartYamlPath, 'utf8');
1173
- const valuesFileContent = fs.readFileSync(valuesYamlPath, 'utf8');
1174
- const chartDoc = parseDocument(chartFileContent);
1175
- const valuesDoc = parseDocument(valuesFileContent);
1176
- // Update specific values for the chart
1177
- if (newChartVersion) {
1178
- chartDoc.set('version', newChartVersion);
1179
- }
1180
- chartDoc.set('appVersion', `v${getTerasliceVersion()}`);
1181
- valuesDoc.setIn(['image', 'nodeVersion'], `v${currentNodeVersion}`);
1182
- // Write the updated YAML back to the files
1183
- fs.writeFileSync(chartYamlPath, chartDoc.toString(), 'utf8');
1184
- fs.writeFileSync(valuesYamlPath, valuesDoc.toString(), 'utf8');
1185
- }
1186
- catch (err) {
1187
- throw new TSError(`Unable to read or write Helm chart YAML files:\n${err}`);
1188
- }
1189
- }
1190
- /**
1191
- * Writes the Opensearch internal_users.yml file with demo user credentials
1192
- * to the specified certificate directory. This file is required by Opensearch
1193
- * security configuration to define local user accounts.
1194
- *
1195
- * @param certDir - Absolute path to the directory where the file will be written
1196
- */
1197
- function createInternalUsersFile(certDir) {
1198
- const content = `# This is the internal user database
1199
- # The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh
1200
-
1201
- _meta:
1202
- type: "internalusers"
1203
- config_version: 2
1204
-
1205
- # Define your internal users here
1206
-
1207
- ## Demo users
1208
- # This hash password is passwordsufhbivbU123%$
1209
- admin:
1210
- hash: "$2y$12$Z234bambHnVJMAXiccuMluNgGhNNdOFIY6pFT2/lk3ZC.RDoDIFme"
1211
- reserved: true
1212
- backend_roles:
1213
- - "admin"
1214
- description: "Demo admin user"
1215
- anomalyadmin:
1216
- hash: "$2y$12$TRwAAJgnNo67w3rVUz4FIeLx9Dy/llB79zf9I15CKJ9vkM4ZzAd3."
1217
- reserved: false
1218
- opendistro_security_roles:
1219
- - "anomaly_full_access"
1220
- description: "Demo anomaly admin user, using internal role"
1221
- kibanaserver:
1222
- hash: "$2a$12$4AcgAt3xwOWadA5s5blL6ev39OXDNhmOesEoo33eZtrq2N0YrU3H."
1223
- reserved: true
1224
- description: "Demo OpenSearch Dashboards user"
1225
- kibanaro:
1226
- hash: "$2a$12$JJSXNfTowz7Uu5ttXfeYpeYE0arACvcwlPBStB1F.MI7f0U9Z4DGC"
1227
- reserved: false
1228
- backend_roles:
1229
- - "kibanauser"
1230
- - "readall"
1231
- attributes:
1232
- attribute1: "value1"
1233
- attribute2: "value2"
1234
- attribute3: "value3"
1235
- description: "Demo OpenSearch Dashboards read only user, using external role mapping"
1236
- logstash:
1237
- hash: "$2a$12$u1ShR4l4uBS3Uv59Pa2y5.1uQuZBrZtmNfqB3iM/.jL0XoV9sghS2"
1238
- reserved: false
1239
- backend_roles:
1240
- - "logstash"
1241
- description: "Demo logstash user, using external role mapping"
1242
- readall:
1243
- hash: "$2a$12$ae4ycwzwvLtZxwZ82RmiEunBbIPiAmGZduBAjKN0TXdwQFtCwARz2"
1244
- reserved: false
1245
- backend_roles:
1246
- - "readall"
1247
- description: "Demo readall user, using external role mapping"
1248
- snapshotrestore:
1249
- hash: "$2y$12$DpwmetHKwgYnorbgdvORCenv4NAK8cPUg8AI6pxLCuWf/ALc0.v7W"
1250
- reserved: false
1251
- backend_roles:
1252
- - "snapshotrestore"
1253
- description: "Demo snapshotrestore user, using external role mapping"
1254
- `;
1255
- fs.writeFileSync(path.join(certDir, 'internal_users.yml'), content, 'utf8');
1256
- }
1257
- /**
1258
- * Generates TLS certs using mkcert and organizes the output files
1259
- * into the layout expected by each service format.
1260
- *
1261
- * mkcert creates a locally-trusted CA and issues certificates signed by it.
1262
- * This function handles the full lifecycle:
1263
- * 1. Runs mkcert to produce a key + certificate for the given DNS names
1264
- * 2. Copies the root CA certificate into a CAs/ subdirectory
1265
- * 3. Renames/copies files into the structure each service expects
1266
- * 4. Cleans up the raw mkcert output files
1267
- *
1268
- * Supported formats:
1269
- * - `minio` → private.key, public.crt
1270
- * - `opensearch` → opensearch-key.pem, opensearch-cert.pem, internal_users.yml
1271
- * - `kafka` → kafka-keypair.pem (key + cert concatenated)
1272
- *
1273
- * @param formats - List of service formats to produce (e.g. ['minio', 'opensearch'])
1274
- * @param dirPath - Absolute path to the output directory (recreated if it already exists)
1275
- * @param dnsNames - DNS names / IPs the certificate should be valid for
1276
- * @throws {Error} If mkcert is not installed, dirPath is relative, or cert files cannot be found
1277
- */
1278
- async function generateCerts(formats, dirPath, dnsNames) {
1279
- // mkcert must be installed
1280
- try {
1281
- await execa('mkcert', ['--version']);
1282
- }
1283
- catch {
1284
- throw new Error('mkcert is not installed. Please install mkcert and try again.');
1285
- }
1286
- if (!path.isAbsolute(dirPath)) {
1287
- throw new Error('dirPath must be an absolute path');
1288
- }
1289
- if (dnsNames.length === 0) {
1290
- throw new Error('At least one DNS name is required');
1291
- }
1292
- // Always start with a clean dir in the case there is stuff here
1293
- if (fse.existsSync(dirPath)) {
1294
- await fse.remove(dirPath);
1295
- }
1296
- await fse.mkdirp(dirPath);
1297
- // Run mkcert in the output directory — it writes the key and cert files there
1298
- await execa('mkcert', ['--client', ...dnsNames], { cwd: dirPath });
1299
- // mkcert stores its root CA in a system dir. copy it into our cert dir
1300
- const { stdout: caRoot } = await execa('mkcert', ['-CAROOT']);
1301
- fse.copySync(path.join(caRoot.trim(), 'rootCA.pem'), path.join(dirPath, 'rootCA.pem'));
1302
- // Move the root CA into a CAs/ subdirectory. Keeps it organized
1303
- await fse.mkdirp(path.join(dirPath, 'CAs'));
1304
- fse.moveSync(path.join(dirPath, 'rootCA.pem'), path.join(dirPath, 'CAs', 'rootCA.pem'));
1305
- // mkcert names its output files based on the DNS names, so we locate them by pattern
1306
- const files = fs.readdirSync(dirPath);
1307
- const privateKeyName = files.find((f) => f.toLowerCase().includes('key.pem'));
1308
- const publicCertName = files.find((f) => f.toLowerCase().endsWith('.pem') && !f.toLowerCase().includes('key.pem'));
1309
- if (!privateKeyName || !publicCertName) {
1310
- throw new Error(`Could not locate key.pem or public cert in ${dirPath}`);
1311
- }
1312
- const privateKeyPath = path.join(dirPath, privateKeyName);
1313
- const publicCertPath = path.join(dirPath, publicCertName);
1314
- // Copy/rename files into the layout each service expects. Each service like it in
1315
- // a specific format.
1316
- for (const format of formats) {
1317
- switch (format) {
1318
- case 'minio':
1319
- // https://min.io/docs/minio/linux/operations/network-encryption.html
1320
- fse.copySync(privateKeyPath, path.join(dirPath, 'private.key'));
1321
- fse.copySync(publicCertPath, path.join(dirPath, 'public.crt'));
1322
- break;
1323
- case 'opensearch':
1324
- // https://opensearch.org/docs/latest/security/configuration/tls/#x509-pem-certificates-and-pkcs-8-keys
1325
- fse.copySync(privateKeyPath, path.join(dirPath, 'opensearch-key.pem'));
1326
- fse.copySync(publicCertPath, path.join(dirPath, 'opensearch-cert.pem'));
1327
- // Opensearch also needs a user database file alongside the certs
1328
- createInternalUsersFile(dirPath);
1329
- break;
1330
- case 'kafka': {
1331
- // https://kafka.apache.org/42/security/encryption-and-authentication-using-ssl/
1332
- // Kafka expects key and cert in a single PEM file
1333
- const keyContent = fs.readFileSync(privateKeyPath, 'utf8');
1334
- const certContent = fs.readFileSync(publicCertPath, 'utf8');
1335
- fs.writeFileSync(path.join(dirPath, 'kafka-keypair.pem'), keyContent + certContent, 'utf8');
1336
- break;
1337
- }
1338
- default:
1339
- signale.warn(`Unknown format '${format}' ignored.`);
1340
- }
1341
- }
1342
- // Remove the original mkcert files now that we've copied and renamed them
1343
- if (formats.length > 0) {
1344
- fs.unlinkSync(privateKeyPath);
1345
- fs.unlinkSync(publicCertPath);
1346
- }
1347
- // Make sure all cert files are readable
1348
- await execa('chmod', ['-R', 'a+rX', dirPath]);
1349
- }
1350
- /**
1351
- * Generates CA certificates for encrypted services in the test environment if needed
1352
- *
1353
- * @throws {Error} If certificate generation fails.
1354
- */
1355
- export async function generateTestCaCerts() {
1356
- const encryptedServices = [];
1357
- const hostNames = ['localhost'];
1358
- if (config.ENCRYPT_OPENSEARCH) {
1359
- encryptedServices.push('opensearch');
1360
- hostNames.push('opensearch2.services-dev1', 'opensearch3.services-dev1', 'opensearch', config.OPENSEARCH_HOSTNAME);
1361
- }
1362
- if (config.ENCRYPT_MINIO) {
1363
- encryptedServices.push('minio');
1364
- hostNames.push('minio.services-dev1', 'minio', config.MINIO_HOSTNAME);
1365
- }
1366
- if (config.ENCRYPT_KAFKA) {
1367
- encryptedServices.push('kafka');
1368
- hostNames.push('kafka-headless.services-dev1.svc.cluster.local', 'kafka-headless.services-dev1', 'kafka-headless', 'kafka', config.KAFKA_HOSTNAME);
1369
- }
1370
- if (encryptedServices.length > 0) {
1371
- // Formats the encrypted service list to print with the user feedback
1372
- const serviceList = encryptedServices.length === 1
1373
- ? encryptedServices[0]
1374
- : encryptedServices.length === 2
1375
- ? encryptedServices.join(' and ')
1376
- : `${encryptedServices.slice(0, -1).join(', ')} and ${encryptedServices[encryptedServices.length - 1]}`;
1377
- try {
1378
- signale.pending(`Generating new ca-certificates for ${serviceList}...`);
1379
- signale.debug('Generating certs: ', { formats: encryptedServices, dirPath: config.CERT_PATH, dnsNames: hostNames });
1380
- await generateCerts(encryptedServices, config.CERT_PATH, hostNames);
1381
- signale.success(`Created certs in ${config.CERT_PATH}`);
1382
- }
1383
- catch (err) {
1384
- throw new Error(`Error generating ca-certificates for ${serviceList}: ${err.message}`);
1385
- }
1386
- }
1387
- }
1388
- export async function getConfigValueFromCustomYaml(configFilePath, valuePath) {
1389
- const customConfig = load(fs.readFileSync(configFilePath, 'utf8'));
1390
- const value = get(customConfig, valuePath, undefined);
1391
- return value;
1392
- }
1393
- export async function setConfigValuesForCustomYaml(configFilePath, valuePath, valueToSet) {
1394
- try {
1395
- const customConfig = parseDocument(fs.readFileSync(configFilePath, 'utf8'));
1396
- const splitPath = valuePath.split('.');
1397
- customConfig.setIn(splitPath, valueToSet);
1398
- fs.writeFileSync(configFilePath, customConfig.toString(), 'utf8');
1399
- }
1400
- catch (err) {
1401
- throw new Error(`Failed to set ${valuePath} to ${valueToSet} in config file ${configFilePath}. Reason: ${err.message}`);
1402
- }
1403
- }
1404
- //# sourceMappingURL=scripts.js.map