@soft-artel/ci 2.0.59 → 2.0.62

package/src/commands/k8s-build.ts

@@ -1,447 +0,0 @@
-#!/usr/bin/env node
-import '../utils/prototype';
-
-import {program, OptionValues} from 'commander';
-import { promises as fsAsync } from 'fs';
-import Mustache from 'mustache';
-import yaml from 'js-yaml';
-
-import Shell from '../utils/Shell';
-import { Jira } from '../services/Jira';
-import Git from '../services/Git';
-import { Config, Project, RunCommandHandler } from '../Project';
-import { Reporter } from '../services/Reporter';
-import { log } from '../utils/Logger';
-import { md5, resolvePath } from '../utils/helpers';
-import { Exception } from '../utils/Exception';
-
-const ENV = process.env;
-
-// ============================================
-// Comand flow
-
-const runHandler: RunCommandHandler = async ( prj: Project, reporter: Reporter, opts: OptionValues, config: Config ) => {
-
-	reporter.$message_footer_changes_from = prj.version;
-	reporter.$message_footer_changes_to   = ENV.CI_COMMIT_REF_NAME || 'dev';
-
-	const release = opts.changelog ? await Jira.resolve(prj) : { tasks:[], bugs:[], changelog:''};
-
-	// --build: Build and push docker image to registry
-	Shell.cd( prj.rootPath );
-
-	// -----------------------------
-	// 1. Start
-	prj.incrementBuild();
-
-	const pkg = JSON.parse(await Shell.cat(prj.rootPath +'/package.json') || 'no package.json file!');
-	pkg.version = prj.version;
-	await fsAsync.writeFile(prj.rootPath +'/package.json', JSON.stringify(pkg, null, 3), { encoding: 'utf8', flag: 'w+'});
-
-	await reporter.startBuild();
-	log.info(`1. build ${ prj.stage }`);
-
-	// -----------------------------
-	// 2. Chek updated apps
-	await reporter.append(` + Load apps and check updated`);
-	log.info(`2. Load apps and check updated`);
-
-	const updateFiles =  await Git.getDiffFiles();
-	log.info(`\n\nUPDATED FILES:\n$`, updateFiles);
-
-	const { apps, updated, images } = await getAppsAndImages(prj, config, updateFiles);
-
-	if(!updated){
-		log.info('SKIP BUILD: NO Apps to update!');
-		await reporter.skip(`SKIP - no apps to update\n ${ updateFiles.join('\n    ')}`);
-		process.exit(0);
-	}
-	
-	// -----------------------------
-	// 4. write k8s manifests!
-	await reporter.append(` + Write K8s manifests`);
-	log.info(`3. Write K8s manifests`);
-	
-	let imagesToBuild: Record<string, DockerImage> = {}
-
-	for (const appName of updated) {
-		const app = apps[ appName ]
-		const image = images[ app.dockerImageKey ]
-
-		if( !image ){
-			throw new Exception('No dockerfile found for app: '+ app.name, { app, images })
-		}
-
-		await writeManifest(prj, app, image);
-		imagesToBuild[ image.key ] = image
-	}
-
-	// -----------------------------
-	// 5. Build images
-	await Shell.execRepeat(`docker login -u ${ ENV.CI_REGISTRY_USER } -p ${ ENV.CI_REGISTRY_PASSWORD } ${ ENV.CI_REGISTRY }`);
-
-	for (const image of Object.values( imagesToBuild )) {
-
-		// 5.1. distrib
-		log.info(`${ image.tag } distrib`);		
-		await reporter.append(` + ${ image.tag } distrib`);
-		const distribCommand = image.distribRun || 'npm run distrib'
-		log.info( distribCommand )
-		await Shell.exec( distribCommand );
-
-		// 5.2. build
-		log.info(`${ image.tag } build docker image`);		
-		await reporter.append(` + ${ image.tag } build docker image`);
-		Shell.cd(image.path);
-		await Shell.exec(`docker build --platform amd64 -t ${ image.fullTag } -f ${ image.path }/dockerfile ${ image.path }/.`);
-
-		// 5.3 push
-		log.dbg(`${ image.tag } Docker push`);
-		await reporter.append(` + ${ image.tag } push image to registry`);
-		await Shell.execRepeat(`docker login -u ${ ENV.CI_REGISTRY_USER } -p ${ ENV.CI_REGISTRY_PASSWORD } ${ ENV.CI_REGISTRY }`);
-		await Shell.execRepeat(`docker push ${ image.fullTag }`);
-		log.dbg(`Clear local docker image: ${ image.tag }`);
-		await Shell.exec(`docker rmi -f  ${ image.fullTag }`);
-
-	}
-
-	// -----------------------------
-	// 5. Save new build to GitLab CI Variable
-
-	await reporter.append(` + Save current build: ${ prj.build }`);
-	log.dbg(`7. Put current build:${ prj.build } to GitLab`);
-	await prj.saveGitLabBuild();
-
-	// -----------------------
-
-	
-	if(opts.changelog){
-		await prj.updateChangeLog(release.changelog);
-		await reporter.append(` + Update <a href=\"${ prj.url }/-/blob/${ ENV.CI_COMMIT_REF_NAME || 'dev' }/CHANGELOG.md\">changelog</a>.`);
-	}
-
-	await Git.setOrigin();
-	await Shell.exec(`git add -A`, { ignoreError: true, silent: false });
-	await Shell.exec(`git commit -a -m "v${ prj.version }"`, { ignoreError: true, silent: false });
-
-	try{
-		await Git.push(ENV.CI_COMMIT_REF_NAME || 'dev');
-	}catch(e){
-		log.error(e);
-		await Shell.exec(`git checkout --ours .`, { ignoreError: true, silent: false });
-		await Shell.exec(`git add -u`, { ignoreError: true, silent: false });
-		await Shell.exec(`git commit -a -m "v${ prj.version } resolved"`, { ignoreError: true, silent: false });
-		await Git.push(ENV.CI_COMMIT_REF_NAME || 'dev');
-	}
-
-
-	if(opts.gittag){
-		await reporter.append(` + Git tag new version`);
-		await Git.makeTag(prj.version);
-	}
-
-	await reporter.succses();
-}
-
-
-// ========================================================
-// Command
-
-async function main() {
-
-	program
-		.option('--path <project dir>',  'Path to project directory [default - current]')
-
-		.option('--no-prebuild', 'Skip prebuild phase')
-
-		.option('--no-changelog', 'Skip create gitlog')
-		.option('--no-gittag', 'Skip create and push new git tag')
-		.action( await Project.run(program.opts() || {}, runHandler) );
-
-	await program.parseAsync(process.argv);
-}
-main();
-
-
-// ================================================================================================
-// Clasess
-
-class App{
-
-	group = '';
-	name = '';
-	app  = '';
-	path = '';
-
-	$pathComp: string[] = [ './' ];
-
-	dockerImageKey = '' 
-
-	// ----------
-
-	constructor(path: string = ''){
-		let pathComp = path.split('/');
-
-		log.dbg( 'App try load by path:'+ path, pathComp )
-
-		if( pathComp.length > 1 && pathComp[0] === 'src' ){
-			// Remove src folder reference
-			pathComp = pathComp.splice(1)
-		}
-
-		if( pathComp.length > 1 && pathComp[1] === 'src' ){
-			// Remove src folder reference
-			pathComp = pathComp.splice(0,-1)
-		}
-
-		if(pathComp.length > 1 && pathComp[1].length > 1 ){
-		  
-			this.group = pathComp[0];
-		  this.app  = pathComp[1];
-		  this.name = `${this.group}-${this.app}`;
-		  this.path = path;
-
-			this.$pathComp.push(`./${ this.group }/`);
-			this.$pathComp.push(`./${ this.path }/`);
-
-		}else{
-			this.app  = pathComp.length == 1 ? pathComp[0] : path;
-			this.name = this.app;
-			this.group = ''
-		  this.path = '/'+path;
-			this.$pathComp = [ this.path ]
-		}
-
-		log.dbg( this.name, this )
-	}
-
-}
-
-// ------------------------------------------------------------------
-
-
-export class DockerImage{
-
-	path = '';
-	key = '';
-
-	srcImage = '';
-
-	tag = '';
-	fullTag = '';
-
-	distribRun?: string;
-
-	// -----------
-	constructor( path: string, appName:string, ver: string, build: number, dockerfile?: string ){
-		if( !dockerfile ){
-			throw new Error('Wrong dockerfile: Empty! by path:'+path)
-		}
-
-		this.path = path
-		this.key = md5( dockerfile )
-
-		const dockerImage = /FROM (.+)/.exec(dockerfile);
-		if(!dockerImage || dockerImage.length < 2){
-			throw new Error(`Wrong dockerfile format:\n${dockerfile}`);
-		}
-
-		this.srcImage = dockerImage[1];
-
-		const dockerName = /#[ ]?NAME (.+)/.exec(dockerfile);
-		const name = dockerName && dockerName.length > 1 ? dockerName[1] : appName || this.key;
-
-		// api-v1.2:#321
-		this.tag  = `${name}-v${ver}:${build}`;
-		this.fullTag = `${ ENV.CI_REGISTRY_IMAGE }/${ this.tag }`;
-
-		const action = /#[ ]?DISTRIB RUN (.+)/.exec(dockerfile);
-		this.distribRun = action && action.length > 1 ? action[1] : undefined;
-	}
-}
-
-
-// ================================================================================================
-// Helpers:
-
-export async function getAppsAndImages(prj: Project, config: Config, updatedPaths: string[], maxdepth: number = 4, ignore: string[] = ['*/_*', '*/\.*', '*/node_modules*']): Promise<{ apps: Record<string, App>; updated: string[] | undefined; images: Record<string, DockerImage> }>{
-
-	Shell.cd( config.appsPath || prj.rootPath );
-
-	const appFile = config.appFile || 'app_manifest_tpl.yaml'
-
-	const appsPaths = await Shell.find( config.appsPath || prj.rootPath, appFile, maxdepth, ignore);
-
-	log.dbg( 'Apps paths by appFile:'+ appFile, appsPaths )
-
-	const ignorePaths = [ ...config.ignorePaths || [], ...config.sharedPaths || [] ];
-
-	const apps: Record<string, App> = {};
-	const watchPaths: Record<string, string> = {};
-
-	const dockerImages: Record<string, DockerImage> = {}
-
-	// Load rootDocker Image
-	let rootDockerFileKey = ''
-	const dockerfileRootPath = resolvePath( prj.rootPath )+'/dockerfile'
-	if( await Shell.test('-e', dockerfileRootPath ) ){
-		const image = new DockerImage( prj.rootPath, prj.name, prj.shortVersion, prj.build, await Shell.cat( dockerfileRootPath ) );
-		dockerImages[ image.key ] = image
-		rootDockerFileKey= image.key
-		log.dbg( 'Dokerfile at root path - found! '+ image.key )
-	}
-	else{
-		log.info( 'Dokerfile at root path not found! path:'+ dockerfileRootPath )
-	}
-
-
-	for(const file of appsPaths.sort()) {
-		const path = file.replace(prj.rootPath+'/', '').replace('/src', '').replace( '/'+appFile, '');
-
-		let skip = false;
-		for(const i_path of ignorePaths){
-			if(path.includes(i_path)){
-				skip = true;
-				break;
-			}
-		}
-
-		if(skip){
-			continue;
-		}
-
-		const app = new App(path);
-
-		if( app.group === '' ){
-			app.group = prj.name
-		}
-
-		if(app.name === '' ){
-			app.name = prj.name;
-			app.app  = prj.name;
-			app.group = '';
-			app.path = '';
-		}
-
-		const dockerfileAppPath = resolvePath( prj.rootPath + '/' + app.path )+'/dockerfile'
-		if(  app.path !== '' && await Shell.test('-e', dockerfileAppPath ) ){
-			log.info( 'Dokerfile at app "'+ app.name +'" found! path:'+ dockerfileAppPath )
-			const image = new DockerImage( app.path, app.name, prj.shortVersion, prj.build, await Shell.cat( dockerfileAppPath ) );
-			dockerImages[ image.key ] = image
-			app.dockerImageKey = image.key
-
-		}
-		else{
-			log.dbg( 'App dokerfile not found at:"'+ dockerfileAppPath +'", and set as default:'+ rootDockerFileKey )
-			app.dockerImageKey = rootDockerFileKey
-		}
-
-		apps[ app.name ] = app;
-		watchPaths[ app.path ] = app.name;
-	}
-
-	log.dbg(`Find apps by ${ appFile }`, Object.keys( apps ).sort());
-
-	if( Object.keys( dockerImages ).length === 0 ){
-		throw new Exception('No any dockerfile found in project! Pleace collect it on root or app path')
-	}
-
-	const sharedPaths = config.sharedPaths || [];
-
-	sharedPaths.push('package.json');
-	sharedPaths.push('app_manifest_spec.json');
-	sharedPaths.push('app_manifest_tpl.yaml');
-	sharedPaths.push('dockerfile');
-
-	const watchPathsArr = Object.keys(watchPaths);
-
-	let appsChanged: Record<string, App> = {};
-
-	for (const file of updatedPaths) {
-
-		let allApps = false;
-		for(const i_path of sharedPaths){
-			if(file.includes(i_path)){
-				allApps = true;
-				break;
-			}
-		}
-		if(allApps){
-			appsChanged = apps;
-			break;
-		}
-        
-		for(const testPath of watchPathsArr){
-			const appName = watchPaths[ testPath ];
-			if((RegExp(testPath).exec(file)) && apps[ appName ] && appsChanged[ appName ] === undefined){
-				appsChanged[ appName ] = apps[ appName ];
-			}
-		}
-	}
-
-	const changed = Object.keys(appsChanged);
-	return { apps, updated: changed.length > 0 ? changed : undefined, images: dockerImages };
-
-}
-
-
-// ------------------------------------------------------------------
-
-
-async function writeManifest(prj: Project, app: App, image: DockerImage){
-
-	const k8s_tpl_fileName = 'app_manifest_tpl.yaml';
-	const k8s_spec_fileName = 'app_manifest_spec.json';
-
-	// 1. Находим шаблон манифеста для app и специфицации ресурсов
-
-	const appFullPath = resolvePath( prj.rootPath + '/' + app.path + '/' )
-
-
-	let k8s_spec: Record<string, any> = {};
-
-	const k8s_tpl = await Shell.cat( appFullPath + k8s_tpl_fileName, {silent: !log.isDebug, ignoreError:true });
-	const k8s_specFile = await Shell.cat( appFullPath + k8s_spec_fileName, {silent: !log.isDebug, ignoreError:true });
-
-	if(k8s_specFile && k8s_specFile !== ''){
-		try {
-			k8s_spec = JSON.parse( k8s_specFile ) as Record<string, any>;
-		} catch(e){
-			log.error(e);
-		}
-	}
-
-
-	if(!k8s_tpl || k8s_tpl.length < 10){
-		throw new Exception(`No K8s template [${k8s_tpl_fileName}] found for app:${ app.name }`, { appFullPath });
-	}
-
-	// 4.2. Ходим по стейджам, формируем и сохраняем манифесты
-
-	for (const stage of Object.keys( prj.$stagesState )) {
-
-		const k8s_data: Record<string, any> = {};
-
-		k8s_data.stage = stage;
-		k8s_data.app  = app;
-		k8s_data.image = image;
-		k8s_data.prj  = prj;
-		k8s_data.spec = k8s_spec[ stage ] || {};
-
-		if(!k8s_data.spec.replicas){
-			k8s_data.spec.replicas = 1;
-		}
-
-		log.dbg(`${app.name} k8s specs`, k8s_spec);
-		// k8s_data.spec.volumeMounts = k8s_data.spec.volumes;
-		const manifest = Mustache.render(k8s_tpl, k8s_data);
-
-		yaml.loadAll(manifest); // Test yaml
-
-		await Shell.mkdir(`${ prj.rootPath}/_k8s/${stage}`, {ignoreError: false});
-		await fsAsync.writeFile(`${ prj.rootPath}/_k8s/${stage}/${app.name}.yml`, manifest);
-
-		log.dbg(`[${stage}] ${app.name} writed -> ${ prj.rootPath}/_k8s/${stage}/${app.name}.yml `);
-
-	}
-}

package/src/commands/k8s-deploy.ts

@@ -1,215 +0,0 @@
-#!/usr/bin/env node
-import '../utils/prototype';
-
-import {program, OptionValues} from 'commander';
-
-import Shell, { ExecOptions } from '../utils/Shell';
-import { Jira, ReleaseInfo } from '../services/Jira';
-import Git from '../services/Git';
-import { Config, Project } from '../Project';
-import { Reporter } from '../services/Reporter';
-import { log } from '../utils/Logger';
-import { RunCommandHandler } from '../Project';
-import { Exception } from '../utils/Exception';
-import { resolvePath } from '../utils/helpers';
-
-const ENV = process.env;
-
-
-// ============================================
-// Command flow
-
-const runHandler: RunCommandHandler = async ( prj: Project, reporter: Reporter, opts: OptionValues, config: Config ) => {
-
-	// ------------------
-	// Инициализируемся
-
-	reporter.$message_footer_changes_from = prj.prevVersion || prj.version;
-	reporter.$message_footer_changes_to   = prj.version;
-
-	const release: ReleaseInfo = opts.changelog ? await Jira.resolve(prj) : { commits:[], components:[], tasks:[], bugs:[], changelog:''};
-
-	// 1. Меняем стейдж на новый, потому что деплоим в него!
-	const stage = opts.stage;
-
-	const configKey = `K8S_${ stage.toUpperCase() }_CONFIG`;
-	let   configFile = ENV[ configKey ];
-
-	if(!configFile){
-		throw new Exception(`CI VARIABLE K8s config file not found: ${ configKey }`);
-	}
-
-	configFile = resolvePath(configFile) ;
-
-
-	prj.stage = stage;
-	await reporter.startDeploy(stage);
-
-	// предполагаем для для каждого стейджа у нас отдельныая k8s нода!
-	// namespace = default для всех ресурсов
-
-	// 2. Получаем путь
-	const k8s_dir = `${ prj.rootPath }/_k8s/${stage}/`;
-	Shell.cd(k8s_dir);
-
-	// 3. Обходим ВСЕ файлы - те что не поменялись не применятся
-	const deployedApps: Record<string, string> = {};
-	const k8s_files = await Shell.ls(k8s_dir);
-
-	try{
-		
-		// Деплоим все(!) аппы - те что не поменялись не применятся
-		for (const k8s_file of k8s_files) {
-
-			const appName = k8s_file.replace('.yml', '');
-			const status = await k8sApply( configFile, `${ k8s_dir }/${k8s_file}`, appName, opts.namespace );
-
-			let msg = '';
-			if(status.valid){
-				deployedApps[ appName ] = status.msg;
-				msg = ` + <b>${appName}</b>`;
-				await reporter.append(msg);
-			} else {
-				msg = ` ❌ <b>${appName}</b> yaml is invalid!\n${ status.msg }`;
-				await reporter.append(msg);
-				throw new Error(`FAIL: Deployment app: ${appName}`);
-			}
-
-		}
-
-	} catch(error){
-
-		// Деплой сфейлился - надо откатыватся
-		if(Object.keys(deployedApps).length > 0){
-
-			log.fail('FAIL: Deployments to rollout', Object.keys(deployedApps));
-			await reporter.append(` 🚫 <b>APPLAY FAIL</b>: Rollout all deployed apps`);
-	
-			for (const appName of Object.keys(deployedApps)) {
-				await k8sExec(`rollout undo deployment/${ appName }`, configFile, undefined, opts.namespace );
-			}
-	
-		}
-
-		throw error;
-	}
-
-	// deploy succsess, push changes to stage brunch
-	if(opts.gitpush){
-		await Git.setOrigin();
-		await Git.push(stage);
-	}
-
-	// await reporter.append(` + Save ${ prj.stage} -> ${ prj.version} to GitLab`);
-	await prj.saveGitLabStagesVersions();
-
-	if(opts.changelog){
-		await reporter.append(` ----------------\n+ Resolved tasks:${ release.tasks.length }, bugs: ${ release.bugs.length }`);
-
-		const bugsDeployed = await Jira.moveBugsAsDeployed(release.bugs);
-		if(bugsDeployed.length > 0){
-			await reporter.append(` + Deployed ${ bugsDeployed.length } bugs.`);
-		} else {
-			// await reporter.append(` - Deploy bugs. [SKIPED]`);
-		}
-	}
-	
-	await reporter.succses(release);
-	
-}
-
-// ========================================================
-// Command
-
-async function main() {
-
-	program
-		.option('--path <project dir>',  'Path to project directory [default - current]')
-		.option('--stage <stage>', 'K8s node stage <stage>')
-		.option('--namespace <namespace>', 'Deploy manifests to K8s specific namespace')
-
-		.option('--no-changelog', 'Skip create gitlog')
-		.option('--no-gitpush', 'Skip pushing to stage branch after deploy')
-
-		.option('--debug', 'Output extra debugging info')
-		.action( await Project.run( program.opts() || {}, runHandler ) );
-
-	await program.parseAsync(process.argv);
-}
-main();
-
-
-// ================================================================================================
-// Helpers:
-
-export async function k8sApply( configFile: string, manifestFile: string, appName: string, namespace?: string): Promise<{valid: boolean; msg: string}>{
-
-	let status: string;
-
-	try{
-		await k8sExec(`apply -f ${ manifestFile } --dry-run=server --validate`, configFile, undefined, namespace);
-		status = (await Shell.exec(`cat ${ manifestFile } | grep 'image:'`)).replace('image: ', '').replace(ENV.CI_REGISTRY_IMAGE+'/', '').trim();
-	}catch(e){
-		log.error(e);
-		log.fail('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
-		log.fail(`VALIDATE FAIL: ${appName} at ${manifestFile}`);
-		log.fail(e.message);
-		log.fail(`${ await Shell.cat(manifestFile) }`);
-		log.fail('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
-
-		return { valid:false, msg: e.message };
-	}
-
-	try{
-		status = await k8sExec(`apply -f ${ manifestFile }`, configFile, undefined, namespace);
-		await k8sExec(`rollout status deploy/${ appName } --watch=true --timeout=300s`, configFile);
-
-		log.info('+++++++++++++++++++++++++++++');
-		log.info(`DEPLOYED: ${appName}`);
-		log.info(await k8sExec(`get pods --no-headers -l app=${appName} -o wide`, configFile, { ignoreError:true, silent:true }), namespace);
-		log.info('+++++++++++++++++++++++++++++');
-
-		return { valid:true, msg: status };
-
-	} catch(error){
-		log.fail('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
-		log.fail(`DEPLOY FAIL: ${appName}`);
-		log.fail('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
-		log.fail(await k8sExec(`get pods --no-headers -l app=${appName} -o wide`, configFile, { ignoreError:true, silent:true }), namespace);
-		throw error;
-	}
-
-}
-
-// ------------------------------
-
-export async function  k8sStatus(configFile: string, stage: string, reporter: Reporter | undefined = undefined, appsOnly: boolean = true, namespace?: string): Promise<string[]>{
-
-	const cmd = `get pods -o=jsonpath='{range .items[*]}{"\\n"}{"<b>"}{.metadata.labels.app}{"</b>   "}{range .spec.containers[*]}{.image}{end}{"    "}{.metadata.labels.pod-template-hash}{" "}{.status.phase}{end}'`;
-	const statusArr = (await k8sExec(cmd, configFile, undefined, namespace)).split('\n');
-	
-	const result: string[] = [];
-	for (const row of statusArr) {
-		if(row.length < 4 || (appsOnly && (RegExp((ENV.CI_REGISTRY_IMAGE || '')).exec(row)) === null)){
-			continue;
-		}
-		result.push(row.replace(ENV.CI_REGISTRY_IMAGE+'/', ''));
-	}
-	
-	log.info(`K8S Node: ${ stage } [${ namespace || 'default' }]`, result);
-
-	if(reporter){
-		const msg = `\n<b>${ stage.toUpperCase() }</b> Pods:\n-----------------\n ` + result.join('\n-----------------\n ');
-		await reporter.append(msg, '');
-	}
-
-	return result;
-}
-
-
-// ------------------------------
-
-
-export async function  k8sExec( cmd: string, configFile: string, opt: ExecOptions | undefined = undefined, namespace?: string, cmdPrefix = ''): Promise<string>{
-	return Shell.exec(`${cmdPrefix}kubectl --kubeconfig ${ configFile } ${ namespace ? `-n ${ namespace }` : '' } ${ cmd }`, opt);
-}