@soft-artel/ci 1.3.15 → 1.3.18

package/k8s/App.ts

@@ -1,200 +0,0 @@
-import '../libs/prototype';
-
-import Mustache from 'mustache';
-import yaml from 'js-yaml';
-import { promises as asyncFs } from 'fs';
-
-import Shell from '../libs/Shell';
-import { log } from '../libs/helpers';
-import { Project } from '../libs/Project';
-import { Exception } from '../libs/Exception';
-import { DockerImage } from './DockerImage';
-
-// ================================================================================================
-
-export interface Config {
-
-	appFile: string;
-
-	sharedPaths?: string[];
-	ignorePaths?: string[];
-
-}
-
-// ================================================================================================
-
-export class App{
-
-	group = '';
-	app = '';
-
-	name = '';
-	path = '';
-
-	$pathComp: string[] = [ './' ];
-
-	// ----------
-
-	constructor(path: string = ''){
-		const pathComp = path.split('/');
-
-		if(pathComp.length > 1){
-		  
-			this.group = pathComp[0];
-		  this.app  = pathComp[1];
-		  this.name = `${this.group}-${this.app}`;
-		  this.path = path;
-
-			this.$pathComp.push(`./${ this.group }/`);
-			this.$pathComp.push(`./${ this.path }/`);
-
-		}else{
-			this.app  = path;
-			this.name = path;
-		  this.path = '';
-		}
-	}
-
-	// ----------
-	
-	async writeManifest(prj: Project, image: DockerImage){
-
-		const k8s_tpl_fileName = 'k8s_tpl.yaml';
-		const k8s_spec_fileName = 'k8s_spec.json';
-
-		// 1. Находим шаблон манифеста для app и специфицации ресурсов
-
-		let k8s_tpl  = '';
-		const k8s_spec: Record<string, any> = {};
-
-		for (const path of this.$pathComp) {
-
-			const k8s_tplFile = await Shell.cat(path + k8s_tpl_fileName, {silent: log.debug, ignoreError:true });
-			k8s_tpl =  k8s_tplFile && k8s_tplFile !== '' ? k8s_tplFile : k8s_tpl;
-
-			const k8s_specFile = await Shell.cat(path + k8s_spec_fileName, {silent: log.debug, ignoreError:true });
-			if(k8s_specFile && k8s_specFile !== ''){
-				try {
-					const sObj = JSON.parse(k8s_specFile) as Record<string, any>;
-					k8s_spec.mergeFrom(sObj, true);
-				} catch(e){
-					log.error(e);
-				}
-			}
-		}
-
-		if(k8s_tpl.length < 10){
-			throw new Exception(`No K8s template [${k8s_tpl_fileName}] found for app:${ this.name }`, this.$pathComp);
-		}
-
-		log.dbg(`${this.name}  k8s specs`, k8s_spec);
-
-		// 4.2. Ходим по стейджам, формируем и сохраняем манифесты
-
-		for (const stage of prj.$stages) {
-
-			const k8s_data: Record<string, any> = {};
-
-			k8s_data.stage = stage;
-			k8s_data.app  = this;
-			k8s_data.image = image;
-			k8s_data.prj  = prj;
-			k8s_data.spec = k8s_spec[ stage ] || {};
-
-			if(!k8s_data.spec.replicas){
-				k8s_data.spec.replicas = 1;
-			}
-
-			// k8s_data.spec.volumeMounts = k8s_data.spec.volumes;
-			const manifest = Mustache.render(k8s_tpl, k8s_data);
-
-			yaml.loadAll(manifest); // Test yaml
-
-			await Shell.mkdir(`${ prj.rootPath}/_k8s/${stage}`, {ignoreError: false});
-			await asyncFs.writeFile(`${ prj.rootPath}/_k8s/${stage}/${this.name}.yml`, manifest);
-
-			log.dbg(`[${stage}] ${this.name} writed -> ${ prj.rootPath}/_k8s/${stage}/${this.name}.yml `);
-
-		}
-	}
-
-}
-
-// ================================================================================================
-
-export async function getApps(prj: Project, config: Config, updatedPaths: string[], maxdepth: number = 3, ignore: string[] = ['*/_*', '*/\.*', '*/node_modules*']): Promise<{ apps: Record<string, App>; updated: string[] | undefined }>{
-
-	Shell.cd(prj.rootPath);
-	const appsPaths = await Shell.find(prj.rootPath, config.appFile, maxdepth, ignore);
-
-	const ignorePaths = [ ...config.ignorePaths || [], ...config.sharedPaths || [] ];
-
-	const apps: Record<string, App> = {};
-	const watchPaths: Record<string, string> = {};
-
-	for(const file of appsPaths.sort()) {
-		const path = file.replace(prj.rootPath+'/', '').replace('/' + config.appFile, '');
-
-		let skip = false;
-		for(const i_path of ignorePaths){
-			if(path.includes(i_path)){
-				skip = true;
-				break;
-			}
-		}
-		if(path === '' || skip){
-			continue;
-		}
-
-		const app = new App(path);
-
-		if(app.name === config.appFile){
-			app.name = prj.name;
-			app.app  = prj.name;
-			app.group = '';
-			app.path = '';
-		}
-
-		apps[ app.name ] = app;
-		watchPaths[ app.path ] = app.name;
-	}
-
-	log.dbg(`Find apps by ${config.appFile}`, Object.keys(apps).sort());
-
-	const sharedPaths = config.sharedPaths || [];
-
-	sharedPaths.push('package.json');
-	sharedPaths.push('k8s_spec.json');
-	sharedPaths.push('k8s_tpl.yaml');
-	sharedPaths.push('dockerfile');
-
-	const watchPathsArr = Object.keys(watchPaths);
-
-	let appsChanged: Record<string, App> = {};
-
-	for (const file of updatedPaths) {
-
-		let allApps = false;
-		for(const i_path of sharedPaths){
-			if(file.includes(i_path)){
-				allApps = true;
-				break;
-			}
-		}
-		if(allApps){
-			appsChanged = apps;
-			break;
-		}
-        
-		for(const testPath of watchPathsArr){
-			const appName = watchPaths[ testPath ];
-			if((RegExp(testPath).exec(file)) && apps[ appName ] && appsChanged[ appName ] === undefined){
-				appsChanged[ appName ] = apps[ appName ];
-			}
-		}
-	}
-
-	const changed = Object.keys(appsChanged);
-	return { apps, updated: changed.length > 0 ? changed : undefined };
-
-}

package/k8s/DockerImage.ts

@@ -1,147 +0,0 @@
-import fs from 'fs';
-
-import { log, md5, resolvePath } from '../libs/helpers';
-import Shell from '../libs/Shell';
-
-const ENV = process.env;
-
-// ========================================================
-
-type DockerImagePrebuildAction = (img: DockerImage) => void;
-const PREBUILD: Record<string, DockerImagePrebuildAction> = {};
-
-export class DockerImage{
-
-	path = '';
-	key = '';
-
-	srcImage = '';
-
-	tag = '';
-	fullTag = '';
-
-	$prebuild_typed?: DockerImagePrebuildAction = undefined;
-
-	// -----------
-
-	constructor(path: string,  ver: string, build: number){
-		this.path  = resolvePath(path);
-
-		const dockerfile = fs.readFileSync(this.path + '/dockerfile', 'utf8');
-		this.key   = md5(dockerfile);
-
-		const dockerImage = /FROM (.+)/.exec(dockerfile);
-		if(!dockerImage || dockerImage.length < 2){
-			throw new Error(`Wrong dockerfile format:\n${dockerfile}`);
-		}
-
-		this.srcImage = dockerImage[1];
-
-		const dockerName = /#[ ]?IMAGE NAME (.+)/.exec(dockerfile);
-		const name = dockerName && dockerName.length > 1 ? dockerName[1] : ENV.CI_PROJECT_NAME || this.key;
-
-		// api-v1.2:#321
-		this.tag  = `${name}-v${ver}:${build}`;
-		this.fullTag = `${ ENV.CI_REGISTRY_IMAGE }/${ this.tag }`;
-
-		const dockerType = /#[ ]?IMAGE TYPE (.+)/.exec(dockerfile);
-		const type = dockerType && dockerType.length > 1 ? dockerType[1].toLowerCase() : this.srcImage.split(':')[0].toLowerCase();
-
-		this.$prebuild_typed = PREBUILD[ type ];
-	}
-
-	// -----------
-
-	async prebuild(){
-		if(!this.$prebuild_typed){
-			throw new Error(`No prebuild action for ${ this.tag }`);
-		}
-		await this.$prebuild_typed(this);
-	}
-	
-	async build(){
-		Shell.cd(this.path);
-		log.info(`Build docker image: ${ this.tag }`);
-
-		await Shell.exec(`docker build --platform amd64 -t ${ this.fullTag } -f ${ this.path }/dockerfile ${ this.path }/.`);
-	}
-
-	async push(){
-		await Shell.execRepeat(`docker login -u ${ ENV.CI_REGISTRY_USER } -p ${ ENV.CI_REGISTRY_PASSWORD } ${ ENV.CI_REGISTRY }`);
-
-		log.info(`Push docker image: ${ this.tag } to registry`);
-		await Shell.execRepeat(`docker push ${ this.fullTag }`);
-	
-		log.dbg(`Clear local docker image: ${ this.tag }`);
-		await Shell.exec(`docker rmi -f  ${ this.fullTag }`);
-	}
-}
-
-// ========================================================
-
-PREBUILD.nginx = async (img: DockerImage) => {
-	Shell.cd(img.path);
-
-	const mounts = { '/etc/nginx':`${ img.path }/nginx` };
-
-	log.info(`Test nginx configs`);
-	await run(img.srcImage, 'nginx -t', mounts);
-	await run('yandex/gixy', 'yandex/gixy /etc/nginx/nginx.conf', mounts);
-
-	log.info(`Nginx config file:`)
-	log.info( await Shell.cat(`${ img.path }/nginx/nginx.conf`) )
-
-};
-
-// ------------
-
-PREBUILD.gateway = async (img: DockerImage) => {
-	Shell.cd(img.path);
-
-	const mounts = { '/etc/nginx':`${ img.path }/nginx`, '/etc/certs':`${ img.path }/certs` };
-
-	log.info(`Test gateway nginx configs`);
-	await run(img.srcImage, 'nginx -t', mounts);
-	await run('yandex/gixy', 'yandex/gixy /etc/nginx/nginx.conf', mounts);
-
-	log.info(`Nginx config file:`);
-	log.info(await Shell.cat(`${ img.path }/nginx/nginx.conf`))
-
-};
-
-// ------------
-
-PREBUILD.spa = async (img: DockerImage) => {
-	Shell.cd(img.path);
-	await PREBUILD.nginx(img);
-	log.info(`Test and Distrib WEB SPA sources`);
-
-	// ? надо ли собирать веб внутри контейнера?!
-	// const mounts = { '/etc/nginx':`${ img.path }/nginx` }
-	// await run( img.srcImage, 'nginx -t', mounts );
-
-	await Shell.exec(`npm run distrib`);
-};
-
-// ------------
-
-PREBUILD.node = async (img: DockerImage) => {
-	Shell.cd(img.path);
-	log.info(`Test and Distrib Node sources`);
-	await Shell.exec(`npm run distrib`);
-};
-
-// ========================================================
-
-async function run(srcImage: string, cmd: string, mnt?: Record<string, string>){
-	let extCmd = `docker run --rm`;
-		
-	for (const path in mnt) {
-		if(mnt.hasOwnProperty(path)) {
-			extCmd += ` -v ${mnt[path]}:${path}`;
-		}
-	}
-	
-	extCmd += ` ${ srcImage } ${ cmd }`;
-	await Shell.exec(`${ extCmd }`);
-}

package/k8s/Node.ts

@@ -1,119 +0,0 @@
-import '../libs/prototype';
-
-import yaml from 'js-yaml';
-
-import Shell, { ExecOptions } from '../libs/Shell';
-import { log, resolvePath } from '../libs/helpers';
-import { Stage } from '../libs/Project';
-import { Exception } from '../libs/Exception';
-import { Reporter } from '../libs/Reporter';
-
-const ENV = process.env;
-
-
-// ================================================================================================
-
-export class Node{
-
-	configFile = '';
-	namespace = '';
-
-	// ----------
-
-	constructor(stage: Stage){
-		this.namespace  = stage;
-
-		const configKey = `K8S_${ stage.toUpperCase() }_CONFIG`;
-		let   configFile = ENV[ configKey ];
-
-		if(stage !== 'prod' && !configFile){
-			configFile = ENV.K8S_STAGES_CONFIG;
-		}
-
-		if(!configFile){
-			throw new Exception(`K8s config file not found: ${ configKey }`);
-		}
-
-		this.configFile = resolvePath(configFile) ;
-	}
-
-	// ----------
-
-	async applyObject(manifestObj: Record<string, any>){
-		const manifestYaml = yaml.dump(manifestObj);
-		await this.exec(`apply -f -\n${ manifestYaml }`, {silent: false}, 'cat <<EOF | ');
-	}
-
-	// ----------
-
-	async apply(manifestFile: string, appName: string): Promise<{valid: boolean; msg: string}>{
-
-		let status: string;
-
-		try{
-			await this.exec(`apply -f ${ manifestFile } --dry-run=server --validate`);
-			status = (await Shell.exec(`cat ${ manifestFile } | grep 'image:'`)).replace('image: ', '').replace(ENV.CI_REGISTRY_IMAGE+'/', '').trim();
-		}catch(e){
-			log.error(e);
-			log.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
-			log.error(`VALIDATE FAIL: ${appName} at ${manifestFile}`);
-			log.error(e.message);
-			log.error(`${ await Shell.cat(manifestFile) }`);
-			log.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
-
-			return { valid:false, msg: e.message };
-		}
-
-		try{
-			status = await this.exec(`apply -f ${ manifestFile }`);
-			await this.exec(`rollout status deploy/${ appName } --watch=true --timeout=300s`);
-
-			log.info('+++++++++++++++++++++++++++++');
-			log.info(`DEPLOYED: ${appName}`);
-			log.info(await this.exec(`get pods --no-headers -l app=${appName} -o wide`, { ignoreError:true, silent:true }));
-			log.info('+++++++++++++++++++++++++++++');
-
-			return { valid:true, msg: status };
-
-		} catch(error){
-			log.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
-			log.error(`DEPLOY FAIL: ${appName}`);
-			log.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
-			log.error(await this.exec(`get pods --no-headers -l app=${appName} -o wide`, { ignoreError:true, silent:true }));
-			throw error;
-		}
-
-	}
-
-	// ----------
-
-	async status(reporter: Reporter | undefined = undefined, appsOnly: boolean = true): Promise<string[]>{
-
-		const cmd = `get pods -o=jsonpath='{range .items[*]}{"\\n"}{"<b>"}{.metadata.labels.app}{"</b>   "}{range .spec.containers[*]}{.image}{end}{"    "}{.metadata.labels.pod-template-hash}{" "}{.status.phase}{end}'`;
-		const statusArr = (await this.exec(cmd)).split('\n');
-    
-		const result: string[] = [];
-		for (const row of statusArr) {
-			if(row.length < 4 || (appsOnly && (RegExp((ENV.CI_REGISTRY_IMAGE || '')).exec(row)) === null)){
-				continue;
-			}
-			result.push(row.replace(ENV.CI_REGISTRY_IMAGE+'/', ''));
-		}
-    
-		log.info(`K8S Node: ${ this.namespace.toUpperCase() }`, result);
-
-		if(reporter){
-			const msg = `\n<b>${ this.namespace.toUpperCase() }</b> Pods:\n-----------------\n ` + result.join('\n-----------------\n ');
-			await reporter.send(msg, '');
-		}
-
-		return result;
-	}
-
-	// ----------
-
-	async exec(cmd: string, opt: ExecOptions | undefined = undefined, cmdPrefix = ''): Promise<string>{
-		return Shell.exec(`${cmdPrefix}kubectl --kubeconfig ${ this.configFile } -n ${ this.namespace } ${ cmd }`, opt);
-	}
-
-}

package/k8s-build.ts

@@ -1,175 +0,0 @@
-#!/usr/bin/env node
-import './libs/prototype';
-
-import {program, OptionValues} from 'commander';
-
-import Shell from './libs/Shell';
-import { Jira } from './libs/Jira';
-import Git from './libs/Git';
-import { Project } from './libs/Project';
-import { Reporter } from './libs/Reporter';
-import { log, checkEnvVars } from './libs/helpers';
-import { DockerImage } from './k8s/DockerImage';
-import { Config, getApps } from './k8s/App';
-
-const ENV = process.env;
-
-// ========================================================
-
-
-async function main() {
-
-	program
-		.option('--path <project dir>',  'Path to project directory [default - current]')
-
-		.option('--no-prebuild', 'Skip prebuild phase')
-
-		.option('--no-changelog', 'Skip create gitlog')
-		.option('--no-gittag', 'Skip create and push new git tag')
-		.action(
-			async () => {
-
-				let reporter: Reporter | undefined;
-
-				try {
-
-					// ------------------
-					// Инициализируемся
-					checkEnvVars(['CI_STAGES', 'REPORTER']);
-
-					const opts = program.opts() || {};
-					const prj = new Project(opts.path);
-
-					const reporterOpts = JSON.parse(ENV.REPORTER!);
-					reporterOpts.postTitle = opts.scheme;
-
-					const pkg = JSON.parse(await Shell.cat(prj.rootPath +'/package.json') || 'no package.json file!');
-					const pkgBuild =  Number(pkg.version.split('.')[2]);
-				
-					prj.build = prj.build < pkgBuild ? pkgBuild : prj.build;
-					prj.version = `${ pkg.version.split('.').splice(0, 2).join('.') }.${ prj.build }`;
-				
-					if(!pkg.ci){
-						throw new Error('No ci config in package.json file!');
-					}
-									
-					reporter = new Reporter(prj, reporterOpts);
-
-					const config: Config = pkg.ci;
-
-					// Запускаем команду
-					await run(prj, reporter, opts, config);
-
-				} catch(e){
-
-					log.error(e);
-
-					if(reporter){
-						try {
-							await reporter.fail(e);
-						} catch(e2){
-							log.error(e2);
-						}
-				
-					}
-
-					process.exit(1);
-				}
-			}
-		);
-
-	await program.parseAsync(process.argv);
-}
-main();
-
-// ============================================
-
-async function run(prj: Project, reporter: Reporter, opts: OptionValues, config: Config) {
-
-	// ------------------
-	// Инициализируемся
-
-	const release = opts.changelog ? await Jira.resolve(prj) : { tasks:[], bugs:[], changelog:''};
-
-	// --build: Build and push docker image to registry
-	Shell.cd(prj.rootPath);
-
-	// -----------------------------
-	// 1. Start
-	prj.incrementBuild();
-	await reporter.startBuild();
-	log.info(`1. build ${ prj.stage }`);
-
-	// -----------------------------
-	// 2. Chek updated apps
-	await reporter.send(` + Load apps and check updated`);
-	log.info(`2. Load apps and check updated`);
-
-	const updateFiles =  await Git.getDiffFiles();
-	log.info(`\n\nUPDATED FILES:\n$`, updateFiles);
-
-	const { apps, updated } = await getApps(prj, config, updateFiles);
-
-	if(!updated){
-		log.info('SKIP BUILD: NO Apps to update!');
-		await reporter.send(`⏩ SKIP - no apps to update\n ${ updateFiles.join('\n    ')}`, ' ');
-		process.exit(0);
-	}
-
-	await reporter.send(`   ~ <b>${ updated.join('</b>\n   ~ <b>') }</b>`);
-
-	// -----------------------------
-	// 4. write k8s manifests!
-	await reporter.send(` + Write K8s manifests`);
-	log.info(`3. Write K8s manifests`);
-	
-	const image = new DockerImage(prj.rootPath, prj.shortVersion, prj.build);
-	log.dbg(`DockerImage`, image);
-
-	for (const appName of updated) {
-		await apps[ appName ]?.writeManifest(prj, image);
-	}
-
-	// -----------------------------
-	// 5. Build
-	if(opts.prebuild){
-		await reporter.send(` + Test and distrib`);
-		log.dbg(`4. Pre build`);
-		await image.prebuild();
-	}
-
-	await reporter.send(` + Build docker image: ${ image.tag }`);
-	log.dbg(`5. Build`);
-	await image.build();
-
-	await reporter.send(` + Push docker image to registry`);
-	log.dbg(`6. Docker push`);
-	await image.push();
-
-	// -----------------------------
-	// 5. Save new build to GitLab CI Variable
-
-	await reporter.send(` + Save current build: ${ prj.build }`);
-	log.dbg(`7. Put current build:${ prj.build } to GitLab`);
-	await prj.saveGitLabBuild();
-
-	// -----------------------
-
-	
-	if(opts.changelog){
-		await prj.updateChangeLog(release.changelog);
-		await reporter.send(` + Update <a href=\"${ prj.url }/-/blob/${ ENV.CI_COMMIT_REF_NAME || 'dev' }/CHANGELOG.md\">changelog</a>.`);
-	}
-
-	await Git.setOrigin();
-	await Shell.exec(`git add -A`, { ignoreError: true, silent: false });
-	await Shell.exec(`git commit -a -m "v${ prj.version }"`, { ignoreError: true, silent: false });
-	await Git.push(ENV.CI_COMMIT_REF_NAME || 'dev');
-
-	if(opts.gittag){
-		await reporter.send(` + Git tag new version`);
-		await Git.makeTag(prj.version);
-	}
-
-	await reporter.send(`\n✅ <b>${ reporter.title }</b> builded!`, ' ');
-}