codefresh 0.78.2

package/lib/interface/cli/commands/hybrid/installation-process.js

@@ -0,0 +1,51 @@
+const Promise = require('bluebird');
+const { to } = require('./../../../../logic/cli-config/errors/awaitTo');
+
+async function create(installer, event) {
+    const [err, res] = await to(installer.createNewInstallationProgress(event));
+    if (err) {
+        return {};
+    }
+    return res.progress;
+}
+
+function buildReporter(installer, progress) {
+    return {
+        report: async (event, status, options = {}) => {
+            if (!progress.id) {
+                return Promise.resolve();
+            }
+            const data = {
+                event,
+                content: {
+                    status,
+                    ...options,
+                },
+            };
+            return installer.reportInstallationProgressEvent({ id: progress.id }, data);
+        },
+    };
+}
+
+module.exports = {
+    create,
+    buildReporter,
+    events: {
+        ACCEPTANCE_TESTS_RAN: 'acceptance-tests-ran',
+        AGENT_INSTALLED: 'agent-installed',
+        AGENT_CREATED: 'agent-created',
+        RUNNER_INSTALLED: 'runner-installed',
+        MONITOR_INSTALLED: 'monitor-installed',
+        CLUSTER_INTEGRATION_ADDED: 'cluster-integration-added',
+        AGENT_UPDATED: 'agent-updated',
+        RUNTIME_INSTALLED: 'runtime-installed',
+        APP_PROXY_INSTALLED: 'app-proxy-installed',
+        PIPELINE_EXECUTED: 'demo-pipeline-executed',
+        PIPELINE_CREATED: 'demo-pipeline-created',
+        FINISHED: 'finished',
+    },
+    status: {
+        SUCCESS: 'success',
+        FAILURE: 'failure',
+    },
+};

package/lib/interface/cli/commands/hybrid/key-helper.js

@@ -0,0 +1,82 @@
+const forge = require('node-forge');
+const rp = require('request-promise');
+const AdmZip = require('adm-zip');
+
+const sdk = require('../../../../logic/sdk');
+// eslint-disable-next-line prefer-destructuring
+const pki = forge.pki;
+// eslint-disable-next-line prefer-destructuring
+const rsa = pki.rsa;
+
+const defaultCertCN = 'docker.codefresh.io';
+
+function generateKeys() {
+    return new Promise((resolve, reject) => {
+        function keyGenerated(err, { publicKey, privateKey }) {
+            if (err) {
+                reject(err);
+                return;
+            }
+
+            const csr = pki.createCertificationRequest();
+            csr.publicKey = publicKey;
+            csr.setSubject([{
+                name: 'commonName',
+                value: defaultCertCN,
+            }]);
+            csr.sign(privateKey);
+
+            resolve({
+                key: pki.privateKeyToPem(privateKey),
+                csr: pki.certificationRequestToPem(csr),
+            });
+        }
+
+        rsa.generateKeyPair({
+            bits: 2048,
+        }, keyGenerated);
+    });
+}
+
+function extractZipCerts(zip) {
+    const data = {};
+    const admZip = new AdmZip(zip);
+    admZip.forEach(e => {
+        if (e.entryName === 'cf-ca.pem') {
+            data.ca = admZip.readAsText(e);
+        }
+        if (e.entryName === 'cf-server-cert.pem') {
+            data.serverCert = admZip.readAsText(e);
+        }
+    });
+    return data;
+}
+
+async function produceVenonaKeys(namespace) {
+    const keys = await generateKeys();
+    const body = JSON.stringify({
+        reqSubjectAltName: `IP:127.0.0.1,DNS:dind,DNS:*.dind.${namespace},DNS:*.dind.${namespace}.svc,DNS:*.cf-cd.com,DNS:*.codefresh.io`,
+        csr: keys.csr,
+    });
+
+    const zip = await rp.post({
+        uri: `${sdk.config.context.url}/api/custom_clusters/signServerCerts`,
+        body,
+        encoding: null,
+        headers: {
+            'Content-type': 'application/json',
+            Authorization: sdk.config.context.token,
+        },
+    });
+
+    const allKeys = {
+        ...keys,
+        ...extractZipCerts(zip),
+    };
+
+    return allKeys;
+}
+
+module.exports = {
+    produceVenonaKeys,
+};

package/lib/interface/cli/commands/hybrid/migration.js

@@ -0,0 +1,192 @@
+const _ = require('lodash');
+const { to } = require('./../../../../logic/cli-config/errors/awaitTo');
+const { sdk } = require('../../../../logic');
+const installAgent = require('../agent/install.cmd');
+const installMonitoring = require('../monitor/install.cmd');
+const colors = require('colors');
+const inquirer = require('inquirer');
+const fs = require('fs');
+const {
+    getTestPipeline,
+    updateTestPipelineRuntime,
+    createTestPipeline,
+    executeTestPipeline,
+    createProgressBar,
+    runUpgrade,
+    serealizeToKeyValuePairs,
+} = require('./helper');
+const { getNewAgentName } = require('../agent/helper');
+
+
+const TEST_PIPELINE_NAME = 'CF_Runner_Migration_test';
+
+async function createAndRunTestPipeline(runtimeName, errHandler) {
+    let testPipeline;
+    const [getPipelineErr, _testPipeline] = await to(getTestPipeline(TEST_PIPELINE_NAME));
+    testPipeline = _testPipeline;
+    await errHandler(getPipelineErr, 'Could not get test pipeline');
+    if (!testPipeline) {
+        // eslint-disable-next-line no-shadow
+        const [createPipelineErr, _testPipeline] = await to(createTestPipeline(
+            runtimeName,
+            TEST_PIPELINE_NAME,
+            [`echo runtime ${runtimeName} migrated successfully!`],
+        ));
+        await errHandler(createPipelineErr, 'Failed to create test pipeline');
+        testPipeline = _testPipeline;
+    } else {
+        const [updatePipelineErr] = await to(updateTestPipelineRuntime(testPipeline, runtimeName));
+        if (updatePipelineErr) {
+            console.log(colors.yellow('*warning* failed to update test pipeline runtime, you can' +
+                ' change it manually if you want to run it again on this runtime'));
+        }
+    }
+    const [runPipelineErr] = await to(executeTestPipeline(
+        runtimeName,
+        testPipeline.metadata.name,
+    ));
+    await errHandler(runPipelineErr, 'Failed to run test pipeline');
+}
+
+async function migrate({
+    runtimeName,
+    kubeContextName,
+    kubeNamespace,
+    installMonitor,
+    agentName,
+    handleError,
+    kubeConfigPath,
+    shouldMakeDefaultRe,
+    storageClassName,
+    setValue,
+    setFile,
+    verbose,
+    agents,
+}) {
+    const newAgentName = agentName || await getNewAgentName(kubeContextName, kubeNamespace, agents);
+    const [getRuntimeErr, runtime] = await to(sdk.runtimeEnvs.get({ name: runtimeName }));
+    handleError(getRuntimeErr, `Failed to get runtime with name "${runtimeName}"`);
+    let oldNodeSelector = _.get(runtime, 'dockerDaemonScheduler.cluster.nodeSelector');
+    const oldStorageClassName = _.get(runtime, 'dockerDaemonScheduler.pvcs.dind.storageClassName');
+
+    // prompt migration process confirmation
+    console.log(`${colors.red('This migration process will do the following:')}`);
+    console.log('\u2022 Delete the old venona deployment and secrets from the selected namespace');
+    console.log(`\u2022 Create a new Codefresh runner with the name "${colors.cyan(newAgentName)}" on the selected namespace`);
+    console.log(`\u2022 Attach runtime "${colors.cyan(runtimeName)}" to the new Codefresh runner`);
+    console.log('\u2022 Install Codefresh runner monitoring components on the selected namespace');
+    console.log('\u2022 Run a test pipeline to check that the migration was successful\n');
+
+    const answer = await inquirer.prompt({
+        type: 'confirm',
+        name: 'migrationConfirmed',
+        default: false,
+        message: 'Are you sure you want to proceed with the migration process?',
+    });
+    if (!answer.migrationConfirmed) {
+        console.log('Migration process aborted, exiting...');
+        process.exit(1);
+    }
+
+    // prepare old runtime
+    if (oldStorageClassName && oldStorageClassName.startsWith('dind-local-volumes-venona')) {
+        // need to replace to start with 'dind-local-volumes-runner'
+        const newRe = _.set(runtime, 'dockerDaemonScheduler.pvcs.dind.storageClassName', oldStorageClassName.replace('venona', 'runner'));
+        const [err] = await to(sdk.runtimeEnvs.update({ name: runtimeName }, newRe));
+        handleError(err, 'Failed to update runtime storage class name');
+    }
+
+    // delete old agent and runtime
+    console.log(`Running migration script on runtime: ${colors.cyan(runtimeName)}`);
+    const [migrateScriptErr, migrateScriptExitCode] = await to(sdk.agents.migrate({
+        kubeContextName,
+        kubeNamespace,
+        verbose,
+        events: createProgressBar(),
+    }));
+    handleError(migrateScriptErr, 'Failed to execute migration script');
+    if (migrateScriptExitCode !== 0) {
+        handleError(new Error(`migration script exited with code ${migrateScriptExitCode}`), 'Migration failed');
+    }
+
+    // read old deploment configuration
+    const filename = './migration.json';
+    let oldConfig = {};
+    let oldEnv;
+    try {
+        const data = fs.readFileSync(filename).toString('utf-8');
+        oldConfig = JSON.parse(data);
+    } catch (err) {
+        console.log(colors.yellow(`could not read old deployment configuration file: ${err}`));
+    }
+
+    if (oldConfig.tolerations) {
+        oldConfig.tolerations = oldConfig.tolerations.filter(t => !t.key.includes('node.kubernetes.io'));
+    }
+
+    if (oldConfig.nodeSelector) {
+        oldConfig.nodeSelector = serealizeToKeyValuePairs(oldConfig.nodeSelector);
+    }
+
+    if (oldNodeSelector) {
+        oldNodeSelector = serealizeToKeyValuePairs(oldNodeSelector);
+    }
+    if (oldConfig.env) {
+        oldEnv = serealizeToKeyValuePairs(oldConfig.env);
+    }
+
+    // install new agent and runtime
+    console.log(`Creating new codefresh runner with name: ${colors.cyan(newAgentName)}`);
+    const agentInstallOptions = {
+        name: newAgentName,
+        'kube-context-name': kubeContextName,
+        'kube-node-selector': oldConfig.nodeSelector,
+        'build-node-selector': oldNodeSelector,
+        'kube-namespace': kubeNamespace,
+        'agent-kube-namespace': kubeNamespace,
+        'agent-kube-context-name': kubeContextName,
+        'agent-kube-config-path': kubeConfigPath,
+        tolerations: JSON.stringify(oldConfig.tolerations),
+        'kube-config-path': kubeConfigPath,
+        'install-runtime': true,
+        'runtime-name': runtimeName,
+        'skip-re-creation': true,
+        verbose,
+        'make-default-runtime': shouldMakeDefaultRe,
+        'storage-class-name': storageClassName || oldStorageClassName,
+        terminateProcess: false,
+        'set-value': setValue,
+        'set-file': setFile,
+        envVars: oldEnv,
+    };
+    const [agentInstallErr] = await to(installAgent.handler(agentInstallOptions));
+    handleError(agentInstallErr, 'Failed to install new agent and runtime');
+
+    // Install new monitoring components
+    if (installMonitor) {
+        console.log('Installing monitoring components');
+        const monitorInstallOptions = {
+            'kube-config-path': kubeConfigPath,
+            'cluster-id': kubeContextName,
+            'kube-context-name': kubeContextName,
+            'kube-namespace': kubeNamespace,
+            token: _.get(sdk, 'config.context.token'),
+            verbose,
+            noExit: true, // to prevent if from calling inner: process.exit()
+        };
+        const [monitorErr] = await to(installMonitoring.handler(monitorInstallOptions));
+        await handleError(monitorErr, 'Monitor installation failed');
+    }
+
+    // Execute test pipeline on new runner
+    await createAndRunTestPipeline(runtimeName, handleError);
+}
+
+async function upgrade({ kubeContextName, kubeNamespace, handleError }) {
+    const [err] = await to(runUpgrade({ kubeNamespace, kubeContextName }));
+    await handleError(err, 'Upgrade failed');
+}
+module.exports = {
+    migrate,
+    upgrade,
+};

package/lib/interface/cli/commands/hybrid/pipeline-helper.js

@@ -0,0 +1,71 @@
+const sdk = require('../../../../logic/sdk');
+const colors = require('colors');
+const installationProgress = require('./installation-process');
+const _ = require('lodash');
+
+const {
+    createTestPipeline,
+    executeTestPipeline,
+    updateTestPipelineRuntime,
+    INSTALLATION_DEFAULTS,
+} = require('./helper');
+
+const defaultDockerRegistry = 'quay.io';
+
+async function addPipelineToInstallationPlan(installationPlan, dockerRegistry = '', executePipeline = true) {
+    if (!dockerRegistry) {
+        // eslint-disable-next-line no-param-reassign
+        dockerRegistry = defaultDockerRegistry;
+    }
+
+    const pipelines = await sdk.pipelines.list({ id: `${INSTALLATION_DEFAULTS.PROJECT_NAME}/${INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME}` });
+    const testPipelineExists = !!_.get(pipelines, 'docs.length');
+
+    if (!testPipelineExists) {
+        installationPlan.addStep({
+            name: 'create test pipeline',
+            func: async () => {
+                await createTestPipeline(
+                    installationPlan.getContext('runtimeName'),
+                    INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME,
+                    ['echo hello Codefresh Runner!'],
+                    dockerRegistry,
+                );
+            },
+            installationEvent: installationProgress.events.PIPELINE_CREATED,
+        });
+    } else {
+        installationPlan.addStep({
+            name: 'update test pipeline runtime',
+            func: async () => {
+                await updateTestPipelineRuntime(
+                    undefined,
+                    installationPlan.getContext('runtimeName'),
+                    INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME,
+                    dockerRegistry,
+                );
+            },
+            errMessage: colors.yellow('*warning* could not update test pipeline runtime, you can' +
+                ' change it manually if you want to run it again on this runtime'),
+            successMessage: 'Updated test pipeline runtime',
+            exitOnError: false,
+        });
+    }
+
+    installationPlan.addStep({
+        name: 'execute test pipeline',
+        func: async () => {
+            await executeTestPipeline(
+                installationPlan.getContext('runtimeName'),
+                INSTALLATION_DEFAULTS.DEMO_PIPELINE_NAME,
+            );
+        },
+        errMessage: 'Failed to execute test pipeline',
+        installationEvent: installationProgress.events.PIPELINE_EXECUTED,
+        condition: executePipeline,
+    });
+}
+
+module.exports = {
+    addPipelineToInstallationPlan,
+};

package/lib/interface/cli/commands/hybrid/upgrade.cmd.js

@@ -0,0 +1,195 @@
+
+/* eslint-disable max-len */
+const Command = require('../../Command');
+const runnerRoot = require('../root/runner.cmd');
+const inquirer = require('inquirer');
+const { getAllKubeContexts, getKubeContext, getAllNamespaces } = require('../../helpers/kubernetes');
+const colors = require('colors');
+const DEFAULTS = require('../../defaults');
+const sdk = require('../../../../logic/sdk');
+const _ = require('lodash');
+const semver = require('semver');
+const { to } = require('./../../../../logic/cli-config/errors/awaitTo');
+const {
+    createErrorHandler,
+    getRuntimesWithVersions,
+    getRuntimeVersion,
+    drawCodefreshFiglet,
+} = require('./helper');
+const { migrate, upgrade } = require('./migration');
+
+const openIssueMessage = `If you had any issues with the upgrade process please report them at: ${colors.blue('https://github.com/codefresh-io/cli/issues/new')}`;
+const handleError = createErrorHandler(openIssueMessage);
+
+function isOldVersion(version) {
+    const v = version.split('x').join('0');
+    return semver.lt(v, '1.0.0');
+}
+
+function getStrategy(shouldDoMigration) {
+    return shouldDoMigration ? migrate : upgrade;
+}
+
+const upgradeCmd = new Command({
+    root: false,
+    parent: runnerRoot,
+    command: 'upgrade',
+    requiresAuthentication: false,
+    description: 'Upgrades codefresh runner solution\'s components',
+    webDocs: {
+        category: 'Runner',
+        title: 'Upgrade',
+        weight: 100,
+    },
+    builder: yargs => yargs
+        .env('CF_ARG_')
+        .option('agent-name', {
+            describe: 'The name of the agent to be upgraded or created',
+        })
+        .option('runtime-name', {
+            describe: 'The name of the runtime to be upgraded',
+        })
+        .option('url', {
+            describe: 'Codefresh system custom url',
+            default: DEFAULTS.URL,
+        })
+        .option('kube-context-name', {
+            describe: 'Name of the kubernetes context from which the Codefresh Agent and Runtime should be removed',
+        })
+        .option('kube-namespace', {
+            describe: 'Name of the kubernetes namespace from which the Codefresh Agent and Runtime should be removed',
+        })
+        .option('kube-config-path', {
+            describe: 'Path to kubeconfig file (default is $HOME/.kube/config)',
+        })
+        .option('install-monitor', {
+            describe: 'Install a monitoring component that will help provide valueable data about your cluster to Codefresh (only useable when running migration)',
+            type: 'boolean',
+            default: true,
+        })
+        .option('verbose', {
+            describe: 'Print logs',
+        }),
+    handler: async (argv) => {
+        const {
+            'kube-config-path': kubeConfigPath,
+            'agent-name': agentName,
+            'install-monitor': installMonitor,
+            verbose,
+        } = argv;
+        let {
+            'kube-context-name': kubeContextName,
+            'kube-namespace': kubeNamespace,
+            'runtime-name': runtimeName,
+        } = argv;
+
+        const [listReErr, runtimes] = await to(sdk.runtimeEnvs.list({ }));
+        await handleError(listReErr, 'Failed to get runtime environments');
+        const runtimesByName = runtimes.reduce((acc, cur) => ({ ...acc, [cur.metadata.name]: cur }), {});
+
+        const [listAgentsErr, agents] = await to(sdk.agents.list({ }));
+        await handleError(listAgentsErr, 'Failed to get agents');
+        const agentsByName = agents.reduce((acc, cur) => ({ ...acc, [cur.name]: cur }), {});
+
+        console.log(colors.green('This upgrader will guide you through the runner upgrade process'));
+
+        let runtimeVersion;
+
+        if (agentName && agentsByName[agentName]) {
+            const agent = agentsByName[agentName];
+            const attachedRuntimes = agent.runtimes || [];
+            if (attachedRuntimes.length === 1) {
+                [runtimeName] = attachedRuntimes;
+                console.log(`Chose to upgrade runner: "${colors.cyan(agentName)}" and runtime: "${colors.cyan(runtimeName)}"`);
+            }
+        }
+
+        if (!runtimeName) {
+            const runtimesWithVersions = getRuntimesWithVersions(runtimes, agents);
+            if (!_.size(runtimesWithVersions)) {
+                handleError(new Error('Found 0 runtime environments that can be upgraded or migrated'), 'no runtimes found');
+            }
+
+            // for fast lookup of the choice
+            const runtimesWithVersionsStrings = {};
+            _.map(_.keys(runtimesWithVersions), (re) => { runtimesWithVersionsStrings[`${re}\t(version ${runtimesWithVersions[re]})\t${runtimesWithVersions[re] === '0.x.x' ? '[migrate]' : '[upgrade]'}`] = re; });
+
+            const answer = await inquirer.prompt({
+                type: 'list',
+                name: 'runtime',
+                message: 'The runtimes that can be upgraded or migrated (if version < 1.x.x)',
+                choices: _.keys(runtimesWithVersionsStrings),
+            });
+
+            runtimeName = runtimesWithVersionsStrings[answer.runtime];
+            runtimeVersion = runtimesWithVersions[runtimeName];
+        } else {
+            runtimeVersion = getRuntimeVersion(runtimeName, agents);
+        }
+        const runtime = runtimesByName[runtimeName];
+        if (!runtime) {
+            handleError(new Error(`runtime environment "${colors.cyan(runtimeName)}" does not exist on this account. run "${colors.cyan('codefresh get re')}" to get all runtimes-environments of this account`), 'runtime environment not found');
+        }
+        runtime.metadata.version = runtimeVersion;
+
+        if (!kubeContextName) {
+            const contexts = getAllKubeContexts(kubeConfigPath);
+            const currentKubeContext = getKubeContext(kubeConfigPath);
+
+            const answer = await inquirer.prompt({
+                type: 'list',
+                name: 'context',
+                message: 'Name of Kubernetes context to use',
+                default: currentKubeContext,
+                choices: contexts,
+            });
+
+            kubeContextName = answer.context;
+        }
+
+        if (!kubeNamespace) {
+            kubeNamespace = _.get(runtime, 'runtimeScheduler.cluster.namespace');
+        }
+
+        if (!kubeNamespace) {
+            const [err, namespaces] = await to(getAllNamespaces(kubeConfigPath, kubeContextName));
+            handleError(err, 'Failed to get cluster namespaces');
+
+            const answer = await inquirer.prompt({
+                type: 'list',
+                name: 'namespace',
+                message: 'Kubernetes namespace on which the runtime to be upgraded/migrated is installed',
+                choices: namespaces,
+            });
+
+            kubeNamespace = answer.namespace;
+        }
+
+        console.log(`\n${colors.green('Upgrade options summary:')} 
+1. Kubernetes Context: ${colors.cyan(kubeContextName)}
+2. Kubernetes Namespace: ${colors.cyan(kubeNamespace)}
+3. Runtime name: ${colors.cyan(runtimeName)}
+`);
+
+        const shouldDoMigration = isOldVersion(runtime.metadata.version);
+        const strategy = getStrategy(shouldDoMigration);
+        const [err] = await to(strategy({
+            kubeContextName,
+            kubeNamespace,
+            agentName,
+            agents,
+            runtimeName,
+            installMonitor,
+            verbose,
+            handleError,
+        }));
+        handleError(err, `Failed to ${shouldDoMigration ? 'migrate' : 'upgrade'} codefresh runner`);
+
+        console.log(colors.green(`\nSuccessfully ${shouldDoMigration ? 'migrated' : 'upgraded'} your Codefresh Runner to the latest version`));
+        console.log(colors.green(`If you had any issues with the migration process please report them at: ${colors.blue('https://github.com/codefresh-io/cli/issues/new')}`));
+        await drawCodefreshFiglet();
+        process.exit(); // TODO : This is not needed - needed to be fixed
+    },
+});
+
+module.exports = upgradeCmd;

package/lib/interface/cli/commands/image/annotate.cmd.js

@@ -0,0 +1,61 @@
+const Command = require('../../Command');
+const { parseFamiliarName } = require('@codefresh-io/docker-reference');
+const annotateRoot = require('../root/annotate.cmd');
+const CFError = require('cf-errors');
+const { sdk } = require('../../../../logic');
+const annotationLogic = require('../annotation/annotation.logic');
+
+const command = new Command({
+    command: 'image <id>',
+    aliases: ['img'],
+    parent: annotateRoot,
+    description: 'Annotate an image',
+    usage: 'Annotating an image gives the ability to add extra context on your already existing images',
+    webDocs: {
+        category: 'Images',
+        title: 'Annotate Image',
+    },
+    builder: (yargs) => {
+        return yargs
+            .positional('id', {
+                describe: 'Docker image full name or id',
+            })
+            .option('label', {
+                describe: 'annotations to add to the image',
+                default: [],
+                alias: 'l',
+                array: true,
+            })
+            .example('codefresh annotate image 2dfacdaad466 -l coverage=75%', 'Annotate an image with a single label')
+            .example('codefresh annotate image 2dfacdaad466 -l coverage=75% -l tests_passed=true', 'Annotate an image with multiple labels');
+    },
+    handler: async (argv) => {
+        let dockerImageId = argv.id;
+        const useFullName = dockerImageId.includes(':');
+
+        if (useFullName) {
+            const { repository, tag } = parseFamiliarName(dockerImageId);
+            const results = await sdk.images.list({
+                imageDisplayNameRegex: repository,
+                tag,
+                select: 'internalImageId',
+            });
+
+            if (!results.length) {
+                throw new CFError('Image does not exist');
+            }
+
+            if (results.length > 1) {
+                throw new CFError(`Could not get image id. ${results.length} images found.`);
+            }
+
+            dockerImageId = results[0].internalImageId;
+        }
+
+        await annotationLogic.createAnnotations({ entityId: dockerImageId, entityType: 'image', labels: argv.label });
+        console.log('Annotations added successfully');
+    },
+});
+
+module.exports = command;
+