codefresh 0.78.2

package/lib/interface/cli/commands/offline-logs/utils.js

@@ -0,0 +1,159 @@
+const readline = require("readline");
+const moment = require("moment");
+const { ObjectId } = require("mongodb")
+
+const { join } = require("path");
+const fs = require("fs");
+
+const defaultCollections = ["logs", "metadata"];
+
+// converts date to objectId for filter purposes
+const objectIdFromDate = function (date) {
+  return Math.floor(date.getTime() / 1000).toString(16) + "0000000000000000";
+};
+
+const writeLogsToFile = function (
+  upperBound,
+  lowerBound,
+  collection,
+  logsToArchive,
+  path
+) {
+  const fileName = `${createRangeStr(lowerBound, upperBound, collection)}.json`
+  const absPath = join(path, fileName);
+  const data = JSON.stringify(logsToArchive);
+  fs.writeFileSync(absPath, data);
+  console.info(`logs from collection '${collection}' were offloaded to '${absPath}'`)
+};
+
+const createRangeStr = function(lowerBound, upperBound, collection) {
+  const upperBoundDate = moment(upperBound).subtract(1, "days");
+  const fileDateRange = `${moment(lowerBound).format(
+    "YYYY-MM-DD"
+  )}-${upperBoundDate.format("YYYY-MM-DD")}`;
+  return `${fileDateRange}-${collection}`;
+}
+
+const getMinDate = function(minLog) {
+  const minLogId = ObjectId(minLog._id)
+  return moment(
+    minLogId.getTimestamp()
+  ).startOf("day");
+}
+
+const checkRemnant = function (lowerBound, cutoffDateObj) {
+  return Math.ceil(cutoffDateObj.diff(lowerBound, "days"));
+};
+///////////////////////////////////////////////////////////////
+const ensureIndex = async function (database, collection) {
+  const collectionObj = database.collection(collection);
+  const indexName = "accountId_1_jobId_1";
+
+  const indexExist = await collectionObj.indexExists(indexName)
+
+  if (indexExist) {
+    console.info(
+      `index '${indexName}' already exists in collection '${collection}'`
+    );
+    return;
+  }
+
+  const documentsCount = await collectionObj.estimatedDocumentCount();
+
+  const queryStr = `There are approximately ${documentsCount} documents on collection '${collection}'. 
+    do you want to create an index for it? (creating an index for a large collection may take a while, 
+    but can improve performance on future read operations) [yes/no] `;
+
+  const userInput = await this.getUserInput(queryStr);
+
+  if (!userInput) {
+    console.info(`skipping collection '${collection}'`);
+    return;
+  }
+
+  await collectionObj.createIndex(
+    { accountId: 1, jobId: 1 },
+    { background: true },
+  );
+  console.info(
+    `index '${indexName}' was created for collection '${collection}'`
+  );
+};
+
+const getUserInput = async function (queryStr) {
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  const questionAsync = (query) =>
+    new Promise((resolve) => rl.question(query, resolve));
+
+  try {
+    const userInput = (await questionAsync(queryStr)).toLowerCase();
+
+    switch (userInput) {
+      case "yes":
+      case "y":
+        return true;
+      case "no":
+      case "n":
+        return false;
+      default:
+        console.warn("invalid input");
+        return false;
+    }
+  } finally {
+    rl.close();
+  }
+};
+
+const collectionExists = async function(dbObject, targetCollection) {
+  const collectionsObjectList = await dbObject.listCollections().toArray();
+  const collectionsList = collectionsObjectList.map((object) => {
+    return object.name;
+  })
+  return (collectionsList.indexOf(targetCollection) >= 0);
+}
+
+const findMinLog = async function(collectionObj, UpperBoundDateId, collection) {
+  const minLog = await collectionObj
+    .find({ _id: { $lt: ObjectId(UpperBoundDateId) } })
+    .sort({ _id: 1 })
+    .limit(1)
+    .toArray();
+
+  if (minLog.length === 0) {
+    return undefined;
+  }
+
+  return minLog[0];
+}
+
+const checkCursorState = function (cursor, collection, targetCollection) {
+  if (!cursor.cursorState.killed) {
+    console.info(
+      `logs from '${collection}' were archived to '${targetCollection}'`
+    );
+    return;
+  } else {
+    throw new Error(
+      "Cursor error. Archiving operation may not be completed.\
+      The old logs were not deleted from the source collection."
+    );
+  }
+}
+
+module.exports = {
+  objectIdFromDate,
+  writeLogsToFile,
+  checkRemnant,
+  ensureIndex,
+  getUserInput,
+  collectionExists,
+  findMinLog,
+  checkCursorState,
+  createRangeStr,
+  getMinDate,
+  defaultCollections,
+};

package/lib/interface/cli/commands/pipeline/delete.cmd.js

@@ -0,0 +1,31 @@
+const Command = require('../../Command');
+const deleteRoot = require('../root/delete.cmd');
+const { sdk } = require('../../../../logic');
+
+
+const command = new Command({
+    command: 'pipeline [name]',
+    aliases: ['pip'],
+    parent: deleteRoot,
+    description: 'Delete a pipeline',
+    webDocs: {
+        category: 'Pipelines',
+        title: 'Delete Pipeline',
+    },
+    builder: (yargs) => {
+        return yargs
+            .positional('name', {
+                describe: 'Pipeline name',
+            });
+    },
+    handler: async (argv) => {
+        const { name } = argv;
+
+        await sdk.pipelines.delete({ name });
+        console.log(`Pipeline '${name}' deleted.`);
+    },
+});
+
+
+module.exports = command;
+

package/lib/interface/cli/commands/pipeline/get.cmd.js

@@ -0,0 +1,125 @@
+const debug = require('debug')('codefresh:cli:create:pipelines2');
+const Command = require('../../Command');
+const CFError = require('cf-errors');
+const _ = require('lodash');
+const DEFAULTS = require('../../defaults');
+const { prepareKeyValueFromCLIEnvOption } = require('../../helpers/general');
+const Output = require('../../../../output/Output');
+const { sdk } = require('../../../../logic');
+const Pipeline = require('../../../../logic/entities/Pipeline');
+
+const getRoot = require('../root/get.cmd');
+
+/**
+ * returns a limit based on the passed `--limit, --all` and default values
+ * The limit for the flag `all` (10k) was set in сli defaults values,
+ * so as not to touch the pipeline-manager logic (since requests from other places can go to it)
+ * @param argvLimit {number}- value from `--limit`
+ * @param allFlag {boolean} - value from `--all`
+ * @returns {number|*}
+ * @private
+ */
+function _getLimit(argvLimit, allFlag) {
+    if (allFlag && _.isUndefined(argvLimit)) return DEFAULTS.GET_ALL_PIPELINES_LIMIT;
+    return argvLimit >= 0 ? argvLimit : DEFAULTS.GET_LIMIT_RESULTS;
+}
+
+const command = new Command({
+    command: 'pipelines [id..]',
+    aliases: ['pip', 'pipeline'],
+    parent: getRoot,
+    description: 'Get a specific pipeline or an array of pipelines',
+    webDocs: {
+        category: 'Pipelines',
+        title: 'Get Pipeline',
+    },
+    builder: (yargs) => {
+        return yargs
+            .positional('id', {
+                describe: 'Pipeline name/id',
+            })
+            .option('decrypt-variables', {
+                alias: 'd',
+                describe: 'Will decrypt encrypted variables',
+            })
+            .option('name', {
+                describe: 'Filter pipelines by name',
+            })
+            .option('project-id', {
+                describe: 'Filter pipelines by project id',
+            })
+            .option('project', {
+                describe: 'Filter pipelines by project name',
+            })
+            .option('label', {
+                describe: 'Filter by a label',
+                alias: 'l',
+                default: [],
+            })
+            .option('limit', {
+                describe: `Limit amount of returned results [default: ${DEFAULTS.GET_LIMIT_RESULTS}]`,
+            })
+            .option('all', {
+                describe: 'Remove default limit of returned result',
+            })
+            .option('page', {
+                describe: 'Paginated page',
+                default: DEFAULTS.GET_PAGINATED_PAGE,
+            });
+    },
+    handler: async (argv) => {
+        const { id: ids, name, d: decryptVariables, projectId, project: projectName, all } = argv;
+        const limit = _getLimit(argv.limit, all);
+        const offset = (argv.page - 1) * (limit || 0);
+        const labels = prepareKeyValueFromCLIEnvOption(argv.label);
+        debug(`decrypt: ${decryptVariables}`);
+        if (!_.isEmpty(ids)) {
+            const pipelines = [];
+            for (const id of ids) {
+                try {
+                    const currPipeline = await sdk.pipelines.get({
+                        name: id,
+                        decryptVariables,
+                    });
+                    pipelines.push(Pipeline.fromResponse(currPipeline));
+                } catch (err) {
+                    if (pipelines.length) {
+                        Output.print(pipelines);
+                    }
+
+                    debug(err.toString());
+                    const message = err.toString()
+                        .includes('not find') ? `Pipeline '${id}' was not found.` : 'Error occurred';
+                    throw new CFError({
+                        cause: err,
+                        message,
+                    });
+                }
+            }
+            Output.print(pipelines);
+        } else {
+            let _projectId = projectId;
+            if (!projectId && projectName) {
+                const project = await sdk.projects.getByName({ name: projectName })
+                    .catch(e => Promise.reject(new CFError({
+                        message: `Could not get project "${projectName}"`,
+                        cause: e,
+                    })));
+                _projectId = project.id;
+            }
+
+            const pipelines = await sdk.pipelines.list({
+                limit,
+                offset,
+                id: name,
+                labels,
+                projectId: _projectId,
+            });
+            Output.print(_.map(_.get(pipelines, 'docs'), Pipeline.fromResponse));
+        }
+    },
+});
+
+module.exports = command;
+module.exports._getLimit = _getLimit;
+

package/lib/interface/cli/commands/pipeline/get.completion.js

@@ -0,0 +1,14 @@
+const _ = require('lodash');
+const pipelines = () => require('../../../../logic').sdk.pipelines;
+const Pipeline = require('../../../../logic/entities/Pipeline');
+const { authContextWrapper } = require('../../completion/helpers');
+
+const positionalHandler = async ({word, argv}) => {
+    const pips = await pipelines().list({ limit: 25, offset: 0 });
+    return _.map(_.get(pips, 'docs'), Pipeline.fromResponse).map(p => p.name);
+};
+
+module.exports = {
+    positionalHandler: authContextWrapper(positionalHandler),
+};
+

package/lib/interface/cli/commands/pipeline/pipeline.sdk.spec.js

@@ -0,0 +1,137 @@
+const DEFAULTS = require('../../defaults');
+const getCmd = require('./get.cmd').toCommand();
+const deleteCmd = require('./delete.cmd').toCommand();
+const BasePipeline = require('./run.base');
+const CfPipeline = require('./run.cf');
+const { positionalHandler } = require('./get.completion');
+const { sdk } = require('../../../../logic');
+
+jest.mock('../../helpers/validation'); // eslint-disable-line
+jest.mock('../../../../../check-version');
+jest.mock('../../completion/helpers', () => { // eslint-disable-line
+    return {
+        authContextWrapper: func => func,
+    };
+});
+
+jest.mock('../../../../logic/entities/Pipeline', () => { // eslint-disable-line
+    return {
+        fromResponse: res => res,
+    };
+});
+
+const request = require('requestretry');
+
+const DEFAULT_RESPONSE = request.__defaultResponse();
+
+describe('pipeline', () => {
+    beforeEach(async () => {
+        request.__reset();
+        request.mockClear();
+        await configureSdk(); // eslint-disable-line
+    });
+
+    describe('commands', () => {
+        describe('get', () => {
+            const { _getLimit } = require('./get.cmd');
+            it('should handle getting given id', async () => {
+                const argv = { id: ['some id'] };
+                await getCmd.handler(argv);
+                await verifyResponsesReturned([DEFAULT_RESPONSE]); // eslint-disable-line
+            });
+
+            it('should handle getting all', async () => {
+                const argv = {};
+                const response = { statusCode: 200, body: { docs: [DEFAULT_RESPONSE.body] } };
+                request.__setResponse(response);
+                await getCmd.handler(argv);
+                await verifyResponsesReturned([response]); // eslint-disable-line
+            });
+
+            it('should return limit from arg', async () => {
+                expect(_getLimit(10, true)).toEqual(10);
+            });
+
+            it('should return zero limit', async () => {
+                expect(_getLimit(0, true)).toEqual(0);
+            });
+
+            it('should return default limit', async () => {
+                expect(_getLimit(undefined,false)).toEqual(DEFAULTS.GET_LIMIT_RESULTS);
+            });
+
+            it('should return `unlimited` value', async () => {
+                expect(_getLimit(undefined,true)).toEqual(DEFAULTS.GET_ALL_PIPELINES_LIMIT);
+            });
+        });
+
+        // decided to follow unit tests modularity concept
+        describe('run', () => {
+            describe('cf', () => {
+                describe('preRunAll', () => {
+                    it('should handle pipeline getting on pre run phase', async () => {
+                        const argv = { name: 'some name' };
+                        const pip = new BasePipeline(argv);
+                        await pip.preRunAll();
+                        await verifyResponsesReturned([DEFAULT_RESPONSE]); // eslint-disable-line
+                    });
+
+                    it('should handle context getting on pre run phase', async () => {
+                        const argv = { name: 'some name', context: ['context'] };
+                        const pip = new BasePipeline(argv);
+                        await pip.preRunAll();
+                        await verifyResponsesReturned([DEFAULT_RESPONSE, DEFAULT_RESPONSE]); // eslint-disable-line
+                    });
+                });
+
+                describe('runImpl', () => {
+                    it('should handle running pipeline', async () => {
+                        const argv = { name: 'some name', detach: true };
+                        const pip = new CfPipeline(argv);
+                        await pip.runImpl({ pipelineName: argv.name, options: {} });
+                        await verifyResponsesReturned([DEFAULT_RESPONSE]); // eslint-disable-line
+                    });
+
+                    it('should handle running pipeline in debug mode', async () => {
+                        jest.spyOn(sdk.pipelines, 'debug').mockImplementation();
+                        const argv = { name: 'some name', detach: true };
+                        const pip = new CfPipeline(argv);
+                        await pip.runImpl({ pipelineName: argv.name, options: { debug: true } });
+                        expect(sdk.pipelines.debug).toBeCalled();
+                        sdk.pipelines.debug.mockReset();
+                    });
+
+                    it('should handle showing logs aaa', async () => {
+                        jest.spyOn(sdk.logs, 'showWorkflowLogs').mockImplementation();
+                        const argv = { name: 'some name' };
+                        const pip = new CfPipeline(argv);
+                        pip.executionRequests = ['test'];
+                        await pip.runImpl({ pipelineName: argv.name, options: {} });
+
+                        await verifyResponsesReturned([DEFAULT_RESPONSE, DEFAULT_RESPONSE]); // eslint-disable-line
+                        expect(sdk.logs.showWorkflowLogs).toBeCalled();
+                    });
+                });
+            });
+        });
+
+        describe('delete', () => {
+            it('should handle deletion given name', async () => {
+                const argv = { name: 'some name' };
+                await deleteCmd.handler(argv);
+                await verifyResponsesReturned([DEFAULT_RESPONSE]); // eslint-disable-line
+            });
+        });
+    });
+
+    describe('completions', () => {
+        describe('get', () => {
+            it('should handle getting completion', async () => {
+                const response = { statusCode: 200, body: { docs: [{ name: 'some pip' }] } };
+                request.__setResponse(response);
+                await positionalHandler({});
+                await verifyResponsesReturned([response]); // eslint-disable-line
+            });
+        });
+    });
+});

package/lib/interface/cli/commands/pipeline/run.base.js

@@ -0,0 +1,153 @@
+const _ = require('lodash');
+const Promise = require('bluebird');
+const { prepareKeyValueFromCLIEnvOption } = require('../../helpers/general');
+const { validatePipelineYaml } = require('../../helpers/validation');
+const { printResult } = require('../root/validate.cmd');
+const CFError = require('cf-errors');
+const { sdk } = require('../../../../logic');
+
+class RunBaseCommand {
+    constructor(argv) {
+        this.argv = argv;
+        this.executionRequests = [];
+    }
+
+    async run() {
+        await this.preRunRequest();
+        const {
+            debug, skip, only, trigger, cpu, memory, branch, sha, disk,
+        } = this.argv;
+        const variablesFromFile = this.argv['var-file'];
+        const pipelineName = this.argv.name;
+        const noCache = this.argv['no-cache'];
+        const runtimeName = this.argv['runtime-name'];
+        const enableNotifications = this.argv['enable-notifications'];
+        const resetVolume = this.argv['reset-volume'];
+        const userYamlDescriptor = this.argv.yaml;
+        const contexts = this.argv.context;
+        const noCfCache = this.argv['no-cf-cache'];
+        const packName = this.argv['pack-name'];
+
+        if (process.env.CF_BUILD_ID) {
+            this.argv.annotation.push(`cf_predecessor=${process.env.CF_BUILD_ID}`);
+        }
+
+        const annotations = prepareKeyValueFromCLIEnvOption(this.argv.annotation);
+        const executionRequestTemplate = {
+            pipelineName,
+            options: {
+                noCache,
+                resetVolume,
+                branch,
+                sha,
+                enableNotifications,
+                userYamlDescriptor,
+                noCfCache,
+                trigger,
+                only,
+                skip,
+                annotations,
+                debug,
+                cpu,
+                memory,
+                disk,
+                runtimeName,
+                packName,
+            },
+        };
+
+        if (variablesFromFile) {
+            _.forEach(variablesFromFile, (variables) => {
+                const request = _.cloneDeep(executionRequestTemplate);
+                request.options.variables = variables;
+                this.executionRequests.push(request);
+            });
+        } else {
+            const variables = prepareKeyValueFromCLIEnvOption(this.argv.variable);
+            const request = _.cloneDeep(executionRequestTemplate);
+            request.options.variables = variables;
+            request.options.contexts = contexts;
+            this.executionRequests.push(request);
+        }
+
+        const results = await Promise.all(this.executionRequests.map(request => this.runImpl(request)));
+        const findMaxReducer = (accumulator, currentValue) => (currentValue > accumulator ? currentValue : accumulator);
+        const exitCode = results.reduce(findMaxReducer);
+        await this.postRunRequest();
+        return exitCode;
+        // let p = Promise.resolve();
+        // this.executionRequests.forEach(request => p = p.then(() => this.runImpl(request)));
+        // await p;
+    }
+
+    // eslint-disable-next-line class-methods-use-this
+    async runImpl() {
+        throw new Error('To be implemented in the derived class');
+    }
+
+    // eslint-disable-next-line class-methods-use-this
+    async preRunAll() {
+        const pipelineName = this.argv.name;
+        const userYamlDescriptor = this.argv.yaml;
+        const contexts = this.argv.context;
+        if (!pipelineName) {
+            if (!userYamlDescriptor) {
+                throw new CFError({
+                    message: 'You need to specify pipelineName or the yaml file',
+                });
+            }
+        } else {
+            try {
+                await sdk.pipelines.get({ name: pipelineName });
+            } catch (err) {
+                if (err.statusCode === 404) {
+                    throw new CFError({
+                        message: `Passed pipeline id: ${pipelineName} does not exist`,
+                    });
+                }
+
+                throw err;
+            }
+        }
+
+        if (userYamlDescriptor) {
+            const result = await validatePipelineYaml(undefined, userYamlDescriptor);
+            printResult(result);
+            if (result && !result.valid) {
+                throw new Error(result);
+            }
+        }
+
+        if (!_.isEmpty(contexts)) {
+            await Promise.map(contexts, async (name) => { // eslint-disable-line
+                try {
+                    await sdk.contexts.get({ name });
+                } catch (err) {
+                    throw new CFError(err, `Failed to verify context ${name} with error: ${err.message}`);
+                }
+            });
+        }
+    }
+
+    // eslint-disable-next-line class-methods-use-this
+    get isParalel() {
+        return true;
+    }
+
+    // eslint-disable-next-line class-methods-use-this
+    async preRunRequest() {
+        return Promise.resolve();
+    }
+
+    // eslint-disable-next-line class-methods-use-this
+    async postRunRequest() {
+        return Promise.resolve();
+    }
+
+    // eslint-disable-next-line class-methods-use-this
+    async postRunAll() {
+        return Promise.resolve();
+    }
+}
+
+module.exports = RunBaseCommand;

package/lib/interface/cli/commands/pipeline/run.cf.js

@@ -0,0 +1,58 @@
+const RunBaseCommand = require('./run.base');
+const { sdk } = require('../../../../logic');
+const { followLogs } = require('../../helpers/logs');
+
+function _buildBody(data) {
+    const body = {
+        options: {},
+        userMachineConfig: {},
+    };
+
+    ['branch', 'variables', 'annotations', 'sha', 'userYamlDescriptor', 'contexts', 'trigger']
+        .filter(property => data[property])
+        .forEach(property => Object.assign(body, { [property]: data[property] }));
+
+    ['noCache', 'noCfCache', 'enableNotifications', 'resetVolume', 'skip', 'only']
+        .filter(property => data[property])
+        .forEach(property => Object.assign(body.options, { [property]: data[property] }));
+
+    ['cpu', 'memory', 'disk', 'runtimeName', 'packName']
+        .filter(property => data[property])
+        .forEach(property => Object.assign(body.userMachineConfig, { [property]: data[property] }));
+
+    return body;
+}
+
+class RunExternalCommand extends RunBaseCommand {
+    async runImpl(request) {
+        const { pipelineName, options = {} } = request;
+        const interfaceMethod = this._selectInterfaceMethod({
+            pipelineName,
+            options,
+        });
+        this.workflowId = await sdk.pipelines[interfaceMethod]({ name: pipelineName }, _buildBody(options));
+        if (this.executionRequests.length === 1) {
+            if (this.argv.returnWorkflowId) {
+                return this.workflowId;
+            }
+            if (this.argv.detach) {
+                console.log(this.workflowId);
+                return 0;
+            }
+            return followLogs(this.workflowId);
+        }
+        console.log(this.workflowId);
+    }
+
+    _selectInterfaceMethod({ pipelineName, options }) {
+        if (options.debug) {
+            return 'debug';
+        }
+        if (!pipelineName && options.userYamlDescriptor) {
+            return 'runYaml';
+        }
+        return 'run';
+    }
+}
+
+module.exports = RunExternalCommand;