@kumologica/sdk 3.0.0-alpha4

package/cli/commands/start.js

@@ -0,0 +1,165 @@
+/**
+ * This command should be used to start the kumologica runtime in local mode. And it will
+ * be assisting the UI on all aspects of the project development.
+ *
+ * Example:
+ * kl start ./myproject
+ * kl start ./myproject/flow.json
+ * kl start (this will look for a flow.json in the current directory)
+ *
+ * If not flow is found, the process will be exited (with code 1) for the time being.
+ * Ideally, we want to start the server and serve the Runtime API to allow the user to
+ * open/create new projects fromt he UI.
+ *
+ * The Runtime API can be found in: packages/runtime/src/runtime/lib/api/rest/index.js
+ * The actual runtime is in: packages/runtime/src/runtime/lib/index.js
+ *
+ * Naming convention used during the code:
+ * - AdminApp (express app) is the server that serves the Runtime API
+ * - NodeApp (express app) is the server that runs the flow
+ *
+ * WARNING:
+ * Current implementation seems to be mounting the runtime api in the NodeApp (see runtime/lib/index.js:534)
+ * Ideally we wouldl like to separate the two apps and have the AdminApp running on a different port.
+ *
+ */
+
+const path = require('path');
+const fs = require('fs');
+const { codegen } = require('@kumologica/builder');
+const { DesignerServer } = require('../../src/server/DesignerServer');
+const { logError, logNotice, logInfo, logFatal } = require('../utils/logger');
+// const opn = require('better-opn');
+
+exports.command = 'start [project_directory]';
+exports.desc = `Run kumologica runtime in local mode`;
+
+exports.builder = (yargs) => {
+  yargs.positional(`project_directory`, {
+    type: 'string',
+    describe:
+      'Path to a valid kumologica project directory or flow file. (Optional)',
+  });
+  yargs.option(`loglevel`, {
+    describe: 'Logging level: [error, warn, info, debug, trace]',
+    type: 'string',
+    nargs: 1,
+  });
+  yargs.option(`port`, {
+    describe: 'Specifies the listening port utilized by the application',
+    type: 'number',
+    nargs: 1,
+  });
+  yargs.option(`adminport`, {
+    describe: 'Specifies the listening port utilized by the admin API',
+    type: 'number',
+    nargs: 1,
+  });
+  yargs.option(`secured`, {
+    describe: 'Enable security control to the admin API',
+    type: 'boolean',
+    nargs: 1,
+  });
+  yargs.option(`noadmin`, {
+    describe: 'Start the runtime with admin API disabled',
+    type: 'boolean',
+    nargs: 0,
+  });
+};
+
+exports.desc = 'Starting Kumologica Runtime for Development';
+
+exports.handler = ({
+  project_directory,
+  loglevel,
+  port,
+  adminport,
+  secured,
+  noadmin,
+  env
+}) => {
+  logNotice(`Launching Kumologica Runtime...`);
+  let absProjectDirectory;
+
+  try {
+    absProjectDirectory = fs.realpathSync(project_directory);
+  } catch (err) {
+    logFatal(`Project directory not found: ${project_directory}`);
+  }
+
+  try {
+    // Resolve the path to the flow path
+    const [projectFlowDirname, projectFlowFullPath] = resolveProjectFlowPath(absProjectDirectory);
+    logInfo(`> Flow file found: ${projectFlowFullPath}`);
+
+    // Gather all cli params
+    const cliParams = {
+      env,
+      loglevel,
+      port,
+      adminport,
+      secured,
+      noadmin
+    }
+
+    // Start a server 
+    let server = new DesignerServer(projectFlowFullPath, true, cliParams);
+    server.listen();
+    if (!noadmin) {
+      server.listenAdminServer();
+    }
+
+    // opn("http://localhost:1880/hello");
+  } catch (e) {
+    console.log(e);
+    logFatal(e.message);
+
+  }
+};
+
+/**
+ * Resolve a given path to a new path that points to the project flow json file.
+ * Scenarios:
+ *  - If path undefined will resolve using current directory
+ *  - If path directory, it should look up for a valid flow.json file, if it does not exist create it
+ *  - If path is pointing to an actual flowFile, just return it.
+ *
+ * @param {string} projectDir
+ */
+function resolveProjectFlowPath(projectDir) {
+  // output
+  let projectFlowFullPath;
+  let projectFlowDirname;
+
+  let projectDirOrFile = projectDir || process.cwd();
+
+  let isDir = isDirectory(projectDirOrFile);
+  if (isDir) {
+    let flowFileName = codegen.findFlowFile(projectDirOrFile); // returns only the flowname
+
+    if (!flowFileName) {
+      logError(`No flow found in directory: ${projectDirOrFile}`);
+      process.exit(1);
+    } else {
+      projectFlowDirname = projectDirOrFile;
+      projectFlowFullPath = path.join(projectDirOrFile, flowFileName);
+    }
+  } else if (isDir === false) {
+    projectFlowDirname = path.dirname(projectDirOrFile);
+    projectFlowFullPath = projectDirOrFile;
+  } else {
+    logError(`Directory does not exist: ${project_directory}`);
+    process.exit(1);
+  }
+
+  return [projectFlowDirname, projectFlowFullPath];
+}
+
+function isDirectory(dir) {
+  try {
+    let stats = fs.statSync(dir);
+    return stats.isDirectory();
+  } catch (err) {
+    return undefined;
+  }
+}

package/cli/commands/test/TestSuiteRunner.js

@@ -0,0 +1,76 @@
+const { performance } = require('perf_hooks');
+const { InMemoryReporter, TerminalReporter } = require("../../../src/api/tools/test/lib/reporters");
+const { TestCaseRunner } = require('../../../src/api/tools/test/lib/TestCaseRunner');
+const { logError, logNotice, logInfo, logFatal } = require('../../utils/logger');
+
+
+class TestSuiteRunner {
+    constructor(designerServer) {
+        this.designerServer = designerServer;
+        this.flowServer = this.designerServer.getFlowServer();
+    }
+    /**
+     * 
+     * @param {*} testcases 
+     * return an array containing found errors during execution, or undefined if successful
+     */
+    async runAll(testcases) {
+        logInfo(`TestCases - Executing\n`);
+        let testSuiteTimeStart = performance.now();
+
+        let totalPassedTestCases = 0;
+        let totalFailedCases = 0;
+        const totalTestCases = testcases.length;
+        for (let i = 0; i <= totalTestCases - 1; i++) {
+            let success = await this.runTestCase(testcases[i]);
+            if (success) {
+                totalPassedTestCases = totalPassedTestCases + 1;
+            } else {
+                totalFailedCases = totalFailedCases + 1;
+            }
+        }
+        let testSuiteTimeEnd = performance.now();
+        let testSuiteExecutionTimeInMs = testSuiteTimeEnd - testSuiteTimeStart;
+        this.printSummary(testSuiteExecutionTimeInMs, totalTestCases, totalFailedCases, totalPassedTestCases);
+        return 0;
+    }
+
+    /**
+     * 
+     * @param {*} testcase 
+     * @returns true if success, otherwise false
+     */
+    async runTestCase(testcase) {
+        let testcaseid = testcase.id;
+
+        const inMemReporter = new InMemoryReporter;
+        const terminalReporter = new TerminalReporter;
+
+        const testcaseRunner = new TestCaseRunner(this.flowServer, testcaseid, [inMemReporter, terminalReporter]);
+        try {
+            await testcaseRunner.runAsync();
+            return !inMemReporter.isFailedStatus()
+        } catch (err) {
+            console.log(err);
+            logFatal(`Unexpected error occurred while running testcase: "${testcase.name}" due to: ${err.message}.`)
+        }
+    }
+
+    printSummary(totalExecutionTimeInMs, totalTestCases, totalFailedCases, totalPassedTestCases) {
+        logInfo(`TestCases - Completed
+    
+Final Execution Statistics
+-------------------------------
+ Execution Time: ${Math.floor(totalExecutionTimeInMs)} ms
+ Total Test Cases: ${totalTestCases}
+ Failed: ${totalFailedCases}
+ Passed: ${totalPassedTestCases}
+-------------------------------
+        `);
+    }
+
+}
+
+module.exports = {
+    TestSuiteRunner
+}

package/cli/commands/test.js

@@ -0,0 +1,123 @@
+const path = require('path');
+const { Select } = require('enquirer');
+const wcmatch = require('wildcard-match');
+const { util } = require('@kumologica/runtime');
+const { codegen } = require('@kumologica/builder');
+
+const { TestSuiteRunner } = require('./test/TestSuiteRunner');
+const { DesignerServer } = require('../../src/server/DesignerServer');
+const { logError, logNotice, logInfo, logFatal } = require('../utils/logger');
+
+const isDirectory = util.isDirectorySync;
+
+
+exports.command = 'test [project_directory]';
+exports.desc = `Run test suite`;
+exports.builder = (yargs) => {
+  yargs.positional(`project_directory`, {
+    type: 'string',
+    describe: 'Path to a valid kumologica project directory or flow file. (Optional)'
+  })
+  yargs.option(`testcase`, {
+    describe: "Testcase name to run",
+    type: 'string',
+    alias: 't',
+    nargs: 1
+  });
+  yargs.option(`iterative`, {
+    describe: "Manually select the testcase to run from all available testcases",
+    type: 'boolean',
+    alias: 'i'
+  });
+}
+exports.handler = ({ project_directory, testcase, iterative }) => {
+  let projectDirOrFile = project_directory || process.cwd();
+  let projectFlowPath = projectDirOrFile;
+
+  let isDir = isDirectory(projectDirOrFile);
+  if (isDir) {
+    let flowFileName = codegen.findFlowFile(projectDirOrFile); // returns only the flowname
+    if (!flowFileName) {
+      logFatal(`No flow found in directory: ${projectDirOrFile}`);
+    } else {
+      projectFlowPath = path.join(projectDirOrFile, flowFileName);
+    }
+  } else if (isDir === false) {
+    // do nothing as it was assumed to be a file
+  } else {
+    logFatal(`Directory does not exist: ${project_directory}`);
+  }
+
+  try {
+    runTestOnNewServer(projectFlowPath, testcase, iterative);
+  } catch (e) {
+    logFatal(e.message);
+  }
+}
+
+async function runTestOnNewServer(flowFilePath, testcaseSelected, iterative) {
+  let designerServer = new DesignerServer(
+    flowFilePath,
+    false,
+    {
+      loglevel: "error",
+      noadmin: true
+    });
+
+  try {
+    await designerServer.listen();
+    logInfo(`> Flow file: ${path.resolve(flowFilePath)} \n`);
+    let testSuiteRunner = new TestSuiteRunner(designerServer);
+
+    // If testcase is null, default to universal wildcard
+    testcaseSelected = testcaseSelected || "**";
+
+    // Find out all testcases available on the flow
+    let testcasesAvailable = codegen.findTestCasesFromFlow(flowFilePath);
+    if (!testcasesAvailable || (testcasesAvailable && testcasesAvailable.length === 0)) {
+      logFatal(`No testcases found on flow file: ${flowFileAbsPath}`);
+    };
+    let testcaseAvailableNames = testcasesAvailable.map(tc => tc.name);
+
+    // Capture the testcase from user on iterative mode
+    if (iterative) {
+      const prompt = new Select({
+        name: 'testcase',
+        message: 'What testcase do you want to run?',
+        choices: [...testcaseAvailableNames, 'Run all TestCases...']
+      });
+      await prompt.run()
+        .then(tc => {
+          if (tc === 'Run all TestCases...') {
+            testcaseSelected = "**";
+          } else {
+            testcaseSelected = tc;
+          }
+        })
+        .catch(err => {
+          logFatal(`Error found while running tests on iterative mode due to: `, err.message);
+        });
+    }
+
+    // Filter all testcases to be part of the test suite
+    const isMatch = wcmatch(testcaseSelected);
+    let testCasesSelected = [];
+    testcasesAvailable.forEach(async tc => {
+      if (isMatch(tc.name)) {
+        testCasesSelected.push({ name: tc.name, id: tc.id });
+      }
+    });
+
+    // Execute the testcasesIds if any, otherwise throw an error
+    if (testCasesSelected.length === 0) {
+      logFatal(`No matched testcases found`);
+    } else {
+      const errors = await testSuiteRunner.runAll(testCasesSelected);
+      process.exit(errors > 0);
+    }
+
+  } catch (err) {
+    console.log(err);
+    logFatal(`Unexpected error occurred while starting server due to <${err.message}>`);
+  }
+}

package/cli/utils/download-template-from-repo.js

@@ -0,0 +1,346 @@
+'use strict';
+
+const path = require('path');
+const URL = require('url');
+const BbPromise = require('bluebird');
+const fse = require('fs-extra');
+const qs = require('querystring');
+const spawn = require('child-process-ext/spawn');
+const renameService = require('./rename-service').renameService;
+const KumologicaError = require('../KumologicaError');
+const copyDirContentsSync = require('./fs/copy-dir-contents-sync');
+const dirExistsSync = require('./fs/dir-exists-sync');
+const walkDirSync = require('./fs/walk-dir-sync');
+const { getTmpDirPath, getBaseTmpDirPath } = require('./fs/get-tmp-dir-path');
+
+
+const OFFICAL_TEMPLATES_REPO = 'https://github.com/KumologicaHQ/kumologica-templates.git';
+/**
+ * Returns directory path
+ * @param {Number} length
+ * @param {Array} parts
+ * @returns {String} directory path
+ */
+function getPathDirectory(length, parts) {
+  if (!parts) {
+    return '';
+  }
+  return parts.slice(length).filter(Boolean).join(path.sep);
+}
+
+/**
+ * Validates URL
+ * @param {Object} url
+ * @param {String} hostname
+ * @param {String} service
+ * @param {String} owner
+ * @param {String} repo
+ */
+function validateUrl({ url, hostname, service, owner, repo }) {
+  // validate if given url is a valid url
+  if (url.hostname !== hostname || !owner || !repo) {
+    const errorMessage = `The URL must be a valid ${service} URL in the following format: https://${hostname}/serverless/serverless`;
+    throw new KumologicaError(errorMessage, 'INVALID_TEMPLATE_URL');
+  }
+}
+
+/**
+ * Check if the URL is pointing to a Git repository
+ * @param {String} url
+ */
+function isPlainGitURL(url) {
+  return (url.startsWith('https') || url.startsWith('git@')) && url.endsWith('.git');
+}
+
+/**
+ * @param {Object} url
+ * @returns {Object}
+ */
+function parseGitHubURL(url) {
+  const pathLength = 4;
+  const parts = url.pathname.split('/');
+  const isSubdirectory = parts.length > pathLength;
+  const owner = parts[1];
+  const repo = parts[2];
+  const branch = isSubdirectory ? parts[pathLength] : 'master';
+  const isGitHubEnterprise = url.hostname !== 'github.com';
+
+  if (!isGitHubEnterprise) {
+    // validate if given url is a valid GitHub url
+    validateUrl({ url, hostname: 'github.com', service: 'GitHub', owner, repo });
+  }
+
+  const downloadUrl = `https://${
+    isGitHubEnterprise ? url.hostname : 'github.com'
+  }/${owner}/${repo}/archive/${branch}.zip`;
+
+  return {
+    owner,
+    repo,
+    branch,
+    downloadUrl,
+    isSubdirectory,
+    pathToDirectory: getPathDirectory(pathLength + 1, parts),
+    username: url.username || '',
+    password: url.password || '',
+  };
+}
+
+/**
+ * @param {Object} url
+ * @returns {Object}
+ */
+function parseBitbucketURL(url) {
+  const pathLength = 4;
+  const parts = url.pathname.split('/');
+  const isSubdirectory = parts.length > pathLength;
+  const owner = parts[1];
+  const repo = parts[2];
+
+  const query = qs.parse(url.query);
+  const branch = 'at' in query ? query.at : 'master';
+
+  // validate if given url is a valid Bitbucket url
+  validateUrl({ url, hostname: 'bitbucket.org', service: 'Bitbucket', owner, repo });
+
+  const downloadUrl = `https://bitbucket.org/${owner}/${repo}/get/${branch}.zip`;
+
+  return {
+    owner,
+    repo,
+    branch,
+    downloadUrl,
+    isSubdirectory,
+    pathToDirectory: getPathDirectory(pathLength + 1, parts),
+    username: url.username || '',
+    password: url.password || '',
+  };
+}
+
+function parseBitbucketServerURL(url) {
+  const pathLength = 9;
+  const parts = url.pathname.split('/');
+  const isSubdirectory = parts.length > pathLength;
+  const owner = parts[5];
+  const repo = parts[7];
+
+  const query = qs.parse(url.query);
+  const branch = 'at' in query ? decodeURIComponent(query.at) : 'master';
+
+  const downloadUrl = `${url.protocol}//${url.hostname}/rest/api/latest/projects/${owner}/repos/${repo}/archive${url.search}&format=zip`;
+
+  return {
+    owner,
+    repo,
+    branch,
+    downloadUrl,
+    isSubdirectory,
+    pathToDirectory: getPathDirectory(pathLength + 1, parts),
+    username: url.username || '',
+    password: url.password || '',
+  };
+}
+
+/**
+ * Call `/rest/api/1.0/application-properties` to retrieve server info
+ * @param {Object} url
+ * @returns {Boolean}
+ */
+async function retrieveBitbucketServerInfo(url) {
+  let requestOpts = {
+    url: `${url.protocol}//${url.hostname}/rest/api/1.0/application-properties`,
+    method: 'GET',
+    throwHttpErrors: false,
+    retry: 0
+  }
+  try {
+    let resp = await got(requestOpts)
+    return resp.body.displayName === 'Bitbucket';
+  }catch(err){
+    throw err;
+  }
+  
+}
+
+/**
+ * @param {Object} url
+ * @returns {Object}
+ */
+function parseGitlabURL(url) {
+  const pathLength = 4;
+  const parts = url.pathname.split('/');
+  const isSubdirectory = parts.length > pathLength;
+  const owner = parts[1];
+  const repo = parts[2];
+
+  const branch = isSubdirectory ? parts[pathLength] : 'master';
+
+  // validate if given url is a valid GitLab url
+  validateUrl({ url, hostname: 'gitlab.com', service: 'Bitbucket', owner, repo });
+
+  const downloadUrl = `https://gitlab.com/${owner}/${repo}/-/archive/${branch}/${repo}-${branch}.zip`;
+
+  return {
+    owner,
+    repo,
+    branch,
+    downloadUrl,
+    isSubdirectory,
+    pathToDirectory: getPathDirectory(pathLength + 1, parts),
+    username: url.username || '',
+    password: url.password || '',
+  };
+}
+
+/**
+ * Parses a URL which points to a plain Git repository
+ * such as https://example.com/jdoe/project.git
+ *
+ * @param {String} url
+ * @returns {Object}
+ */
+function parsePlainGitURL(url) {
+  const branch = 'master';
+  const downloadUrl = url;
+  const isSubdirectory = false;
+  const repo = url.match(/.+\/(.+)\.git/)[1];
+  return {
+    repo,
+    branch,
+    downloadUrl,
+    isSubdirectory,
+    username: url.username || '',
+    password: url.password || '',
+  };
+}
+
+/**
+ * Parse URL and call the appropriate adaptor
+ *
+ * @param {string} inputUrl
+ * @throws {KumologicaError}
+ * @returns {Promise}
+ */
+async function parseRepoURL(inputUrl) {
+  return new BbPromise((resolve, reject) => {
+    if (!inputUrl) {
+      return reject(new KumologicaError('URL is required', 'MISSING_TEMPLATE_URL'));
+    }
+
+    const url = URL.parse(inputUrl.replace(/\/$/, ''));
+    if (url.auth) {
+      const [username, password] = url.auth.split(':');
+      url.username = username;
+      url.password = password;
+    }
+
+    // check if url parameter is a valid url
+    if (!url.host && !url.href.startsWith('git@')) {
+      return reject(new KumologicaError('The URL you passed is not valid', 'INVALID_TEMPLATE_URL'));
+    }
+
+    if (isPlainGitURL(url.href)) {
+      return resolve(parsePlainGitURL(inputUrl));
+    } else if (url.hostname === 'github.com' || url.hostname.indexOf('github.') !== -1) {
+      return resolve(parseGitHubURL(url));
+    } else if (url.hostname === 'bitbucket.org') {
+      return resolve(parseBitbucketURL(url));
+    } else if (url.hostname === 'gitlab.com') {
+      return resolve(parseGitlabURL(url));
+    }
+
+    const msg =
+      'The URL you passed is not one of the valid providers: "GitHub", "GitHub Entreprise", "Bitbucket", "Bitbucket Server" or "GitLab".';
+    const err = new KumologicaError(msg, 'INVALID_TEMPLATE_PROVIDER');
+    // test if it's a private bitbucket server
+    return retrieveBitbucketServerInfo(url)
+      .then((isBitbucket) => {
+        if (!isBitbucket) {
+          return reject(err);
+        }
+
+        // build download URL
+        return resolve(parseBitbucketServerURL(url));
+      })
+      .catch(() => reject(err));
+  });
+}
+
+/**
+ * Downloads the content of the template into the projectName directory
+ * @param {string} [repoUrl] - the git repository
+ * @param {string} [templateName] - the name of the project (optional), in case we are only interested in a particular directory within the repo
+ * @param {string} projectName - the local project name / directory to be created
+ * @returns {Promise}
+ */
+async function downloadTemplateFromRepo(repoUrl, templateName, projectPath, projectName) {
+  try{
+    repoUrl = repoUrl || OFFICAL_TEMPLATES_REPO;
+    const repoInformation = await parseRepoURL(repoUrl);
+    
+    const tempBaseDirectory = getBaseTmpDirPath();
+    // clean up temp directory
+    fse.removeSync(tempBaseDirectory);
+    const tempRepoDirectory = getTmpDirPath();
+    // const { username, password } = repoInformation;
+
+    const renamed = templateName !== projectName;
+
+    // Source project directory
+    const srcProjectDir = templateName? path.join(tempRepoDirectory, templateName) : tempRepoDirectory;
+
+    // Target project directory
+    const targetProjectDir = path.join(projectPath, projectName);
+
+    if (dirExistsSync(targetProjectDir)) {
+      const errorMessage = `A project already exist in path: "${targetProjectDir}".`;
+      throw new KumologicaError(errorMessage, 'TARGET_FOLDER_ALREADY_EXISTS');
+    }
+
+    if (isPlainGitURL(repoUrl)) {
+      return spawn('git', ['clone', repoUrl, tempRepoDirectory]).then(() => {
+        try {
+          copyDirContentsSync(srcProjectDir, targetProjectDir);
+          fse.removeSync(tempBaseDirectory);
+          if (renamed) renameService(targetProjectDir, projectName);
+        }catch(err){
+          throw new KumologicaError(`Template name: "${templateName}" does not exist. Run "kl list-templates" to see list of available templates.`);
+        }
+        return "";
+      });
+    } else {
+      throw new KumologicaError(`Git repo URL is incorrect`);
+    }
+  }catch(err){
+    throw new KumologicaError(err.message);
+  }
+}
+
+async function listAvailableTemplates() {
+  try{
+    const tempBaseDirectory = getBaseTmpDirPath();
+    const tempRepoDirectory = getTmpDirPath();
+
+    await parseRepoURL(OFFICAL_TEMPLATES_REPO);
+
+    const listTemplates = spawn('git', ['clone', OFFICAL_TEMPLATES_REPO, tempRepoDirectory]).then(() => {
+        try {
+          const projects = walkDirSync(tempRepoDirectory);
+          fse.removeSync(tempBaseDirectory);
+          return projects;
+        }catch(err){
+          throw new KumologicaError(err.message);
+        }
+      });
+      // ListTemplates: ["t1", "t2"]
+      return listTemplates;
+  } catch(err){
+    throw new KumologicaError(err.message);
+  }
+}
+
+module.exports = {
+  downloadTemplateFromRepo,
+  listAvailableTemplates,
+  isPlainGitURL
+};