@kumologica/sdk 3.0.26 → 3.0.27-beta10

package/cli/commands/export-commands/cloudformation.js

@@ -125,6 +125,11 @@ exports.builder = {
 		type: 'string',
 		nargs: 1
 	},
+	"layers": {
+		describe: `\nList of arn of lambda layers to be added.`,
+		type: 'string',
+		nargs: 1
+	},
 	"policy": {
 		describe: `\nAdditional iam policy to add to the role. It must be in valid aws policy format, for example: '{"Effect": "Allow", "Resource": ["*"],"Action": ["sns:Publish"]}'. if policy is invalid, cloudformation stack will fail. The value can be single policy statement or array of statements if required. This parameter is ignored if role-arn is provided.`,
 		type: 'string',
@@ -156,12 +161,18 @@ Examples:
 3. Creates event trigger using rate expression that runs every minute for event listener with reference "cleanup":\n
 			{"event": {"expression": "rate(1 minute)", "reference": "cleanup"}}\n
 
+SNS:\n----
+			* topicArn - the ARN of sns topic, required
+
+Example:
+			{"sns": {"topicArn": "arn:aws:sns: ..."}}
+
 SQS:\n----
 			* queueArn - the ARN of sqs queue, required
 			* batchSize - the size of batch, optional
 
 Example:
-			{"sqs": {"queueArn": "arn:aws: ..."}}
+			{"sqs": {"queueArn": "arn:aws:sqs: ..."}}
 
 DynamoDB Stream:\n----
 			* streamArn - the ARN of dynamodb stream, required
@@ -265,6 +276,7 @@ function display(argv) {
 	console.log('  log-retention-days:  %s', argv["log-retention-days"]||'');
 	console.log('  role-arn:            %s', argv["role-arn"]||'');
 	console.log('  policy:              %s', argv["policy"] || '');
+	console.log('  layers:              %s', argv["layers"] || '');
 	console.log(`  strict-mode:         ${argv["strict-mode"]}`);
 	console.log('  triggers:            %s', argv.triggers||'');	
 }
@@ -309,6 +321,9 @@ function validate(argv) {
 	if (argv["file-system-configs"]) {
 		validateJson("file-system-configs", argv["file-system-configs"]);
 	}
+	if (argv["layers"]) {
+		validateJson("layers", argv["layers"]);
+	}
 	if (argv.triggers) {
 		validateJson("triggers", argv.triggers);
 		validator.validate("triggers", JSON.parse(argv.triggers));

package/cli/commands/export-commands/utils/validator.js

@@ -58,6 +58,23 @@ const triggersSchema = {
 					}
 				}
 			},
+			"sns": {
+				"$id": "#root/items/sns", 
+				"title": "Sns", 
+				"type": "object",
+				"required": [
+					"topicArn"
+				],
+				additionalProperties: false,
+				"properties": {
+					"topicArn": {
+						"$id": "#root/items/sns/topicArn", 
+						"title": "Topicarn", 
+						"type": "string",
+						"pattern": "^.*$"
+					}
+				}
+			},
 			"api": {
 				"$id": "#root/items/api", 
 				"title": "Api", 

package/cli/commands/test-utils/TestSuiteController.js

@@ -309,27 +309,12 @@ class TestSuiteController {
     return flowFileJson.filter((node) => node.type === 'TestCase' || node.type === 'HTTPTestCase');
   }
 
-  async runTestSuite(testcase) {
-    const flowFileAbsPath = path.join(
-      this._appServer.settings.userDir,
-      this._appServer.settings.flowFile
-    );
-    let testCases = this.findTestCasesFromFlow(flowFileAbsPath);
-    if (!testCases || (testCases && testCases.length === 0)) {
-      logError(`No testcases found on flow file: ${flowFileAbsPath}`);
-      process.exit(1);
-    }
-
-    // If defined, filter the testcase that you want to test
-    if (testcase) {
-      testCases = testCases.filter(tc => tc.name.toLowerCase() === testcase.toLowerCase());
-      if (testCases && testCases.length === 0){
-        logError(`TestCase: "${testcase}" cannot be found`);
-        process.exit(1);
-      }
-      
-    }
-
+  /**
+   * 
+   * @param {*} testcasesSelected - array of {id: nodeId, name: nodeName}
+   * @returns 
+   */
+  async runTestSuite(testcasesSelected) {
     // === Starting the testSuite ===
     this._appServer.events.emit('runtime-event', {
       id: runtimeEvents.TEST_TESTSUITE_START,
@@ -337,7 +322,7 @@ class TestSuiteController {
     });
 
     // === Iterate over all the testCases ===
-    await this.runTestCases(testCases);
+    await this.runTestCases(testcasesSelected);
     await this.waitForResults();
     const errorsNum = this._testReporter.getFailedTestCases().length;
 

package/cli/commands/test.js

@@ -3,6 +3,7 @@ const { Select } = require('enquirer');
 const fs = require('fs');
 const { logError } = require('./utils');
 const path = require('path');
+const wcmatch = require('wildcard-match');
 
 const { AppServer } = require('@kumologica/runtime');
 const { TestSuiteController } = require('./test-utils/TestSuiteController');
@@ -11,7 +12,7 @@ const { codegen } = require('@kumologica/builder');
 const log = console.log;
 const APP_SERVER_PORT = 1990;
 		
-async function runTest(flowFilePath, testcase, iterative) {
+async function runTest(flowFilePath, testcaseSelected, iterative) {
     log(`\n> Starting runtime on port ${APP_SERVER_PORT}...`);
   
     let appServer = new AppServer({
@@ -28,24 +29,31 @@ async function runTest(flowFilePath, testcase, iterative) {
       await appServer.start();
       log(`> FlowFile to be tested: ${chalk.bold(path.resolve(flowFilePath))} \n`);
       let testSuiteController = new TestSuiteController(appServer);
-      let testcasesAvailable = testSuiteController.findTestCasesFromFlow(flowFilePath).map(tc => {
-        return tc.name;
-      })
-      testcasesAvailable.push('Run all TestCases...');
-      // Iterative ?
-        // Check if iterative testing has been selected
+      
+      // If testcase is null, default to universal wildcard
+      testcaseSelected  = testcaseSelected || "**";
+
+      // Find out all testcases available on the flow
+      let testcasesAvailable = testSuiteController.findTestCasesFromFlow(flowFilePath);
+      if (!testcasesAvailable || (testcasesAvailable && testcasesAvailable.length === 0)) {
+        logError(`No testcases found on flow file: ${flowFileAbsPath}`);
+        process.exit(1);
+      };
+      let testcaseAvailableNames = testcasesAvailable.map(tc=>tc.name);
+
+      // Capture the testcase from user on iterative mode
       if (iterative) {
         const prompt = new Select({
           name: 'testcase',
           message: 'What testcase do you want to run?',
-          choices: testcasesAvailable
+          choices: [...testcaseAvailableNames, 'Run all TestCases...']
         });
         await prompt.run()
           .then(tc => {
             if (tc === 'Run all TestCases...'){
-              testcase = null;
+              testcaseSelected = "**";
             }else {
-              testcase = tc;
+              testcaseSelected = tc;
             }
           })
           .catch(err => {
@@ -54,15 +62,31 @@ async function runTest(flowFilePath, testcase, iterative) {
           });
       }
 
+      // Filter all testcases to be part of the test suite
+      const isMatch = wcmatch(testcaseSelected);
+      let testCasesSelected = [];
+      testcasesAvailable.forEach(async tc => {
+        if (isMatch(tc.name)){
+          testCasesSelected.push({ name: tc.name, id: tc.id });
+        }
+      });
+
+      // Execute the testcasesIds if any, otherwise throw an error
+      if (testCasesSelected.length === 0){
+        logError(`No matched testcases found`);
+        process.exit(1);
+      } else {
+        const errors = await testSuiteController.runTestSuite(testCasesSelected); 
+        process.exit(errors > 0);
+      }
       
-      const errors = await testSuiteController.runTestSuite(testcase); // no need to wait for response
-      process.exit(errors > 0);
     } catch (err) {
       log(
         chalk.red(
           `Unexpected error occurred while starting server due to <${err.message}>`
         )
       );
+      console.log(err);
       process.exit(1);
     }
   }

package/package.json

@@ -3,7 +3,7 @@
   "productName": "Kumologica Designer",
   "copyright": "Copyright 2020 Kumologica Pty Ltd, All Rights Reserved.",
   "author": "Kumologica Pty Ltd <contact@kumologica.com>",
-  "version": "3.0.26",
+  "version": "3.0.27-beta10",
   "description": "Kumologica Designer, harnessing Serverless for your cloud integration needs",
   "main": "src/app/main.js",
   "files": [
@@ -65,9 +65,9 @@
   "license": "Proprietary",
   "dependencies": {
     "@electron/remote": "^2.0.8",
-    "@kumologica/builder": "3.0.26",
-    "@kumologica/devkit": "3.0.26",
-    "@kumologica/runtime": "3.0.26",
+    "@kumologica/builder": "3.0.27-beta10",
+    "@kumologica/devkit": "3.0.27-beta10",
+    "@kumologica/runtime": "3.0.27-beta10",
     "adm-zip": "0.4.13",
     "ajv": "8.10.0",
     "aws-sdk": "2.513.0",
@@ -111,12 +111,15 @@
     "request": "2.88.0",
     "rimraf": "3.0.0",
     "shell-path": "2.1.0",
+    "simple-git": "^3.14.1",
     "simplemde": "1.11.2",
     "smoketail": "0.2.2",
     "socket.io": "2.3.0",
     "tcp-port-used": "1.0.2",
     "util": "0.12.1",
     "when": "3.7.8",
+    "wide-align": "^1.1.5",
+    "wildcard-match": "^5.1.2",
     "ws": "7.1.1",
     "xterm": "4.1.0",
     "xterm-addon-fit": "0.2.1",

package/src/app/lib/aws/cf.js

@@ -196,8 +196,7 @@ class AWSCFTemplate {
           
           } else if (nodes[i].type === 'Lambda') {
               //arn:aws:lambda:ap-southeast-2:174450237637:function:kumologica-deployments-flow-lambda
-              console.log(`lambda ${JSON.stringify(lambda)} node ${JSON.stringify(nodes[i])}`)
-              let lambdaArn = this.handleValue(lambda, nodes[i].LambdaArn);
+               let lambdaArn = this.handleValue(lambda, nodes[i].LambdaArn);
               let lambdaParts = lambdaArn.split(":");
               let lambdaName = '*';
               
@@ -220,10 +219,23 @@ class AWSCFTemplate {
             this.handlePolicy(lambdaRole, `KLSES`, `ses:${nodes[i].operation}`, '*');
 
           } else if (nodes[i].type === 'SSM') {
-            const key = this.handleValue(lambda, nodes[i].Key);
-            const ssmArn =  { "Fn::Sub": `arn:aws:ssm:\${AWS::Region}:\${AWS::AccountId}:parameter/${key}` };
-            this.handlePolicy(lambdaRole, `KLSSM${key}`, `ssm:${nodes[i].operation}`, ssmArn);
+            let key;
+            if (nodes[i].operation === "GetParametersByPath") {
+              key = this.handleValue(lambda, nodes[i].Path);
+            } else {  
+              key = this.handleValue(lambda, nodes[i].Key);
+            }
+
+            if (key.startsWith('/')) {
+              key = key.substring(1);
+            }
+            const ssmArn = {'Fn::Sub': 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/' + key};  
+            this.handlePolicy(lambdaRole, `KLSSM${new Date().valueOf()}`, `ssm:${nodes[i].operation}`, ssmArn);
 
+            if (nodes[i].operation === "GetParameter" && nodes[i].Key.startsWith("/aws/reference/secretsmanager/")) {
+              const smArn = {'Fn::Sub': 'arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:*'};  
+              this.handlePolicy(lambdaRole, `KLSM${new Date().valueOf()}`, "secretsmanager:GetSecretValue", smArn);
+            }
           } else if (nodes[i].type === 'S3') {
             let bucketName = this.handleValue(lambda, nodes[i].Bucket);
             const bucketArn = `arn:aws:s3:::${bucketName}/*`;
@@ -260,6 +272,7 @@ class AWSCFTemplate {
     if (!params.role) {
       template.Resources['LambdaRole'] = lambdaRole;
     }
+
     return template;
   }
 
@@ -279,7 +292,6 @@ class AWSCFTemplate {
   // to determine the value 
   handleValue(lambda, key) {
     let value;
-    console.log(`key= ${key}`);
 
     if (!key) {
       return key;
@@ -287,7 +299,6 @@ class AWSCFTemplate {
     let keyValue = key.replace(/\s+/g, '');
 
     if (keyValue.startsWith("env.")) {
-    console.log(`keyValue= ${keyValue}`);
       keyValue = keyValue.replace("env.", "");
       value = lambda.Properties.Environment.Variables[keyValue];
       if (!value) {
@@ -316,7 +327,6 @@ class AWSCFTemplate {
   // - updates existing resource policy (if found) with new permissions 
   //
   handlePolicy(lambdaRole, policyName, operation, id) {
-    console.log(`policyName: ${policyName}`);
     policyName = policyName.replace('/', '-').replace('*', 'all');
     let policy = jp.query(lambdaRole.Properties, `Policies[?(@.PolicyName=='${policyName}')]`);    
     
@@ -373,7 +383,7 @@ class AWSCFTemplate {
       throw new Error(`Missing Trigger Parameter: SQS url`);
     }
     return {
-      PolicyName: 'KumologicaLambdaSQSPolicy',
+      PolicyName: 'KLSQSPolicy'+process.hrtime(),
       PolicyDocument: {
         Version: '2012-10-17',
         Statement: [

package/src/app/lib/aws/index.js

@@ -241,7 +241,7 @@ class AWSDeployer {
       );
 
       flowListeners = this.processFlow(nodes);
-
+  
       const stackDetails = await this.executeStack(
         settings,
         params,
@@ -278,7 +278,7 @@ class AWSDeployer {
         response = await this.runTriggers(params, settings, lambdaArn);
         if (response) {
           const url = `https://${response.apiId}.execute-api.${AWS.config.region}.amazonaws.com/${response.stage}`;
-          this.printSignature(url, flowListeners);
+          this.printSignature(url, flowListeners, params);
         }
       } catch (error) {
         this.log(`${this.chalk('redBright', 'Trigger creation failed.')}`);
@@ -288,8 +288,9 @@ class AWSDeployer {
     return response;
   }
 
-  printSignature(url, s) {
+  printSignature(url, s, p) {
     this.log(' ')
+    
     if (s.api && s.api.length > 0) {
       this.log('API Gateway:');
       s.api.forEach(a => this.log(`  ${('    ' + a.verb.toUpperCase()).slice(-6)}  ${url}${a.url}`), this);
@@ -326,10 +327,13 @@ class AWSDeployer {
       this.log('');
     }
 
-    if (s.sqs && s.sqs.length > 0) {
-      this.log('SQS URL:');
-      s.sqs.forEach(a => this.log(` ${a.queueUrl}`), this);
-      this.log('');
+    if (p.events && p.events.length > 0) {
+      const sqs = p.events.filter(i => i.source == 'sqs');
+      if (sqs && sqs.length > 0) {
+        this.log('SQS URL:');
+        sqs.forEach(a => this.log(` ${a.url}`), this);
+        this.log('');
+      }
     }
 
     if (s.alexa && s.alexa.length > 0) {
@@ -588,6 +592,7 @@ class AWSDeployer {
         if (!params.events[i].stream) {
           throw Error(`Missing Trigger Parameter: SQS Queue Url.`);
         }
+        params.events[i].url = params.events[i].stream;
         params.events[i].stream = await this.sqs.getQueueArn(
           params.events[i].stream
         );

package/src/app/main-process/main-window.js

@@ -85,7 +85,7 @@ class MainWindow {
 
     this.window.webContents.on('did-finish-load', () => {
       // this.checkForUpdates();
-      this.updateProjectInfo(this.lastUserProjectDir, this.lastFlowfile || "untitled");
+      this.updateProjectInfo(this.lastUserProjectDir, this.lastFlowfile || "__untitled__");
       this.emitter.emit('did-load', null);
 
       if (this.startupErrorEvent) {

package/src/app/main-process/menu.js

@@ -117,7 +117,7 @@ let appTemplate = [
     submenu: [
       { role: 'reload' },
       { role: 'forcereload' },
-//      { role: 'toggledevtools' },
+    //  { role: 'toggledevtools' },
       { type: 'separator' },
       { role: 'resetzoom' },
       { role: 'zoomin' },

package/src/app/main.js

@@ -75,7 +75,7 @@ ipcMain.on('modal-welcome:show-dialog-newproject', async () => {
   
   let userDir = codegen.generateInitialCode(
     userPrefStore.getUserDataPath(),
-    'untitled',
+    '__untitled__',
     null,
     version
   );
@@ -314,7 +314,7 @@ ipcMain.on('restart-requested', async (event, msg) => {
   // reload the nodes available in the palette
   try {
     await requestAsync({
-      uri: `http://127.0.0.1:${PORT}/nodes-reload`,
+      uri: `http://127.0.0.1:${RUNTIME_PORT}/nodes-reload`,
       method: 'POST',
       json: true,
     });
@@ -330,7 +330,7 @@ ipcMain.on('modal-node-library:restart-requested', async (event, msg) => {
   // reload the nodes available in the palette
   try {
     await requestAsync({
-      uri: `http://127.0.0.1:${PORT}/nodes-reload`,
+      uri: `http://127.0.0.1:${RUNTIME_PORT}/nodes-reload`,
       method: 'POST',
       json: true,
     });
@@ -477,7 +477,7 @@ async function openWelcomePage() {
   
   let userDir = codegen.generateInitialCode(
     userPrefStore.getUserDataPath(),
-    'untitled',
+    '__untitled__',
     null,
     version 
   );

package/src/app/preload.js

@@ -11,7 +11,7 @@ const { AppServer, lambdify } = require('./lib/runtime-loader');
 const { Terminal } = require('xterm');
 const { FitAddon } = require('xterm-addon-fit');
 const electron = require('electron');
-const remote = require('@electron/remote')
+const remote = require('@electron/remote');
 
 // Parse arguments from: webPreferences.additionalArguments
 const argv = require('yargs/yargs')(process.argv.slice(2)).parse();
@@ -22,6 +22,7 @@ const debounce = require('debounce');
 const { promisify } = require('util');
 
 const dagre = require('dagre');
+const simpleGit = require('simple-git');
 const request = require('request');
 const requestAsync = promisify(request);
 
@@ -168,7 +169,7 @@ async function getRegion(profile) {
   return region;
 }
 
-async function testFlow(testEvent, targetNode, environmentVariables, traceabilityEnabled) {
+async function testFlow(testEvent, targetNode, environmentVariables) {
 
   let event;
 
@@ -185,10 +186,6 @@ async function testFlow(testEvent, targetNode, environmentVariables, traceabilit
   
   headers = { ...headers, 'x-kumologica-testcasenode': targetNode };
 
-  if (traceabilityEnabled === true){
-    headers = { ...headers, 'x-kumologica-traceability': true}
-  }
-
   if (environmentVariables) {
     if (Object.keys(environmentVariables).length > 0) {
       headers = {
@@ -536,6 +533,7 @@ async function azureLogin(profile) {
   await azure.login(alias);
   return {alias: alias};
 }
+
 /**
  * Dispatch an action from the ipcRenderer to the main process
  * @param {string} type
@@ -622,7 +620,7 @@ window.__kumologica.libs.testFlow = testFlow;
 window.__kumologica.libs.openFileOnEditor = openFileOnEditor;
 window.__kumologica.libs.openFolder = openFolder;
 window.__kumologica.libs.dagre = dagre;
-
+window.__kumologica.libs.simpleGit = simpleGit;
 
 // Node Library
 window.__kumologica.nodeLibrary = {};

package/src/app/ui/editor-api/lib/editor/comms.js

@@ -57,7 +57,7 @@ function CommsConnection(ws) {
   var self = this;
 
   log.audit({ event: 'comms.open' });
-  log.trace('comms.open ' + self.session);
+  // log.trace('comms.open ' + self.session);
   var pendingAuth = settings.adminAuth != null;
 
   if (!pendingAuth) {
@@ -65,7 +65,7 @@ function CommsConnection(ws) {
   }
   ws.on('close', function () {
     log.audit({ event: 'comms.close', user: self.user, session: self.session });
-    log.trace('comms.close ' + self.session);
+    // log.trace('comms.close ' + self.session);
     removeActiveConnection(self);
   });
   ws.on('message', function (data, flags) {
@@ -73,7 +73,7 @@ function CommsConnection(ws) {
     try {
       msg = JSON.parse(data);
     } catch (err) {
-      log.trace('comms received malformed message : ' + err.toString());
+      // log.trace('comms received malformed message : ' + err.toString());
       return;
     }
     if (!pendingAuth) {