@friggframework/core 2.0.0-next.0 → 2.0.0-next.2

package/database/index.js

@@ -1,14 +1,15 @@
-const { mongoose} = require('./mongoose');
+const { mongoose } = require('./mongoose');
 const {
     connectToDatabase,
     disconnectFromDatabase,
     createObjectId,
 } = require('./mongo');
-const {IndividualUser} = require('./models/IndividualUser');
-const {OrganizationUser} = require('./models/OrganizationUser');
-const {State} = require('./models/State');
-const {Token} = require('./models/Token');
-const {UserModel} = require('./models/UserModel');
+const { IndividualUser } = require('./models/IndividualUser');
+const { OrganizationUser } = require('./models/OrganizationUser');
+const { State } = require('./models/State');
+const { Token } = require('./models/Token');
+const { UserModel } = require('./models/UserModel');
+const { WebsocketConnection } = require('./models/WebsocketConnection');
 
 module.exports = {
     mongoose,
@@ -19,5 +20,6 @@ module.exports = {
     OrganizationUser,
     State,
     Token,
-    UserModel
-}
+    UserModel,
+    WebsocketConnection,
+};

package/database/models/WebsocketConnection.js

@@ -0,0 +1,49 @@
+const { mongoose } = require('../mongoose');
+const AWS = require('aws-sdk');
+
+const schema = new mongoose.Schema({
+    connectionId: { type: mongoose.Schema.Types.String },
+});
+
+// Add a static method to get active connections
+schema.statics.getActiveConnections = async function () {
+    try {
+        const connections = await this.find({}, 'connectionId');
+        return connections.map((conn) => ({
+            connectionId: conn.connectionId,
+            send: async (data) => {
+                const apigwManagementApi = new AWS.ApiGatewayManagementApi({
+                    apiVersion: '2018-11-29',
+                    endpoint: process.env.WEBSOCKET_API_ENDPOINT,
+                });
+
+                try {
+                    await apigwManagementApi
+                        .postToConnection({
+                            ConnectionId: conn.connectionId,
+                            Data: JSON.stringify(data),
+                        })
+                        .promise();
+                } catch (error) {
+                    if (error.statusCode === 410) {
+                        console.log(`Stale connection ${conn.connectionId}`);
+                        await this.deleteOne({
+                            connectionId: conn.connectionId,
+                        });
+                    } else {
+                        throw error;
+                    }
+                }
+            },
+        }));
+    } catch (error) {
+        console.error('Error getting active connections:', error);
+        throw error;
+    }
+};
+
+const WebsocketConnection =
+    mongoose.models.WebsocketConnection ||
+    mongoose.model('WebsocketConnection', schema);
+
+module.exports = { WebsocketConnection };

package/integrations/integration-base.js

@@ -1,41 +1,122 @@
 const { IntegrationMapping } = require('./integration-mapping');
+const { Options } = require('./options');
+const constantsToBeMigrated = {
+    defaultEvents: {
+        ON_CREATE: 'ON_CREATE',
+        ON_UPDATE: 'ON_UPDATE',
+        ON_DELETE: 'ON_DELETE',
+        GET_CONFIG_OPTIONS: 'GET_CONFIG_OPTIONS',
+        REFRESH_CONFIG_OPTIONS: 'REFRESH_CONFIG_OPTIONS',
+        GET_USER_ACTIONS: 'GET_USER_ACTIONS',
+        GET_USER_ACTION_OPTIONS: 'GET_USER_ACTION_OPTIONS',
+        REFRESH_USER_ACTION_OPTIONS: 'REFRESH_USER_ACTION_OPTIONS',
+        // etc...
+    },
+    types: {
+        LIFE_CYCLE_EVENT: 'LIFE_CYCLE_EVENT',
+        USER_ACTION: 'USER_ACTION',
+    },
+};
 
 class IntegrationBase {
+    static getOptionDetails() {
+        const options = new Options({
+            module: Object.values(this.Definition.modules)[0], // This is a placeholder until we revamp the frontend
+            ...this.Definition,
+        });
+        return options.get();
+    }
     /**
-     * CHILDREN SHOULD SPECIFY A CONFIG
+     * CHILDREN SHOULD SPECIFY A DEFINITION FOR THE INTEGRATION
      */
-    static Config = {
+    static Definition = {
         name: 'Integration Name',
         version: '0.0.0', // Integration Version, used for migration and storage purposes, as well as display
         supportedVersions: [], // Eventually usable for deprecation and future test version purposes
 
-        // an array of events that are process(able) by this Integration
-        events: [],
+        modules: {},
+        display: {
+            name: 'Integration Name',
+            logo: '',
+            description: '',
+            // etc...
+        },
     };
 
     static getName() {
-        return this.Config.name;
+        return this.Definition.name;
     }
 
     static getCurrentVersion() {
-        return this.Config.version;
+        return this.Definition.version;
+    }
+    loadModules() {
+        // Load all the modules defined in Definition.modules
+        const moduleNames = Object.keys(this.constructor.Definition.modules);
+        for (const moduleName of moduleNames) {
+            const { definition } =
+                this.constructor.Definition.modules[moduleName];
+            if (typeof definition.API === 'function') {
+                this[moduleName] = { api: new definition.API() };
+            } else {
+                throw new Error(
+                    `Module ${moduleName} must be a function that extends IntegrationModule`
+                );
+            }
+        }
+    }
+    registerEventHandlers() {
+        this.on = {
+            ...this.defaultEvents,
+            ...this.events,
+        };
     }
 
     constructor(params) {
-        this.delegateTypes = [];
-        this.userActions = [];
+        this.defaultEvents = {
+            [constantsToBeMigrated.defaultEvents.ON_CREATE]: {
+                type: constantsToBeMigrated.types.LIFE_CYCLE_EVENT,
+                handler: this.onCreate,
+            },
+            [constantsToBeMigrated.defaultEvents.ON_UPDATE]: {
+                type: constantsToBeMigrated.types.LIFE_CYCLE_EVENT,
+                handler: this.onUpdate,
+            },
+            [constantsToBeMigrated.defaultEvents.ON_DELETE]: {
+                type: constantsToBeMigrated.types.LIFE_CYCLE_EVENT,
+                handler: this.onDelete,
+            },
+            [constantsToBeMigrated.defaultEvents.GET_CONFIG_OPTIONS]: {
+                type: constantsToBeMigrated.types.LIFE_CYCLE_EVENT,
+                handler: this.getConfigOptions,
+            },
+            [constantsToBeMigrated.defaultEvents.REFRESH_CONFIG_OPTIONS]: {
+                type: constantsToBeMigrated.types.LIFE_CYCLE_EVENT,
+                handler: this.refreshConfigOptions,
+            },
+            [constantsToBeMigrated.defaultEvents.GET_USER_ACTIONS]: {
+                type: constantsToBeMigrated.types.LIFE_CYCLE_EVENT,
+                handler: this.loadUserActions,
+            },
+            [constantsToBeMigrated.defaultEvents.GET_USER_ACTION_OPTIONS]: {
+                type: constantsToBeMigrated.types.LIFE_CYCLE_EVENT,
+                handler: this.getActionOptions,
+            },
+            [constantsToBeMigrated.defaultEvents.REFRESH_USER_ACTION_OPTIONS]: {
+                type: constantsToBeMigrated.types.LIFE_CYCLE_EVENT,
+                handler: this.refreshActionOptions,
+            },
+        };
+        this.loadModules();
     }
 
-    //psuedo delegate for backwards compatability
-    async receiveNotification(notifier, delegateString, object = null) {}
-
-    async notify(delegateString, object = null) {
-        if (!this.delegateTypes.includes(delegateString)) {
+    async send(event, object) {
+        if (!this.on[event]) {
             throw new Error(
-                `delegateString:${delegateString} is not defined in delegateTypes`
+                `Event ${event} is not defined in the Integration event object`
             );
         }
-        return this.receiveNotification(this, delegateString, object);
+        return this.on[event].handler.call(this, object);
     }
 
     async validateConfig() {
@@ -69,28 +150,20 @@ class IntegrationBase {
     async testAuth() {
         let didAuthPass = true;
 
-        try {
-            await this.primary.testAuth();
-        } catch {
-            didAuthPass = false;
-            this.record.messages.errors.push({
-                title: 'Authentication Error',
-                message: `There was an error with your ${this.primary.constructor.getName()} Entity.
+        for (const module of Object.keys(IntegrationBase.Definition.modules)) {
+            try {
+                await this[module].testAuth();
+            } catch {
+                didAuthPass = false;
+                this.record.messages.errors.push({
+                    title: 'Authentication Error',
+                    message: `There was an error with your ${this[
+                        module
+                    ].constructor.getName()} Entity.
                 Please reconnect/re-authenticate, or reach out to Support for assistance.`,
-                timestamp: Date.now(),
-            });
-        }
-
-        try {
-            await this.target.testAuth();
-        } catch {
-            didAuthPass = false;
-            this.record.messages.errors.push({
-                title: 'Authentication Error',
-                message: `There was an error with your ${this.target.constructor.getName()} Entity.
-            Please reconnect/re-authenticate, or reach out to Support for assistance.`,
-                timestamp: Date.now(),
-            });
+                    timestamp: Date.now(),
+                });
+            }
         }
 
         if (!didAuthPass) {
@@ -115,15 +188,6 @@ class IntegrationBase {
         );
     }
 
-    async getAndSetUserActions() {
-        this.userActions = await this.getUserActions();
-        if (this.record?.config) {
-            this.record.config.userActions = this.userActions;
-            await this.record.save();
-        }
-        return this.userActions;
-    }
-
     /**
      * CHILDREN CAN OVERRIDE THESE CONFIGURATION METHODS
      */
@@ -153,11 +217,34 @@ class IntegrationBase {
         return options;
     }
 
-    async getUserActions() {
-        return [];
+    async loadDynamicUserActions() {
+        // Child class should override this method to load dynamic user actions.
+        // Dynamic user actions should return in the same form a valid event object
+
+        return {};
+    }
+    async loadUserActions({ actionType } = {}) {
+        console.log('loadUserActions called with actionType:', actionType);
+        const userActions = {};
+        for (const [key, event] of Object.entries(this.events)) {
+            if (event.type === constantsToBeMigrated.types.USER_ACTION) {
+                if (!actionType || event.userActionType === actionType) {
+                    userActions[key] = event;
+                }
+            }
+        }
+        const dynamicUserActions = await this.loadDynamicUserActions();
+        const filteredDynamicActions = actionType
+            ? Object.fromEntries(
+                  Object.entries(dynamicUserActions).filter(
+                      ([_, event]) => event.userActionType === actionType
+                  )
+              )
+            : dynamicUserActions;
+        return { ...userActions, ...filteredDynamicActions };
     }
 
-    async getActionOptions() {
+    async getActionOptions(actionId, data) {
         const options = {
             jsonSchema: {},
             uiSchema: {},

package/integrations/integration-factory.js

@@ -1,22 +1,28 @@
 const { ModuleFactory, Credential, Entity } = require('../module-plugin');
-const {IntegrationModel} = require("./integration-model");
+const { IntegrationModel } = require('./integration-model');
 const _ = require('lodash');
 
-
 class IntegrationFactory {
     constructor(integrationClasses = []) {
         this.integrationClasses = integrationClasses;
         this.moduleFactory = new ModuleFactory(...this.getModules());
-        this.integrationTypes = this.integrationClasses.map(IntegrationClass => IntegrationClass.getName());
-        this.getIntegrationConfigs = this.integrationClasses.map(IntegrationClass => IntegrationClass.Config);
+        this.integrationTypes = this.integrationClasses.map(
+            (IntegrationClass) => IntegrationClass.getName()
+        );
+        this.getIntegrationDefinitions = this.integrationClasses.map(
+            (IntegrationClass) => IntegrationClass.Definition
+        );
     }
 
     async getIntegrationOptions() {
-        const options = this.integrationClasses.map(IntegrationClass => IntegrationClass.Options);
+        const options = this.integrationClasses.map(
+            (IntegrationClass) => IntegrationClass
+        );
         return {
             entities: {
-                primary: this.getPrimaryName(),
-                options: options.map(val => val.get()),
+                options: options.map((IntegrationClass) =>
+                    IntegrationClass.getOptionDetails()
+                ),
                 authorized: [],
             },
             integrations: [],
@@ -24,65 +30,148 @@ class IntegrationFactory {
     }
 
     getModules() {
-        return  [... new Set(this.integrationClasses.map(integration =>
-            Object.values(integration.modules)
-        ).flat())];
+        return [
+            ...new Set(
+                this.integrationClasses
+                    .map((integration) =>
+                        Object.values(integration.Definition.modules).map(
+                            (module) => module.definition
+                        )
+                    )
+                    .flat()
+            ),
+        ];
     }
 
-    getPrimaryName() {
-        function findMostFrequentElement(array) {
-            const frequencyMap = _.countBy(array);
-            return _.maxBy(_.keys(frequencyMap), (element) => frequencyMap[element]);
-        }
-        const allModulesNames = _.flatten(this.integrationClasses.map(integration =>
-            Object.values(integration.modules).map(module => module.getName())
-        ));
-        return findMostFrequentElement(allModulesNames);
-    }
-
-    getIntegrationClassDefByType(type) {
+    getIntegrationClassByType(type) {
         const integrationClassIndex = this.integrationTypes.indexOf(type);
         return this.integrationClasses[integrationClassIndex];
     }
+    getModuleTypesAndKeys(integrationClass) {
+        const moduleTypesAndKeys = {};
+        const moduleTypeCount = {};
+
+        if (integrationClass && integrationClass.Definition.modules) {
+            for (const [key, moduleClass] of Object.entries(
+                integrationClass.Definition.modules
+            )) {
+                if (
+                    moduleClass &&
+                    typeof moduleClass.definition.getName === 'function'
+                ) {
+                    const moduleType = moduleClass.definition.getName();
+
+                    // Check if this module type has already been seen
+                    if (moduleType in moduleTypesAndKeys) {
+                        throw new Error(
+                            `Duplicate module type "${moduleType}" found in integration class definition.`
+                        );
+                    }
+
+                    // Well how baout now
+
+                    moduleTypesAndKeys[moduleType] = key;
+                    moduleTypeCount[moduleType] =
+                        (moduleTypeCount[moduleType] || 0) + 1;
+                }
+            }
+        }
+
+        // Check for any module types with count > 1
+        for (const [moduleType, count] of Object.entries(moduleTypeCount)) {
+            if (count > 1) {
+                throw new Error(
+                    `Multiple instances of module type "${moduleType}" found in integration class definition.`
+                );
+            }
+        }
+
+        return moduleTypesAndKeys;
+    }
 
     async getInstanceFromIntegrationId(params) {
-        const integrationRecord = await IntegrationHelper.getIntegrationById(params.integrationId);
-        let {userId} = params;
+        const integrationRecord = await IntegrationHelper.getIntegrationById(
+            params.integrationId
+        );
+        let { userId } = params;
         if (!integrationRecord) {
-            throw new Error(`No integration found by the ID of ${params.integrationId}`);
+            throw new Error(
+                `No integration found by the ID of ${params.integrationId}`
+            );
         }
 
         if (!userId) {
             userId = integrationRecord.user._id.toString();
-        } else if (userId !== integrationRecord.user._id.toString()) {
-            throw new Error(`Integration ${params.integrationId} does not belong to User ${userId}, ${integrationRecord.user.id.toString()}`);
+        } else if (userId.toString() !== integrationRecord.user.toString()) {
+            throw new Error(
+                `Integration ${
+                    params.integrationId
+                } does not belong to User ${userId}, ${integrationRecord.user.toString()}`
+            );
         }
 
-        const integrationClassDef = this.getIntegrationClassDefByType(integrationRecord.config.type);
-        const instance = new integrationClassDef({
+        const integrationClass = this.getIntegrationClassByType(
+            integrationRecord.config.type
+        );
+
+        const instance = new integrationClass({
             userId,
             integrationId: params.integrationId,
         });
+
+        if (
+            integrationRecord.entityReference &&
+            Object.keys(integrationRecord.entityReference) > 0
+        ) {
+            // Use the specified entityReference to find the modules and load them according to their key
+            // entityReference will be a map of entityIds with their corresponding desired key
+            for (const [entityId, key] of Object.entries(
+                integrationRecord.entityReference
+            )) {
+                const moduleInstance =
+                    await this.moduleFactory.getModuleInstanceFromEntityId(
+                        entityId,
+                        integrationRecord.user
+                    );
+                instance[key] = moduleInstance;
+            }
+        } else {
+            // for each entity, get the moduleinstance and load them according to their keys
+            // If it's the first entity, load the moduleinstance into primary as well
+            // If it's the second entity, load the moduleinstance into target as well
+            const moduleTypesAndKeys =
+                this.getModuleTypesAndKeys(integrationClass);
+            for (let i = 0; i < integrationRecord.entities.length; i++) {
+                const entityId = integrationRecord.entities[i];
+                const moduleInstance =
+                    await this.moduleFactory.getModuleInstanceFromEntityId(
+                        entityId,
+                        integrationRecord.user
+                    );
+                const moduleType = moduleInstance.getName();
+                const key = moduleTypesAndKeys[moduleType];
+                instance[key] = moduleInstance;
+                if (i === 0) {
+                    instance.primary = moduleInstance;
+                } else if (i === 1) {
+                    instance.target = moduleInstance;
+                }
+            }
+        }
         instance.record = integrationRecord;
-        instance.delegateTypes.push(...integrationClassDef.Config.events);
-        instance.primary = await this.moduleFactory.getModuleInstanceFromEntityId(
-            instance.record.entities[0],
-            instance.record.user
-        );
-        instance.target = await this.moduleFactory.getModuleInstanceFromEntityId(
-            instance.record.entities[1],
-            instance.record.user
-        );
 
         try {
-            await instance.getAndSetUserActions();
-            instance.delegateTypes.push(...Object.keys(instance.userActions));
-        } catch(e) {
-            instance.userActions = {};
+            const additionalUserActions =
+                await instance.loadDynamicUserActions();
+            instance.events = { ...instance.events, ...additionalUserActions };
+        } catch (e) {
             instance.record.status = 'ERROR';
             instance.record.messages.errors.push(e);
             await instance.record.save();
         }
+        // Register all of the event handlers
+
+        await instance.registerEventHandlers();
         return instance;
     }
 
@@ -93,12 +182,15 @@ class IntegrationFactory {
             config,
             version: '0.0.0',
         });
-        return await this.getInstanceFromIntegrationId({integrationId: integrationRecord.id, userId});
+        return await this.getInstanceFromIntegrationId({
+            integrationId: integrationRecord.id,
+            userId,
+        });
     }
 }
 
 const IntegrationHelper = {
-     getFormattedIntegration: async function(integrationRecord) {
+    getFormattedIntegration: async function (integrationRecord) {
         const integrationObj = {
             id: integrationRecord.id,
             status: integrationRecord.status,
@@ -122,14 +214,19 @@ const IntegrationHelper = {
         return integrationObj;
     },
 
-     getIntegrationsForUserId: async function(userId) {
+    getIntegrationsForUserId: async function (userId) {
         const integrationList = await IntegrationModel.find({ user: userId });
-        return await Promise.all(integrationList.map(async (integrationRecord) =>
-            await IntegrationHelper.getFormattedIntegration(integrationRecord)
-        ));
+        return await Promise.all(
+            integrationList.map(
+                async (integrationRecord) =>
+                    await IntegrationHelper.getFormattedIntegration(
+                        integrationRecord
+                    )
+            )
+        );
     },
 
-     deleteIntegrationForUserById: async function(userId, integrationId) {
+    deleteIntegrationForUserById: async function (userId, integrationId) {
         const integrationList = await IntegrationModel.find({
             user: userId,
             _id: integrationId,
@@ -142,13 +239,13 @@ const IntegrationHelper = {
         await IntegrationModel.deleteOne({ _id: integrationId });
     },
 
-     getIntegrationById: async function(id) {
-        return IntegrationModel.findById(id);
+    getIntegrationById: async function (id) {
+        return IntegrationModel.findById(id).populate('entities');
     },
 
-     listCredentials: async function(options) {
+    listCredentials: async function (options) {
         return Credential.find(options);
-    }
-}
+    },
+};
 
 module.exports = { IntegrationFactory, IntegrationHelper };

package/integrations/integration-model.js

@@ -9,6 +9,10 @@ const schema = new mongoose.Schema(
                 required: true,
             },
         ],
+        entityReference: {
+            type: mongoose.Schema.Types.Map,
+            of: String,
+        },
         user: {
             type: mongoose.Schema.Types.ObjectId,
             ref: 'User',

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@friggframework/core",
   "prettier": "@friggframework/prettier-config",
-  "version": "2.0.0-next.0",
+  "version": "2.0.0-next.2",
   "dependencies": {
     "@hapi/boom": "^10.0.1",
     "aws-sdk": "^2.1200.0",
@@ -15,9 +15,9 @@
     "node-fetch": "^2.6.7"
   },
   "devDependencies": {
-    "@friggframework/eslint-config": "^2.0.0-next.0",
-    "@friggframework/prettier-config": "^2.0.0-next.0",
-    "@friggframework/test": "^2.0.0-next.0",
+    "@friggframework/eslint-config": "2.0.0-next.2",
+    "@friggframework/prettier-config": "2.0.0-next.2",
+    "@friggframework/test": "2.0.0-next.2",
     "@types/lodash": "^4.14.191",
     "@typescript-eslint/eslint-plugin": "^8.0.0",
     "chai": "^4.3.6",
@@ -48,5 +48,5 @@
   },
   "homepage": "https://github.com/friggframework/frigg#readme",
   "description": "",
-  "gitHead": "e37f577d3c14f8eed6584517830c6c61815cf759"
+  "gitHead": "d2468c93806cf76851ead37495c10a4532d61489"
 }

package/queues/index.js

@@ -0,0 +1,4 @@
+const { QueuerUtil } = require('./queuer-util');
+module.exports = {
+    QueuerUtil,
+};

package/queues/queuer-util.js

@@ -0,0 +1,61 @@
+const { v4: uuid } = require('uuid');
+const AWS = require('aws-sdk');
+const awsConfigOptions = () => {
+    const config = {};
+    if (process.env.IS_OFFLINE) {
+        console.log('Running in offline mode');
+    }
+    if (process.env.AWS_ENDPOINT) {
+        config.endpoint = process.env.AWS_ENDPOINT;
+    }
+    return config;
+};
+AWS.config.update(awsConfigOptions());
+const sqs = new AWS.SQS();
+
+const QueuerUtil = {
+    batchSend: async (entries = [], queueUrl) => {
+        console.log(
+            `Enqueuing ${entries.length} entries on SQS to queue ${queueUrl}`
+        );
+        const buffer = [];
+        const batchSize = 10;
+
+        for (const entry of entries) {
+            buffer.push({
+                Id: uuid(),
+                MessageBody: JSON.stringify(entry),
+            });
+            // Sends 10, then purges the buffer
+            if (buffer.length === batchSize) {
+                console.log('Buffer at 10, sending batch');
+                await sqs
+                    .sendMessageBatch({
+                        Entries: buffer,
+                        QueueUrl: queueUrl,
+                    })
+                    .promise();
+                // Purge the buffer
+                buffer.splice(0, buffer.length);
+            }
+        }
+        console.log('Buffer at end, sending final batch');
+
+        // If any remaining entries under 10 are left in the buffer, send and return
+        if (buffer.length > 0) {
+            console.log(buffer);
+            return sqs
+                .sendMessageBatch({
+                    Entries: buffer,
+                    QueueUrl: queueUrl,
+                })
+                .promise();
+        }
+
+        // If we're exact... just return an empty object for now
+
+        return {};
+    },
+};
+
+module.exports = { QueuerUtil };