@nebulae/event-store-tpi-rx6 1.1.2

package/.jshintrc

@@ -0,0 +1,4 @@
+{
+    "esversion": 11,
+    "node":true
+}

package/.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "cSpell.words": [
+        "APAR",
+        "BPAR"
+    ]
+}

package/README.md

@@ -0,0 +1,23 @@
+# Event Store
+
+
+# Installation
+
+Assuming that Node.js is already installed & running, 
+
+## Install as dependency
+```sh
+$ npm install @nebulae/event-store-tpi-rx6
+```
+
+# usage
+## Author & Contributors
+
+Developed &amp; maintained by authors: <b>Sebastian Molano & Esteban Zapata on behalf of Nebula Engineering SAS</b><br>
+Follow Nebula Engineering at: <a href="https://github.com/NebulaEngineering" target="_blank">github</a>
+
+## License
+
+The MIT License (MIT)
+
+Copyright (c) 2018 Nebula Engineering SAS

package/index.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const EventStore = require('./lib/EventStore');
+const Event = require('./lib/entities/Event');
+
+module.exports = {
+    EventStore,
+    Event,
+};

package/lib/EventStore.js

@@ -0,0 +1,172 @@
+'use strict'
+
+const Rx = require('rxjs');
+const { filter, map, concatMap, switchMap } = require('rxjs/operators');
+const EventResult = require('./entities/Event');
+
+class EventStore {
+
+
+    /**
+     * Create a new EventStore
+     * 
+     * @param {Object} brokerConfig 
+     *    {
+     *        type,
+     *        eventsTopic,
+     *        brokerUrl,
+     *        eventsTopicSubscription,
+     *        projectId,
+     *    }         
+     * @param {Object} storeConfig 
+     *      {
+     *        type,
+     *        url,
+     *        eventStoreDbName,
+     *        aggregatesDbName
+     *       }
+     */
+    constructor(brokerConfig, storeConfig) {
+        switch (brokerConfig.type) {
+            case "PUBSUB":
+                const PubSubBroker = require('./broker/PubSubBroker');
+                this.broker = new PubSubBroker(brokerConfig);
+                break;
+            case "MQTT":
+                const MqttBroker = require('./broker/MqttBroker');
+                this.broker = new MqttBroker(brokerConfig);
+                break;
+            default:
+                throw new Error(`Invalid EventStore broker type: ${brokerConfig.type} `);
+        }
+
+        switch (storeConfig.type) {
+            case "MONGO":
+                const MongoStore = require('./store/MongoStore');
+                this.storeDB = new MongoStore(storeConfig);
+                break;
+            default:
+                throw new Error(`Invalid EventStore store type: ${storeConfig.type} `);
+        }
+    }
+
+    /**
+     * Starts Event Broker + Store
+     */
+    start$() {
+        return Rx.merge(
+            this.broker.start$(),
+            this.storeDB.start$()
+        );
+    }
+
+    /**
+     * Stops Event Broker + Store
+     */
+    stop$() {
+        return Rx.merge(
+            this.broker.stop$(),
+            this.storeDB.stop$()
+        );
+    }
+
+    /**
+     * Appends and emit a new Event
+     * 
+     * @param {Event} event Event to store and emit
+     * 
+     * Returns an obseravable that resolves to:
+     * {
+     *  storeResult :  {aggregate,event,versionTimeStr},
+     *  brokerResult: { messageId }
+     * }
+     * 
+     * where:
+     *  - aggregate = current aggregate state
+     *  - event = persisted event
+     *  - versionTimeStr = EventStore date index where the event was store
+     *  - messageId: sent message ID
+     *      
+     */
+    emitEvent$(event) {
+        return this.storeDB.pushEvent$(event)
+            .pipe(
+                concatMap((storeResult) =>
+                    this.broker.publish$(storeResult.event)
+                        .pipe(map(messageId => {
+                            return { storeResult, brokerResult: { messageId } };
+                        }))
+                )
+            );
+    }
+
+    /**
+     * Find all events of an especific aggregate
+     * @param {String} aggregateType Aggregate type
+     * @param {String} aggregateId Aggregate Id
+     * @param {number} version version to recover from (exclusive), defualt = 0
+     * @param {limit} limit max number of events to return, default = 20
+     * 
+     * Returns an Observable that emits each found event one by one
+     */
+    retrieveEvents$(aggregateType, aggregateId, version = 0, limit = 20) {
+        return this.storeDB.getEvents$(aggregateType, aggregateId, version, limit)
+    }
+
+    /**
+     * Find all events of an especific aggregate having taken place but not acknowledged,
+     * @param {String} aggregateType Aggregate type
+     * @param {string} key process key (eg. microservice name) that acknowledged the events
+     * 
+     * Returns an Observable that emits each found event one by one
+     */
+    retrieveUnacknowledgedEvents$(aggregateType, key) {
+        return this.storeDB.retrieveUnacknowledgedEvents$(aggregateType, key);
+    }
+
+    /**
+     * Find Aggregates that were created after the given date
+     * 
+     * @param {string} type 
+     * @param {number} createTimestamp 
+     * @param {Object} ops {offset,pageSize}
+     * 
+     * Returns an observable that publish every found aggregate 
+     */
+    findAgregatesCreatedAfter$(type, createTimestamp = 0) {
+        return this.storeDB.findAgregatesCreatedAfter$(type, createTimestamp);
+    }
+
+    /**
+     * Returns an Observable that will emit any event related to the given aggregateType
+     * @param {string} aggregateType 
+     */
+    getEventListener$(aggregateType, key, ignoreSelfEvents = true) {
+        return this.storeDB.findLatestAcknowledgedTimestamp$(aggregateType, key).pipe(
+            switchMap(latestTimeStamp => {
+                //this will ignore all queued messages that were already processed
+                return this.broker.getEventListener$(aggregateType, ignoreSelfEvents)
+                    .pipe(filter(evt => evt.timestamp > latestTimeStamp))
+            })
+        );
+    }
+
+    /**
+     * @param {Event} event event to acknowledge 
+     * @param {string} key process key (eg. microservice name) that is acknowledging the event
+     */
+    acknowledgeEvent$(event, key) {
+        return this.storeDB.acknowledgeEvent$(event, key);
+    }
+
+    /**
+     * Ensure the existence of a registry on the ack database for an aggregate type
+     * @param {string} aggregateType 
+     * @param {string} key backend key 
+     */
+    ensureAcknowledgeRegistry$(aggregateType, key) {
+        return this.storeDB.ensureAcknowledgeRegistry$(aggregateType, key);
+    }
+}
+
+module.exports = EventStore;

package/lib/broker/MqttBroker.js

@@ -0,0 +1,116 @@
+'use strict'
+
+const Rx = require('rxjs');
+const { filter, mapTo, map} = require('rxjs/operators');
+const uuidv4 = require('uuid/v4');
+
+
+
+class MqttBroker {
+
+    constructor({ eventsTopic, brokerUrl }) {
+
+        this.topicName = eventsTopic;
+        this.mqttServerUrl = brokerUrl;
+        this.senderId = uuidv4();
+        /**
+         * Rx Subject for Incoming events
+         */
+        this.incomingEvents$ = new Rx.BehaviorSubject();
+        this.orderedIncomingEvents$ = this.incomingEvents$.pipe(
+            filter(msg => msg)
+        )
+            // .groupBy(msg => msg.data.at)
+            // .mergeMap(groupStream =>
+            //     groupStream.bufferWhen(() => groupStream.debounceTime(250))
+            //         .filter(bufferedArray => bufferedArray && bufferedArray.length > 0)
+            //         .map(bufferedArray => bufferedArray.sort((o1, o2) => { return o1.data.av - o2.data.av }))
+            //         .mergeMap(bufferedArray => Rx.Observable.from(bufferedArray))
+            // )
+            ;
+
+        /**
+         * MQTT Client
+         */
+        this.mqtt = require("async-mqtt");
+    }
+
+    /**
+     * Starts Broker connections
+     * Returns an Obserable that resolves to each connection result
+     */
+    start$() {
+        return Rx.Observable.create(observer => {
+            this.mqttClient = this.mqtt.connect(this.mqttServerUrl);
+            observer.next('MQTT broker connecting ...');
+            this.mqttClient.on('connect', () => {
+                observer.next('MQTT broker connected');
+                this.mqttClient.subscribe(this.topicName);
+                observer.next(`MQTT broker listening messages`);
+                observer.complete();
+            });
+            this.mqttClient.on('message', (topic, message) => {
+                const envelope = JSON.parse(message);
+                //console.log(`**************** Received message id: ${envelope.id}`);
+                // message is Buffer
+                this.incomingEvents$.next(
+                    {
+                        id: envelope.id,
+                        data: envelope.data,
+                        attributes: envelope.attributes,
+                        correlationId: envelope.attributes.correlationId
+                    }
+                );
+            });
+
+
+        });
+
+
+    }
+
+    /**
+     * Disconnect the broker and return an observable that completes when disconnected
+     */
+    stop$() {
+        return Rx.defer(() => this.mqttClient.end());
+    }
+
+
+    /**
+     * Publish data throught the events topic
+     * Returns an Observable that resolves to the sent message ID
+     * @param {string} topicName 
+     * @param {Object} data 
+     */
+    publish$(data) {
+        const uuid = uuidv4();
+        const dataBuffer = JSON.stringify(
+            {
+                id: uuid,
+                data,
+                attributes: {
+                    senderId: this.senderId
+                }
+            }
+        );
+        return Rx.defer(() => this.mqttClient.publish(`${this.topicName}`, dataBuffer, { qos: 1 }))
+            .pipe(mapTo(uuid));
+    }
+
+    /**
+     * Returns an Observable that will emit any event related to the given aggregateType
+     * @param {string} aggregateType 
+     */
+    getEventListener$(aggregateType, ignoreSelfEvents = true) {
+        return this.orderedIncomingEvents$.pipe(
+            filter(msg => msg)
+            ,filter(msg => !ignoreSelfEvents || msg.attributes.senderId !== this.senderId)
+            ,map(msg => msg.data)
+            ,filter(evt => evt.at === aggregateType || aggregateType == "*" )
+        )
+    }
+
+}
+
+module.exports = MqttBroker;

package/lib/broker/PubSubBroker.js

@@ -0,0 +1,144 @@
+'use strict'
+
+const Rx = require('rxjs');
+const { filter, map} = require('rxjs/operators');
+// Imports the Google Cloud client library
+const uuidv4 = require('uuid/v4');
+const PubSub = require('@google-cloud/pubsub');
+
+class PubSubBroker {
+
+    constructor({ eventsTopic, eventsTopicSubscription }) {
+        //this.projectId = projectId;
+        this.eventsTopic = eventsTopic;
+        this.eventsTopicSubscription = eventsTopicSubscription;
+
+        /**
+         * Rx Subject for every incoming event
+         */
+        this.incomingEvents$ = new Rx.BehaviorSubject();
+        this.orderedIncomingEvents$ = this.incomingEvents$.pipe(
+            filter(msg => msg)
+        )
+            // .groupBy(msg => msg.data.at)
+            // .mergeMap(groupStream =>
+            //     groupStream.bufferWhen(() => groupStream.debounceTime(250))
+            //         .filter(bufferedArray => bufferedArray && bufferedArray.length > 0)
+            //         .map(bufferedArray => bufferedArray.sort((o1, o2) => { return o1.data.av - o2.data.av }))
+            //         .mergeMap(bufferedArray => Rx.Observable.from(bufferedArray))
+            // );
+        this.senderId = uuidv4();
+
+        this.pubsubClient = new PubSub({
+            //projectId: projectId,
+        });
+
+        this.topic = this.pubsubClient.topic(eventsTopic);
+    }
+
+    /**
+     * Starts Broker connections
+     * Returns an Obserable that resolves to each connection result
+     */
+    start$() {
+        return Rx.Observable.create(observer => {
+            this.startMessageListener();
+            observer.next(`Event Store PubSub Broker listening: Topic=${this.eventsTopic}, subscriptionName=${this.eventsTopicSubscription}`);
+            observer.complete();
+        });
+    }
+
+    /**
+     * Disconnect the broker and return an observable that completes when disconnected
+     */
+    stop$() {
+        return Rx.Observable.create(observer => {
+            this.getSubscription$().subscribe(
+                (subscription) => {
+                    subscription.removeListener(`message`, this.onMessage);
+                    observer.next(`Event Store PubSub Broker removed listener: Topic=${this.eventsTopic}, subscriptionName=${subscription}`);
+                },
+                (error) => observer.error(error),
+                () => {
+                    this.messageListenerSubscription.unsubscribe();
+                    observer.complete();
+                }
+            );
+
+        });
+
+    }
+
+    /**
+     * Publish data throught the events topic
+     * Returns an Observable that resolves to the sent message ID
+     * @param {string} topicName 
+     * @param {Object} data 
+     */
+    publish$(data) {
+        const dataBuffer = Buffer.from(JSON.stringify(data));
+        return Rx.defer(() =>
+            this.topic.publisher().publish(
+                dataBuffer,
+                { senderId: this.senderId }))
+            //.do(messageId => console.log(`PubSub Message published through ${this.topic.name}, Message=${JSON.stringify(data)}`))
+            ;
+    }
+
+    /**
+     * Returns an Observable that will emit any event related to the given aggregateType
+     * @param {string} aggregateType 
+     */
+    getEventListener$(aggregateType, ignoreSelfEvents = true) {
+        return this.orderedIncomingEvents$.pipe(
+            filter(msg => msg)
+            ,filter(msg => !ignoreSelfEvents || msg.attributes.senderId !== this.senderId)
+            ,map(msg => msg.data)
+            ,filter(evt => evt.at === aggregateType || aggregateType == "*")
+            )
+    }
+
+
+    /**
+     * Returns an Observable that resolves to the subscription
+     */
+    getSubscription$() {
+        return Rx.defer(() =>
+            this.topic.subscription(this.eventsTopicSubscription)
+                .get({ autoCreate: true })).pipe(
+                    map(results => results[0])
+                );
+    }
+
+    /**
+     * Starts to listen messages
+     */
+    startMessageListener() {
+        this.messageListenerSubscription = this.getSubscription$()
+            .subscribe(
+                (pubSubSubscription) => {
+                    this.onMessage = message => {
+                        message.ack();
+                        //console.log(`Received message ${message.id}:`);                                                
+                        this.incomingEvents$.next({
+                            data: JSON.parse(message.data),
+                            id: message.id,
+                            attributes: message.attributes,
+                            correlationId: message.attributes.correlationId
+                        });
+                        //console.log(`Execute ack message ${message.id}:`);                                                                                  
+                    };
+                    pubSubSubscription.on(`message`, this.onMessage);
+                },
+                (err) => {
+                    console.error('Failed to obtain EventStore subscription', err);
+                },
+                () => {
+                    console.log('GatewayEvents listener has completed!');
+                }
+            );
+    }
+
+}
+
+module.exports = PubSubBroker;

package/lib/entities/Event.js

@@ -0,0 +1,47 @@
+'use strict'
+
+const Rx = require('rxjs');
+
+class Event {
+    constructor({ eventType, eventTypeVersion, aggregateType, aggregateId, data, user, aggregateVersion, ephemeral = false }) {
+
+        /**
+         * Event type
+         */
+        this.et = eventType;
+        /**
+         * Event type version
+         */
+        this.etv = eventTypeVersion;
+        /**
+         * Aggregate Type
+         */
+        this.at = aggregateType;
+        /**
+         * Aggregate ID
+         */
+        this.aid = aggregateId;
+        /**
+         * Event data
+         */
+        this.data = data;
+        /**
+         * Responsible user
+         */
+        this.user = user;
+        /**
+         * TimeStamp
+         */
+        this.timestamp = (new Date).getTime();
+        /**
+         * Aggregate version
+         */
+        this.av = aggregateVersion;
+        /**
+        * if ephemeral is true, this event will not be stored 
+        */
+        this.ephemeral = ephemeral;
+    }
+}
+
+module.exports = Event;