mongodb-livedata-server 0.0.1

package/README.md

@@ -0,0 +1,63 @@
+MongoDB Live Data Server
+========================
+
+This project is essentially a MongoDB live data driver (based either on polling or on Oplog tailing) combined with a DDP server, extracted
+out of [Meteor](https://github.com/meteor/meteor), with **Fibers** and **underscore** dependencies removed and code converted to Typescript.
+
+Live data is one of the root concepts of Meteor. Data is served via WebSockets via the DDP protocol and updated automatically whenever something changes in the database. Also, calling server methods via WebSocket is supported.
+
+Using Meteor locks you into the Meteor ecosystem, which has some problems (mostly for historical reasons). Using live data as a separate npm package might be preferable in many scenarios. Also, people who are trying to migrate from Meteor, might find this package useful as an intermediate step.
+
+### Usage
+
+As a most common example, this is how you can use livedata with Express.js:
+
+```js
+const { DDPServer, LiveCursor, LiveMongoConnection } = require('livedata-server')
+const express = require('express')
+const app = express()
+const port = 3000
+
+app.get('/', (req, res) => {
+  res.send('Hello World!')
+})
+
+const httpServer = app.listen(port, () => {
+  console.log(`Example app listening on port ${port}`)
+})
+
+const liveMongoConnection = new LiveMongoConnection(process.env.MONGO_URL, {
+    oplogUrl: process.env.MONGO_OPLOG_URL
+});
+const liveDataServer = new DDPServer({}, httpServer);
+
+liveDataServer.methods({
+    "test-method": async (msg) => {
+        console.log("Test msg: ", msg);
+        return "hello! Current timestamp is: " + Date.now()
+    }
+})
+
+liveDataServer.publish({
+    "test-subscription": async () => {
+        return new LiveCursor(liveMongoConnection, "test-collection", { category: "apples" });
+    }
+})
+
+```
+
+`liveDataServer.methods` and `liveDataServer.publish` have exactly same interface as [Meteor.methods](https://docs.meteor.com/api/methods.html#Meteor-methods) and [Meteor.publish](https://docs.meteor.com/api/pubsub.html#Meteor-publish) respectively, notice however that when publishing subscriptions, you must use `LiveCursor` rather than a normal MongoDB cursor.
+
+### Important notes
+
+- The project is in alpha. Use at your own risk.
+- Neither method context nor subscription context have the `unblock` method anymore (because this package doesn't use Fibers)
+- Meteor syntax for MongoDB queries is not supported. Please always use MongoDB Node.js driver syntax. For example, instead of
+  ```ts
+  const doc = myCollection.findOne(id);
+  ```
+  use
+  ```ts
+  const doc = await myCollection.findOne({ _id: id });
+  ```
+- Neither MongoDB.ObjectId nor it's Meteor.js alternative is supported at the moment. String ids only.

package/dist/livedata_server.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LiveCursor = exports.LiveMongoConnection = exports.DDPServer = void 0;
+var livedata_server_1 = require("./meteor/ddp/livedata_server");
+Object.defineProperty(exports, "DDPServer", { enumerable: true, get: function () { return livedata_server_1.DDPServer; } });
+var live_connection_1 = require("./meteor/mongo/live_connection");
+Object.defineProperty(exports, "LiveMongoConnection", { enumerable: true, get: function () { return live_connection_1.LiveMongoConnection; } });
+var live_cursor_1 = require("./meteor/mongo/live_cursor");
+Object.defineProperty(exports, "LiveCursor", { enumerable: true, get: function () { return live_cursor_1.LiveCursor; } });

package/dist/meteor/binary-heap/max_heap.js

@@ -0,0 +1,186 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MaxHeap = void 0;
+// Constructor of Heap
+// - comparator - Function - given two items returns a number
+// - options:
+//   - initData - Array - Optional - the initial data in a format:
+//        Object:
+//          - id - String - unique id of the item
+//          - value - Any - the data value
+//      each value is retained
+//   - IdMap - Constructor - Optional - custom IdMap class to store id->index
+//       mappings internally. Standard IdMap is used by default.
+class MaxHeap {
+    constructor(comparator, options = {}) {
+        if (typeof comparator !== 'function') {
+            throw new Error('Passed comparator is invalid, should be a comparison function');
+        }
+        // a C-style comparator that is given two values and returns a number,
+        // negative if the first value is less than the second, positive if the second
+        // value is greater than the first and zero if they are equal.
+        this._comparator = comparator;
+        if (!options.IdMap) {
+            options.IdMap = Map;
+        }
+        // _heapIdx maps an id to an index in the Heap array the corresponding value
+        // is located on.
+        this._heapIdx = new options.IdMap;
+        // The Heap data-structure implemented as a 0-based contiguous array where
+        // every item on index idx is a node in a complete binary tree. Every node can
+        // have children on indexes idx*2+1 and idx*2+2, except for the leaves. Every
+        // node has a parent on index (idx-1)/2;
+        this._heap = [];
+        // If the initial array is passed, we can build the heap in linear time
+        // complexity (O(N)) compared to linearithmic time complexity (O(nlogn)) if
+        // we push elements one by one.
+        if (Array.isArray(options.initData)) {
+            this._initFromData(options.initData);
+        }
+    }
+    // Builds a new heap in-place in linear time based on passed data
+    _initFromData(data) {
+        this._heap = data.map(({ id, value }) => ({ id, value }));
+        data.forEach(({ id }, i) => this._heapIdx.set(id, i));
+        if (!data.length) {
+            return;
+        }
+        // start from the first non-leaf - the parent of the last leaf
+        for (let i = parentIdx(data.length - 1); i >= 0; i--) {
+            this._downHeap(i);
+        }
+    }
+    _downHeap(idx) {
+        while (leftChildIdx(idx) < this.size()) {
+            const left = leftChildIdx(idx);
+            const right = rightChildIdx(idx);
+            let largest = idx;
+            if (left < this.size()) {
+                largest = this._maxIndex(largest, left);
+            }
+            if (right < this.size()) {
+                largest = this._maxIndex(largest, right);
+            }
+            if (largest === idx) {
+                break;
+            }
+            this._swap(largest, idx);
+            idx = largest;
+        }
+    }
+    _upHeap(idx) {
+        while (idx > 0) {
+            const parent = parentIdx(idx);
+            if (this._maxIndex(parent, idx) === idx) {
+                this._swap(parent, idx);
+                idx = parent;
+            }
+            else {
+                break;
+            }
+        }
+    }
+    _maxIndex(idxA, idxB) {
+        const valueA = this._get(idxA);
+        const valueB = this._get(idxB);
+        return this._comparator(valueA, valueB) >= 0 ? idxA : idxB;
+    }
+    // Internal: gets raw data object placed on idxth place in heap
+    _get(idx) {
+        return this._heap[idx].value;
+    }
+    _swap(idxA, idxB) {
+        const recA = this._heap[idxA];
+        const recB = this._heap[idxB];
+        this._heapIdx.set(recA.id, idxB);
+        this._heapIdx.set(recB.id, idxA);
+        this._heap[idxA] = recB;
+        this._heap[idxB] = recA;
+    }
+    get(id) {
+        return this.has(id) ?
+            this._get(this._heapIdx.get(id)) :
+            null;
+    }
+    set(id, value) {
+        if (this.has(id)) {
+            if (this.get(id) === value) {
+                return;
+            }
+            const idx = this._heapIdx.get(id);
+            this._heap[idx].value = value;
+            // Fix the new value's position
+            // Either bubble new value up if it is greater than its parent
+            this._upHeap(idx);
+            // or bubble it down if it is smaller than one of its children
+            this._downHeap(idx);
+        }
+        else {
+            this._heapIdx.set(id, this._heap.length);
+            this._heap.push({ id, value });
+            this._upHeap(this._heap.length - 1);
+        }
+    }
+    remove(id) {
+        if (this.has(id)) {
+            const last = this._heap.length - 1;
+            const idx = this._heapIdx.get(id);
+            if (idx !== last) {
+                this._swap(idx, last);
+                this._heap.pop();
+                this._heapIdx.remove(id);
+                // Fix the swapped value's position
+                this._upHeap(idx);
+                this._downHeap(idx);
+            }
+            else {
+                this._heap.pop();
+                this._heapIdx.remove(id);
+            }
+        }
+    }
+    has(id) {
+        return this._heapIdx.has(id);
+    }
+    empty() {
+        return !this.size();
+    }
+    clear() {
+        this._heap = [];
+        this._heapIdx.clear();
+    }
+    // iterate over values in no particular order
+    forEach(iterator) {
+        this._heap.forEach(obj => iterator(obj.value, obj.id));
+    }
+    size() {
+        return this._heap.length;
+    }
+    setDefault(id, def) {
+        if (this.has(id)) {
+            return this.get(id);
+        }
+        this.set(id, def);
+        return def;
+    }
+    clone() {
+        const clone = new MaxHeap(this._comparator, { initData: this._heap });
+        return clone;
+    }
+    maxElementId() {
+        return this.size() ? this._heap[0].id : null;
+    }
+    _selfCheck() {
+        for (let i = 1; i < this._heap.length; i++) {
+            if (this._maxIndex(parentIdx(i), i) !== parentIdx(i)) {
+                throw new Error(`An item with id ${this._heap[i].id}` +
+                    " has a parent younger than it: " +
+                    this._heap[parentIdx(i)].id);
+            }
+        }
+    }
+}
+exports.MaxHeap = MaxHeap;
+const leftChildIdx = i => i * 2 + 1;
+const rightChildIdx = i => i * 2 + 2;
+const parentIdx = i => (i - 1) >> 1;

package/dist/meteor/binary-heap/min_heap.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MinHeap = void 0;
+const max_heap_1 = require("./max_heap");
+class MinHeap extends max_heap_1.MaxHeap {
+    constructor(comparator, options) {
+        super((a, b) => -comparator(a, b), options);
+    }
+    maxElementId() {
+        throw new Error("Cannot call maxElementId on MinHeap");
+    }
+    minElementId() {
+        return super.maxElementId();
+    }
+}
+exports.MinHeap = MinHeap;
+;

package/dist/meteor/binary-heap/min_max_heap.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MinMaxHeap = void 0;
+const max_heap_1 = require("./max_heap");
+const min_heap_1 = require("./min_heap");
+// This implementation of Min/Max-Heap is just a subclass of Max-Heap
+// with a Min-Heap as an encapsulated property.
+//
+// Most of the operations are just proxy methods to call the same method on both
+// heaps.
+//
+// This implementation takes 2*N memory but is fairly simple to write and
+// understand. And the constant factor of a simple Heap is usually smaller
+// compared to other two-way priority queues like Min/Max Heaps
+// (http://www.cs.otago.ac.nz/staffpriv/mike/Papers/MinMaxHeaps/MinMaxHeaps.pdf)
+// and Interval Heaps
+// (http://www.cise.ufl.edu/~sahni/dsaac/enrich/c13/double.htm)
+class MinMaxHeap extends max_heap_1.MaxHeap {
+    constructor(comparator, options) {
+        super(comparator, options);
+        this._minHeap = new min_heap_1.MinHeap(comparator, options);
+    }
+    set(id, value) {
+        super.set(id, value);
+        this._minHeap.set(id, value);
+    }
+    remove(id) {
+        super.remove(id);
+        this._minHeap.remove(id);
+    }
+    clear() {
+        super.clear();
+        this._minHeap.clear();
+    }
+    setDefault(id, def) {
+        super.setDefault(id, def);
+        return this._minHeap.setDefault(id, def);
+    }
+    clone() {
+        const clone = new MinMaxHeap(this._comparator, { initData: this._heap });
+        return clone;
+    }
+    minElementId() {
+        return this._minHeap.minElementId();
+    }
+}
+exports.MinMaxHeap = MinMaxHeap;
+;

package/dist/meteor/callback-hook/hook.js

@@ -0,0 +1,78 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Hook = void 0;
+const hasOwn = Object.prototype.hasOwnProperty;
+class Hook {
+    constructor(options) {
+        options = options || {};
+        this.nextCallbackId = 0;
+        this.callbacks = Object.create(null);
+        // Whether to wrap callbacks with Meteor.bindEnvironment
+        if (options.exceptionHandler) {
+            this.exceptionHandler = options.exceptionHandler;
+        }
+        else if (options.debugPrintExceptions) {
+            if (typeof options.debugPrintExceptions !== "string") {
+                throw new Error("Hook option debugPrintExceptions should be a string");
+            }
+            this.exceptionHandler = options.debugPrintExceptions;
+        }
+    }
+    register(callback) {
+        var exceptionHandler = this.exceptionHandler || function (exception) {
+            // Note: this relies on the undocumented fact that if bindEnvironment's
+            // onException throws, and you are invoking the callback either in the
+            // browser or from within a Fiber in Node, the exception is propagated.
+            throw exception;
+        };
+        callback = dontBindEnvironment(callback, exceptionHandler);
+        var id = this.nextCallbackId++;
+        this.callbacks[id] = callback;
+        return {
+            callback,
+            stop: () => {
+                delete this.callbacks[id];
+            }
+        };
+    }
+    // For each registered callback, call the passed iterator function
+    // with the callback.
+    //
+    // The iterator function can choose whether or not to call the
+    // callback.  (For example, it might not call the callback if the
+    // observed object has been closed or terminated).
+    //
+    // The iteration is stopped if the iterator function returns a falsy
+    // value or throws an exception.
+    each(iterator) {
+        var ids = Object.keys(this.callbacks);
+        for (var i = 0; i < ids.length; ++i) {
+            var id = ids[i];
+            // check to see if the callback was removed during iteration
+            if (hasOwn.call(this.callbacks, id)) {
+                var callback = this.callbacks[id];
+                if (!iterator(callback)) {
+                    break;
+                }
+            }
+        }
+    }
+}
+exports.Hook = Hook;
+function dontBindEnvironment(func, onException, _this) {
+    if (!onException || typeof (onException) === 'string') {
+        var description = onException || "callback of async function";
+        onException = function (error) {
+            console.error("Exception in " + description, error);
+        };
+    }
+    return function (...args) {
+        try {
+            var ret = func.apply(_this, args);
+        }
+        catch (e) {
+            onException(e);
+        }
+        return ret;
+    };
+}

package/dist/meteor/ddp/crossbar.js

@@ -0,0 +1,136 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports._InvalidationCrossbar = exports._Crossbar = void 0;
+const ejson_1 = require("../ejson/ejson");
+// A "crossbar" is a class that provides structured notification registration.
+// See _match for the definition of how a notification matches a trigger.
+// All notifications and triggers must have a string key named 'collection'.
+class _Crossbar {
+    constructor(options) {
+        this.listenersByCollection = {};
+        this.listenersByCollectionCount = {};
+        options = options || {};
+        this.nextId = 1;
+    }
+    // msg is a trigger or a notification
+    _collectionForMessage(msg) {
+        if (!msg.hasOwnProperty('collection')) {
+            return '';
+        }
+        else if (typeof (msg.collection) === 'string') {
+            if (msg.collection === '')
+                throw Error("Message has empty collection!");
+            return msg.collection;
+        }
+        else {
+            throw Error("Message has non-string collection!");
+        }
+    }
+    // Listen for notification that match 'trigger'. A notification
+    // matches if it has the key-value pairs in trigger as a
+    // subset. When a notification matches, call 'callback', passing
+    // the actual notification.
+    //
+    // Returns a listen handle, which is an object with a method
+    // stop(). Call stop() to stop listening.
+    //
+    // XXX It should be legal to call fire() from inside a listen()
+    // callback?
+    listen(trigger, callback) {
+        var self = this;
+        var id = self.nextId++;
+        var collection = self._collectionForMessage(trigger);
+        var record = { trigger: (0, ejson_1.clone)(trigger), callback: callback };
+        if (!self.listenersByCollection.hasOwnProperty(collection)) {
+            self.listenersByCollection[collection] = {};
+            self.listenersByCollectionCount[collection] = 0;
+        }
+        self.listenersByCollection[collection][id] = record;
+        self.listenersByCollectionCount[collection]++;
+        return {
+            stop: function () {
+                delete self.listenersByCollection[collection][id];
+                self.listenersByCollectionCount[collection]--;
+                if (self.listenersByCollectionCount[collection] === 0) {
+                    delete self.listenersByCollection[collection];
+                    delete self.listenersByCollectionCount[collection];
+                }
+            }
+        };
+    }
+    // Fire the provided 'notification' (an object whose attribute
+    // values are all JSON-compatibile) -- inform all matching listeners
+    // (registered with listen()).
+    //
+    // If fire() is called inside a write fence, then each of the
+    // listener callbacks will be called inside the write fence as well.
+    //
+    // The listeners may be invoked in parallel, rather than serially.
+    fire(notification) {
+        var self = this;
+        var collection = self._collectionForMessage(notification);
+        if (!self.listenersByCollection.hasOwnProperty(collection)) {
+            return;
+        }
+        var listenersForCollection = self.listenersByCollection[collection];
+        var callbackIds = [];
+        Object.entries(listenersForCollection).forEach(function ([id, l]) {
+            if (self._matches(notification, l.trigger)) {
+                callbackIds.push(id);
+            }
+        });
+        // Listener callbacks can yield, so we need to first find all the ones that
+        // match in a single iteration over self.listenersByCollection (which can't
+        // be mutated during this iteration), and then invoke the matching
+        // callbacks, checking before each call to ensure they haven't stopped.
+        // Note that we don't have to check that
+        // self.listenersByCollection[collection] still === listenersForCollection,
+        // because the only way that stops being true is if listenersForCollection
+        // first gets reduced down to the empty object (and then never gets
+        // increased again).
+        for (const id of callbackIds) {
+            if (listenersForCollection.hasOwnProperty(id)) {
+                listenersForCollection[id].callback(notification);
+            }
+        }
+    }
+    // A notification matches a trigger if all keys that exist in both are equal.
+    //
+    // Examples:
+    //  N:{collection: "C"} matches T:{collection: "C"}
+    //    (a non-targeted write to a collection matches a
+    //     non-targeted query)
+    //  N:{collection: "C", id: "X"} matches T:{collection: "C"}
+    //    (a targeted write to a collection matches a non-targeted query)
+    //  N:{collection: "C"} matches T:{collection: "C", id: "X"}
+    //    (a non-targeted write to a collection matches a
+    //     targeted query)
+    //  N:{collection: "C", id: "X"} matches T:{collection: "C", id: "X"}
+    //    (a targeted write to a collection matches a targeted query targeted
+    //     at the same document)
+    //  N:{collection: "C", id: "X"} does not match T:{collection: "C", id: "Y"}
+    //    (a targeted write to a collection does not match a targeted query
+    //     targeted at a different document)
+    _matches(notification, trigger) {
+        // Most notifications that use the crossbar have a string `collection` and
+        // maybe an `id` that is a string or ObjectID. We're already dividing up
+        // triggers by collection, but let's fast-track "nope, different ID" (and
+        // avoid the overly generic EJSON.equals). This makes a noticeable
+        // performance difference; see https://github.com/meteor/meteor/pull/3697
+        if (typeof (notification.id) === 'string' && typeof (trigger.id) === 'string' && notification.id !== trigger.id) {
+            return false;
+        }
+        return Object.entries(trigger).every(([key, triggerValue]) => {
+            return !notification.hasOwnProperty(key) || (0, ejson_1.equals)(triggerValue, notification[key]);
+        });
+    }
+}
+exports._Crossbar = _Crossbar;
+// The "invalidation crossbar" is a specific instance used by the DDP server to
+// implement write fence notifications. Listener callbacks on this crossbar
+// should call beginWrite on the current write fence before they return, if they
+// want to delay the write fence from firing (ie, the DDP method-data-updated
+// message from being sent).
+exports._InvalidationCrossbar = new _Crossbar({
+    factName: "invalidation-crossbar-listeners"
+});

package/dist/meteor/ddp/heartbeat.js

@@ -0,0 +1,77 @@
+"use strict";
+// Heartbeat options:
+//   heartbeatInterval: interval to send pings, in milliseconds.
+//   heartbeatTimeout: timeout to close the connection if a reply isn't
+//     received, in milliseconds.
+//   sendPing: function to call to send a ping on the connection.
+//   onTimeout: function to call to close the connection.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Heartbeat = void 0;
+class Heartbeat {
+    constructor(options) {
+        this.heartbeatInterval = options.heartbeatInterval;
+        this.heartbeatTimeout = options.heartbeatTimeout;
+        this._sendPing = options.sendPing;
+        this._onTimeout = options.onTimeout;
+        this._seenPacket = false;
+        this._heartbeatIntervalHandle = null;
+        this._heartbeatTimeoutHandle = null;
+    }
+    stop() {
+        this._clearHeartbeatIntervalTimer();
+        this._clearHeartbeatTimeoutTimer();
+    }
+    start() {
+        this.stop();
+        this._startHeartbeatIntervalTimer();
+    }
+    _startHeartbeatIntervalTimer() {
+        this._heartbeatIntervalHandle = setInterval(() => this._heartbeatIntervalFired(), this.heartbeatInterval);
+    }
+    _startHeartbeatTimeoutTimer() {
+        this._heartbeatTimeoutHandle = setTimeout(() => this._heartbeatTimeoutFired(), this.heartbeatTimeout);
+    }
+    _clearHeartbeatIntervalTimer() {
+        if (this._heartbeatIntervalHandle) {
+            clearInterval(this._heartbeatIntervalHandle);
+            this._heartbeatIntervalHandle = null;
+        }
+    }
+    _clearHeartbeatTimeoutTimer() {
+        if (this._heartbeatTimeoutHandle) {
+            clearTimeout(this._heartbeatTimeoutHandle);
+            this._heartbeatTimeoutHandle = null;
+        }
+    }
+    // The heartbeat interval timer is fired when we should send a ping.
+    _heartbeatIntervalFired() {
+        // don't send ping if we've seen a packet since we last checked,
+        // *or* if we have already sent a ping and are awaiting a timeout.
+        // That shouldn't happen, but it's possible if
+        // `this.heartbeatInterval` is smaller than
+        // `this.heartbeatTimeout`.
+        if (!this._seenPacket && !this._heartbeatTimeoutHandle) {
+            this._sendPing();
+            // Set up timeout, in case a pong doesn't arrive in time.
+            this._startHeartbeatTimeoutTimer();
+        }
+        this._seenPacket = false;
+    }
+    // The heartbeat timeout timer is fired when we sent a ping, but we
+    // timed out waiting for the pong.
+    _heartbeatTimeoutFired() {
+        this._heartbeatTimeoutHandle = null;
+        this._onTimeout();
+    }
+    messageReceived() {
+        // Tell periodic checkin that we have seen a packet, and thus it
+        // does not need to send a ping this cycle.
+        this._seenPacket = true;
+        // If we were waiting for a pong, we got it.
+        if (this._heartbeatTimeoutHandle) {
+            this._clearHeartbeatTimeoutTimer();
+        }
+    }
+}
+exports.Heartbeat = Heartbeat;
+;