mongodb-livedata-server 0.0.1

package/meteor/mongo/polling_observe_driver.ts

@@ -0,0 +1,247 @@
+import { _InvalidationCrossbar } from "../ddp/crossbar";
+import { _WriteFence } from "../ddp/writefence";
+import { LiveMongoConnection } from "./live_connection";
+import { ObserveMultiplexer } from "./observe_multiplexer";
+import { CursorDescription } from "./live_cursor";
+import { DiffSequence } from "../diff-sequence/diff";
+import { _SynchronousQueue } from "./synchronous-queue";
+import { listenAll } from "./observe_driver_utils";
+import { IdMap } from "../id-map/id_map";
+
+var POLLING_THROTTLE_MS = +process.env.METEOR_POLLING_THROTTLE_MS || 50;
+var POLLING_INTERVAL_MS = +process.env.METEOR_POLLING_INTERVAL_MS || 10 * 1000;
+
+interface PollingObserveDriverOptions<TOrdered extends boolean> {
+    cursorDescription: CursorDescription<any>;
+    mongoHandle: LiveMongoConnection;
+    ordered: TOrdered;
+    multiplexer: ObserveMultiplexer;
+    stopCallbacks?: (() => void)[];
+    matcher?: any;
+    sorter?: any;
+}
+
+type ResultsType<TOrdered> = TOrdered extends true ? any[] : IdMap;
+
+export class PollingObserveDriver<TOrdered extends boolean> {
+    private _cursorDescription: CursorDescription<any>;
+    private _mongoHandle: LiveMongoConnection;
+    private _ordered: boolean;
+    private _multiplexer: ObserveMultiplexer;
+    private _stopCallbacks: (() => void)[];
+    private _stopped: boolean;
+
+    private _results: ResultsType<TOrdered> | null;
+    private _pollsScheduledButNotStarted: number;
+    private _pendingWrites: { committed: () => void }[];
+    private _ensurePollIsScheduled: Function;
+    private _taskQueue: _SynchronousQueue;
+
+    constructor(options: PollingObserveDriverOptions<TOrdered>) {
+        var self = this;
+
+        self._cursorDescription = options.cursorDescription;
+        self._mongoHandle = options.mongoHandle;
+        self._ordered = options.ordered;
+        self._multiplexer = options.multiplexer;
+        self._stopCallbacks = [];
+        self._stopped = false;
+
+        // previous results snapshot.  on each poll cycle, diffs against
+        // results drives the callbacks.
+        self._results = null;
+
+        // The number of _pollMongo calls that have been added to self._taskQueue but
+        // have not started running. Used to make sure we never schedule more than one
+        // _pollMongo (other than possibly the one that is currently running). It's
+        // also used by _suspendPolling to pretend there's a poll scheduled. Usually,
+        // it's either 0 (for "no polls scheduled other than maybe one currently
+        // running") or 1 (for "a poll scheduled that isn't running yet"), but it can
+        // also be 2 if incremented by _suspendPolling.
+        self._pollsScheduledButNotStarted = 0;
+        self._pendingWrites = []; // people to notify when polling completes
+
+        // Make sure to create a separately throttled function for each
+        // PollingObserveDriver object.
+        self._ensurePollIsScheduled = throttle(
+            self._unthrottledEnsurePollIsScheduled,
+            self._cursorDescription.options.pollingThrottleMs || POLLING_THROTTLE_MS /* ms */
+        );
+
+        // XXX figure out if we still need a queue
+        self._taskQueue = new _SynchronousQueue();
+
+        var listenersHandle = listenAll(
+            self._cursorDescription, function (notification) {
+                // When someone does a transaction that might affect us, schedule a poll
+                // of the database. If that transaction happens inside of a write fence,
+                // block the fence until we've polled and notified observers.
+                var fence = _WriteFence._CurrentWriteFence;
+                if (fence)
+                    self._pendingWrites.push(fence.beginWrite());
+                // Ensure a poll is scheduled... but if we already know that one is,
+                // don't hit the throttled _ensurePollIsScheduled function (which might
+                // lead to us calling it unnecessarily in <pollingThrottleMs> ms).
+                if (self._pollsScheduledButNotStarted === 0)
+                    self._ensurePollIsScheduled();
+            }
+        );
+        self._stopCallbacks.push(function () { listenersHandle.stop(); });
+
+        // every once and a while, poll even if we don't think we're dirty, for
+        // eventual consistency with database writes from outside the Meteor
+        // universe.
+        var pollingInterval = self._cursorDescription.options.pollingIntervalMs || POLLING_INTERVAL_MS;
+        var intervalHandle = setInterval(() => self._ensurePollIsScheduled(), pollingInterval);
+        self._stopCallbacks.push(function () {
+            clearInterval(intervalHandle);
+        });
+
+        // Make sure we actually poll soon!
+        self._unthrottledEnsurePollIsScheduled();
+
+    };
+
+    // This is always called through _.throttle (except once at startup).
+    _unthrottledEnsurePollIsScheduled() {
+        var self = this;
+        if (self._pollsScheduledButNotStarted > 0)
+            return;
+        ++self._pollsScheduledButNotStarted;
+        self._taskQueue.queueTask(async () => await self._pollMongo());
+    }
+
+    async _pollMongo() {
+        var self = this;
+        --self._pollsScheduledButNotStarted;
+
+        if (self._stopped)
+            return;
+
+        var first = false;
+        var newResults: any;
+        var oldResults: any = self._results;
+        if (!oldResults) {
+            first = true;
+            // XXX maybe use OrderedDict instead?
+            oldResults = self._ordered ? [] : new IdMap();
+        }
+
+        // Save the list of pending writes which this round will commit.
+        var writesForCycle = self._pendingWrites;
+        self._pendingWrites = [];
+
+        // Get the new query results. (This yields.)
+        try {
+            const cursor = self._mongoHandle.db.collection(self._cursorDescription.collectionName).find(self._cursorDescription.selector);
+            if (!self._ordered) {
+                newResults = new IdMap();
+                await cursor.forEach((doc) => (newResults as IdMap).set(doc._id, doc));
+            } else
+                newResults = await cursor.toArray();
+        } catch (e) {
+            if (first && typeof (e.code) === 'number') {
+                // This is an error document sent to us by mongod, not a connection
+                // error generated by the client. And we've never seen this query work
+                // successfully. Probably it's a bad selector or something, so we should
+                // NOT retry. Instead, we should halt the observe (which ends up calling
+                // `stop` on us).
+                self._multiplexer.queryError(
+                    new Error(
+                        "Exception while polling query " +
+                        JSON.stringify(self._cursorDescription) + ": " + e.message));
+                return;
+            }
+
+            // getRawObjects can throw if we're having trouble talking to the
+            // database.  That's fine --- we will repoll later anyway. But we should
+            // make sure not to lose track of this cycle's writes.
+            // (It also can throw if there's just something invalid about this query;
+            // unfortunately the ObserveDriver API doesn't provide a good way to
+            // "cancel" the observe from the inside in this case.
+            Array.prototype.push.apply(self._pendingWrites, writesForCycle);
+            console.warn("Exception while polling query " + JSON.stringify(self._cursorDescription), e);
+            return;
+        }
+
+        // Run diffs.
+        // This will trigger the callbacks via the multiplexer
+        if (!self._stopped) {
+            DiffSequence.diffQueryChanges(self._ordered, oldResults, newResults, self._multiplexer);
+        }
+
+        // Signals the multiplexer to allow all observeChanges calls that share this
+        // multiplexer to return. (This happens asynchronously, via the
+        // multiplexer's queue.)
+        if (first)
+            self._multiplexer.ready();
+
+        // Replace self._results atomically.  (This assignment is what makes `first`
+        // stay through on the next cycle, so we've waited until after we've
+        // committed to ready-ing the multiplexer.)
+        self._results = newResults;
+
+        // Once the ObserveMultiplexer has processed everything we've done in this
+        // round, mark all the writes which existed before this call as
+        // commmitted. (If new writes have shown up in the meantime, there'll
+        // already be another _pollMongo task scheduled.)
+        self._multiplexer.onFlush(function () {
+            for (const w of writesForCycle) {
+                w.committed();
+            }
+        });
+    }
+
+    stop() {
+        var self = this;
+        self._stopped = true;
+        for (const c of self._stopCallbacks) {
+            c();
+        }
+        // Release any write fences that are waiting on us.
+        for (const w of self._pendingWrites) {
+            w.committed();
+        }
+    }
+}
+
+function throttle(func: Function, wait: number, options?) {
+    var timeout, context, args, result;
+    var previous = 0;
+    if (!options) options = {};
+
+    var later = function () {
+        previous = options.leading === false ? 0 : Date.now();
+        timeout = null;
+        result = func.apply(context, args);
+        if (!timeout) context = args = null;
+    };
+
+    var throttled = function () {
+        var _now = Date.now();
+        if (!previous && options.leading === false) previous = _now;
+        var remaining = wait - (_now - previous);
+        context = this;
+        args = arguments;
+        if (remaining <= 0 || remaining > wait) {
+            if (timeout) {
+                clearTimeout(timeout);
+                timeout = null;
+            }
+            previous = _now;
+            result = func.apply(context, args);
+            if (!timeout) context = args = null;
+        } else if (!timeout && options.trailing !== false) {
+            timeout = setTimeout(later, remaining);
+        }
+        return result;
+    };
+
+    (throttled as any).cancel = function () {
+        clearTimeout(timeout);
+        previous = 0;
+        timeout = context = args = null;
+    };
+
+    return throttled;
+}

package/meteor/mongo/synchronous-cursor.ts

@@ -0,0 +1,293 @@
+import MongoDB from "mongodb";
+import { equals } from "../ejson/ejson";
+import { OPLOG_COLLECTION } from "./oplog_tailing";
+import { CursorDescription } from "./live_cursor";
+
+export function _createSynchronousCursor(db: MongoDB.Db, cursorDescription: CursorDescription<any>, options?) {
+    const { useTransform } = options || {};
+
+    var collection = db.collection(cursorDescription.collectionName);
+    var cursorOptions = cursorDescription.options;
+    var mongoOptions = {
+        sort: cursorOptions.sort,
+        limit: cursorOptions.limit,
+        skip: cursorOptions.skip,
+        projection: cursorOptions.projection,
+        readPreference: cursorOptions.readPreference,
+        numberOfRetries: undefined
+    };
+
+    // Do we want a tailable cursor (which only works on capped collections)?
+    if (cursorOptions.tailable) {
+        mongoOptions.numberOfRetries = -1;
+    }
+
+    var dbCursor = collection.find(cursorDescription.selector, mongoOptions);
+
+    // Do we want a tailable cursor (which only works on capped collections)?
+    if (cursorOptions.tailable) {
+        // We want a tailable cursor...
+        dbCursor.addCursorFlag("tailable", true)
+        // ... and for the server to wait a bit if any getMore has no data (rather
+        // than making us put the relevant sleeps in the client)...
+        dbCursor.addCursorFlag("awaitData", true)
+
+        // And if this is on the oplog collection and the cursor specifies a 'ts',
+        // then set the undocumented oplog replay flag, which does a special scan to
+        // find the first document (instead of creating an index on ts). This is a
+        // very hard-coded Mongo flag which only works on the oplog collection and
+        // only works with the ts field.
+        if (cursorDescription.collectionName === OPLOG_COLLECTION &&
+            cursorDescription.selector.ts) {
+            dbCursor.addCursorFlag("oplogReplay", true)
+        }
+    }
+
+    if (typeof cursorOptions.maxTimeMs !== 'undefined') {
+        dbCursor = dbCursor.maxTimeMS(cursorOptions.maxTimeMs);
+    }
+    if (typeof cursorOptions.hint !== 'undefined') {
+        dbCursor = dbCursor.hint(cursorOptions.hint);
+    }
+
+    return new SynchronousCursor(dbCursor, cursorDescription, { useTransform });
+};
+
+export class SynchronousCursor {
+
+    private _transform: any;
+    private _visitedIds: Set<string>;
+
+    constructor(
+        private _dbCursor: MongoDB.FindCursor,
+        private _cursorDescription: CursorDescription<any>,
+        options: { useTransform?: boolean }
+    ) {
+        var self = this;
+
+        if (options.useTransform && _cursorDescription.options.transform) {
+            self._transform = wrapTransform(_cursorDescription.options.transform);
+        } else {
+            self._transform = null;
+        }
+
+        self._visitedIds = new Set();
+    }
+
+    // Returns a Promise for the next object from the cursor, skipping those whose
+    // IDs we've already seen and replacing Mongo atoms with Meteor atoms.
+    async _nextObjectPromise() {
+        var self = this;
+
+        while (true) {
+            var doc = await this._dbCursor.next();
+
+            if (!doc) return null;
+
+            if (!self._cursorDescription.options.tailable && doc.hasOwnProperty('_id')) {
+                // Did Mongo give us duplicate documents in the same cursor? If so,
+                // ignore this one. (Do this before the transform, since transform might
+                // return some unrelated value.) We don't do this for tailable cursors,
+                // because we want to maintain O(1) memory usage. And if there isn't _id
+                // for some reason (maybe it's the oplog), then we don't do this either.
+                // (Be careful to do this for falsey but existing _id, though.)
+                if (self._visitedIds.has(doc._id)) continue;
+                self._visitedIds.add(doc._id);
+            }
+
+            if (self._transform)
+                doc = self._transform(doc);
+
+            return doc;
+        }
+    }
+
+    // Returns a promise which is resolved with the next object (like with
+    // _nextObjectPromise) or rejected if the cursor doesn't return within
+    // timeoutMS ms.
+    async _nextObjectPromiseWithTimeout(timeoutMS: number) {
+        if (!timeoutMS) {
+            return this._nextObjectPromise();
+        }
+        const nextObjectPromise = this._nextObjectPromise();
+        const timeoutErr = new Error('Client-side timeout waiting for next object');
+        const timeoutPromise = new Promise<void>((_resolve, reject) => {
+            setTimeout(() => {
+                reject(timeoutErr);
+            }, timeoutMS);
+        });
+        return Promise.race([nextObjectPromise, timeoutPromise])
+            .catch((err) => {
+                if (err === timeoutErr) {
+                    this._dbCursor.close();
+                }
+                throw err;
+            });
+    }
+
+    close() {
+        this._dbCursor.close();
+    }
+
+    async forEach(callback: (doc: any, index: number, cursor: SynchronousCursor) => void, thisArg?) {
+        var self = this;
+
+        // Get back to the beginning.
+        self._rewind();
+
+        // We implement the loop ourself instead of using self._dbCursor.each,
+        // because "each" will call its callback outside of a fiber which makes it
+        // much more complex to make this function synchronous.
+        var index = 0;
+        while (true) {
+            var doc = await self._nextObjectPromise();
+            if (!doc) return;
+            callback.call(thisArg, doc, index++, self);
+        }
+    }
+
+    _rewind() {
+        var self = this;
+
+        // known to be synchronous
+        self._dbCursor.rewind();
+
+        self._visitedIds = new Set();
+    }
+}
+
+// Wrap a transform function to return objects that have the _id field
+// of the untransformed document. This ensures that subsystems such as
+// the observe-sequence package that call `observe` can keep track of
+// the documents identities.
+//
+// - Require that it returns objects
+// - If the return value has an _id field, verify that it matches the
+//   original _id field
+// - If the return value doesn't have an _id field, add it back.
+function wrapTransform(transform: Function & { __wrappedTransform__?: boolean }) {
+    if (!transform)
+        return null;
+
+    // No need to doubly-wrap transforms.
+    if (transform.__wrappedTransform__)
+        return transform;
+
+    const wrapped = doc => {
+        if (!doc.hasOwnProperty('_id')) {
+            // XXX do we ever have a transform on the oplog's collection? because that
+            // collection has no _id.
+            throw new Error('can only transform documents with _id');
+        }
+
+        const id = doc._id;
+
+        const transformed = transform(doc);
+
+        if (transformed.hasOwnProperty('_id')) {
+            if (!equals(transformed._id, id)) {
+                throw new Error('transformed document can\'t have different _id');
+            }
+        } else {
+            transformed._id = id;
+        }
+
+        return transformed;
+    };
+
+    wrapped.__wrappedTransform__ = true;
+
+    return wrapped;
+};
+
+
+/*
+    forEach(callback, thisArg) {
+        var self = this;
+
+        // Get back to the beginning.
+        self._rewind();
+
+        // We implement the loop ourself instead of using self._dbCursor.each,
+        // because "each" will call its callback outside of a fiber which makes it
+        // much more complex to make this function synchronous.
+        var index = 0;
+        while (true) {
+            var doc = self._nextObject();
+            if (!doc) return;
+            callback.call(thisArg, doc, index++, self._selfForIteration);
+        }
+    }
+
+    // XXX Allow overlapping callback executions if callback yields.
+    map(callback, thisArg) {
+        var self = this;
+        var res = [];
+        self.forEach(function (doc, index) {
+            res.push(callback.call(thisArg, doc, index, self._selfForIteration));
+        });
+        return res;
+    },
+
+    _rewind() {
+        var self = this;
+
+        // known to be synchronous
+        self._dbCursor.rewind();
+
+        self._visitedIds = new LocalCollection._IdMap;
+    }
+
+    // Mostly usable for tailable cursors.
+    close() {
+        var self = this;
+
+        self._dbCursor.close();
+    }
+
+    fetch() {
+        var self = this;
+        return self.map(_.identity);
+    }
+
+    // This method is NOT wrapped in Cursor.
+    getRawObjects(ordered) {
+        var self = this;
+        if (ordered) {
+            return self.fetch();
+        } else {
+            var results = new LocalCollection._IdMap;
+            self.forEach(function (doc) {
+                results.set(doc._id, doc);
+            });
+            return results;
+        }
+    }
+
+    [Symbol.iterator]() {
+        var self = this;
+
+        // Get back to the beginning.
+        self._rewind();
+
+        return {
+            next() {
+                const doc = self._nextObject();
+                return doc ? {
+                    value: doc
+                } : {
+                    done: true
+                };
+            }
+        };
+    };
+
+    [Symbol.asyncIterator]() {
+        const syncResult = this[Symbol.iterator]();
+        return {
+            async next() {
+                return Promise.resolve(syncResult.next());
+            }
+        };
+    }
+*/

package/meteor/mongo/synchronous-queue.ts

@@ -0,0 +1,119 @@
+import DoubleEndedQueue from "double-ended-queue";
+
+type AsyncFunction = (...args: any[]) => Promise<any>;
+type TaskHandle = { task: AsyncFunction, name: string, runTaskResolve?: Function, runTaskReject?: Function };
+
+export class _SynchronousQueue {
+    // List of tasks to run (not including a currently-running task if any). Each
+    // is an object with field 'task' (the task function to run) and 'future' (the
+    // Future associated with the blocking runTask call that queued it, or null if
+    // called from queueTask).
+    private _taskHandles = new DoubleEndedQueue<TaskHandle>();
+
+    // This is true if self._run() is either currently executing or scheduled to
+    // do so soon.
+    private _runningOrRunScheduled = false;
+
+    // This is true if we're currently draining.  While we're draining, a further
+    // drain is a noop, to prevent infinite loops.  "drain" is a heuristic type
+    // operation, that has a meaning like unto "what a naive person would expect
+    // when modifying a table from an observe"
+    private _draining = false;
+
+    constructor() { }
+
+    async runTask(task: AsyncFunction) {
+        return new Promise((resolve, reject) => {
+            var handle: TaskHandle = {
+                task: task,
+                name: task.name,
+                runTaskResolve: resolve,
+                runTaskReject: reject
+            };
+            this._taskHandles.push(handle);
+            this._scheduleRun();
+            // Yield. We'll get back here after the task is run (and will throw if the
+            // task throws).
+                
+        })
+    }
+
+    queueTask(task: AsyncFunction) {
+        var self = this;
+        self._taskHandles.push({
+            task: task,
+            name: task.name
+        });
+        self._scheduleRun();
+        // No need to block.
+    }
+
+    async flush() {
+        var self = this;
+        await self.runTask(async () => { });
+    }
+
+    async drain() {
+        var self = this;
+        if (self._draining)
+            return;
+        self._draining = true;
+        while (!self._taskHandles.isEmpty()) {
+            await self.flush();
+        }
+        self._draining = false;
+    };
+
+    _scheduleRun() {
+        // Already running or scheduled? Do nothing.
+        if (this._runningOrRunScheduled)
+            return;
+
+        this._runningOrRunScheduled = true;
+        setImmediate(async () => {
+            await this._run();
+        });
+    };
+
+    async _run() {
+        var self = this;
+
+        if (!self._runningOrRunScheduled)
+            throw new Error("expected to be _runningOrRunScheduled");
+
+        if (self._taskHandles.isEmpty()) {
+            // Done running tasks! Don't immediately schedule another run, but
+            // allow future tasks to do so.
+            self._runningOrRunScheduled = false;
+            return;
+        }
+        var taskHandle = self._taskHandles.shift();
+
+        // Run the task.
+        var exception = undefined;
+        try {
+            await taskHandle.task();
+        } catch (err) {
+            if (taskHandle.runTaskReject) {
+                // We'll throw this exception through runTask.
+                exception = err;
+            } else {
+                console.error("Exception in queued task", err);
+            }
+        }
+
+        // Soon, run the next task, if there is any.
+        self._runningOrRunScheduled = false;
+        self._scheduleRun();
+
+        // If this was queued with runTask, let the runTask call return (throwing if
+        // the task threw).
+        if (taskHandle.runTaskReject) {
+            if (exception)
+                taskHandle.runTaskReject(exception);
+            else
+                taskHandle.runTaskResolve();
+        }
+    }
+
+}