mongodb-livedata-server 0.0.4 → 0.0.6

package/meteor/mongo/polling_observe_driver.ts

@@ -1,247 +0,0 @@
-import { _InvalidationCrossbar } from "../ddp/crossbar";
-import { _WriteFence } from "../ddp/writefence";
-import { LiveMongoConnection } from "./live_connection";
-import { ObserveMultiplexer } from "./observe_multiplexer";
-import { CursorDescription } from "./live_cursor";
-import { DiffSequence } from "../diff-sequence/diff";
-import { _SynchronousQueue } from "./synchronous-queue";
-import { listenAll } from "./observe_driver_utils";
-import { IdMap } from "../id-map/id_map";
-
-var POLLING_THROTTLE_MS = +process.env.METEOR_POLLING_THROTTLE_MS || 50;
-var POLLING_INTERVAL_MS = +process.env.METEOR_POLLING_INTERVAL_MS || 10 * 1000;
-
-interface PollingObserveDriverOptions<TOrdered extends boolean> {
-    cursorDescription: CursorDescription<any>;
-    mongoHandle: LiveMongoConnection;
-    ordered: TOrdered;
-    multiplexer: ObserveMultiplexer;
-    stopCallbacks?: (() => void)[];
-    matcher?: any;
-    sorter?: any;
-}
-
-type ResultsType<TOrdered> = TOrdered extends true ? any[] : IdMap;
-
-export class PollingObserveDriver<TOrdered extends boolean> {
-    private _cursorDescription: CursorDescription<any>;
-    private _mongoHandle: LiveMongoConnection;
-    private _ordered: boolean;
-    private _multiplexer: ObserveMultiplexer;
-    private _stopCallbacks: (() => void)[];
-    private _stopped: boolean;
-
-    private _results: ResultsType<TOrdered> | null;
-    private _pollsScheduledButNotStarted: number;
-    private _pendingWrites: { committed: () => void }[];
-    private _ensurePollIsScheduled: Function;
-    private _taskQueue: _SynchronousQueue;
-
-    constructor(options: PollingObserveDriverOptions<TOrdered>) {
-        var self = this;
-
-        self._cursorDescription = options.cursorDescription;
-        self._mongoHandle = options.mongoHandle;
-        self._ordered = options.ordered;
-        self._multiplexer = options.multiplexer;
-        self._stopCallbacks = [];
-        self._stopped = false;
-
-        // previous results snapshot.  on each poll cycle, diffs against
-        // results drives the callbacks.
-        self._results = null;
-
-        // The number of _pollMongo calls that have been added to self._taskQueue but
-        // have not started running. Used to make sure we never schedule more than one
-        // _pollMongo (other than possibly the one that is currently running). It's
-        // also used by _suspendPolling to pretend there's a poll scheduled. Usually,
-        // it's either 0 (for "no polls scheduled other than maybe one currently
-        // running") or 1 (for "a poll scheduled that isn't running yet"), but it can
-        // also be 2 if incremented by _suspendPolling.
-        self._pollsScheduledButNotStarted = 0;
-        self._pendingWrites = []; // people to notify when polling completes
-
-        // Make sure to create a separately throttled function for each
-        // PollingObserveDriver object.
-        self._ensurePollIsScheduled = throttle(
-            self._unthrottledEnsurePollIsScheduled,
-            self._cursorDescription.options.pollingThrottleMs || POLLING_THROTTLE_MS /* ms */
-        );
-
-        // XXX figure out if we still need a queue
-        self._taskQueue = new _SynchronousQueue();
-
-        var listenersHandle = listenAll(
-            self._cursorDescription, function (notification) {
-                // When someone does a transaction that might affect us, schedule a poll
-                // of the database. If that transaction happens inside of a write fence,
-                // block the fence until we've polled and notified observers.
-                var fence = _WriteFence._CurrentWriteFence;
-                if (fence)
-                    self._pendingWrites.push(fence.beginWrite());
-                // Ensure a poll is scheduled... but if we already know that one is,
-                // don't hit the throttled _ensurePollIsScheduled function (which might
-                // lead to us calling it unnecessarily in <pollingThrottleMs> ms).
-                if (self._pollsScheduledButNotStarted === 0)
-                    self._ensurePollIsScheduled();
-            }
-        );
-        self._stopCallbacks.push(function () { listenersHandle.stop(); });
-
-        // every once and a while, poll even if we don't think we're dirty, for
-        // eventual consistency with database writes from outside the Meteor
-        // universe.
-        var pollingInterval = self._cursorDescription.options.pollingIntervalMs || POLLING_INTERVAL_MS;
-        var intervalHandle = setInterval(() => self._ensurePollIsScheduled(), pollingInterval);
-        self._stopCallbacks.push(function () {
-            clearInterval(intervalHandle);
-        });
-
-        // Make sure we actually poll soon!
-        self._unthrottledEnsurePollIsScheduled();
-
-    };
-
-    // This is always called through _.throttle (except once at startup).
-    _unthrottledEnsurePollIsScheduled() {
-        var self = this;
-        if (self._pollsScheduledButNotStarted > 0)
-            return;
-        ++self._pollsScheduledButNotStarted;
-        self._taskQueue.queueTask(async () => await self._pollMongo());
-    }
-
-    async _pollMongo() {
-        var self = this;
-        --self._pollsScheduledButNotStarted;
-
-        if (self._stopped)
-            return;
-
-        var first = false;
-        var newResults: any;
-        var oldResults: any = self._results;
-        if (!oldResults) {
-            first = true;
-            // XXX maybe use OrderedDict instead?
-            oldResults = self._ordered ? [] : new IdMap();
-        }
-
-        // Save the list of pending writes which this round will commit.
-        var writesForCycle = self._pendingWrites;
-        self._pendingWrites = [];
-
-        // Get the new query results. (This yields.)
-        try {
-            const cursor = self._mongoHandle.db.collection(self._cursorDescription.collectionName).find(self._cursorDescription.selector);
-            if (!self._ordered) {
-                newResults = new IdMap();
-                await cursor.forEach((doc) => (newResults as IdMap).set(doc._id, doc));
-            } else
-                newResults = await cursor.toArray();
-        } catch (e) {
-            if (first && typeof (e.code) === 'number') {
-                // This is an error document sent to us by mongod, not a connection
-                // error generated by the client. And we've never seen this query work
-                // successfully. Probably it's a bad selector or something, so we should
-                // NOT retry. Instead, we should halt the observe (which ends up calling
-                // `stop` on us).
-                self._multiplexer.queryError(
-                    new Error(
-                        "Exception while polling query " +
-                        JSON.stringify(self._cursorDescription) + ": " + e.message));
-                return;
-            }
-
-            // getRawObjects can throw if we're having trouble talking to the
-            // database.  That's fine --- we will repoll later anyway. But we should
-            // make sure not to lose track of this cycle's writes.
-            // (It also can throw if there's just something invalid about this query;
-            // unfortunately the ObserveDriver API doesn't provide a good way to
-            // "cancel" the observe from the inside in this case.
-            Array.prototype.push.apply(self._pendingWrites, writesForCycle);
-            console.warn("Exception while polling query " + JSON.stringify(self._cursorDescription), e);
-            return;
-        }
-
-        // Run diffs.
-        // This will trigger the callbacks via the multiplexer
-        if (!self._stopped) {
-            DiffSequence.diffQueryChanges(self._ordered, oldResults, newResults, self._multiplexer);
-        }
-
-        // Signals the multiplexer to allow all observeChanges calls that share this
-        // multiplexer to return. (This happens asynchronously, via the
-        // multiplexer's queue.)
-        if (first)
-            self._multiplexer.ready();
-
-        // Replace self._results atomically.  (This assignment is what makes `first`
-        // stay through on the next cycle, so we've waited until after we've
-        // committed to ready-ing the multiplexer.)
-        self._results = newResults;
-
-        // Once the ObserveMultiplexer has processed everything we've done in this
-        // round, mark all the writes which existed before this call as
-        // commmitted. (If new writes have shown up in the meantime, there'll
-        // already be another _pollMongo task scheduled.)
-        self._multiplexer.onFlush(function () {
-            for (const w of writesForCycle) {
-                w.committed();
-            }
-        });
-    }
-
-    stop() {
-        var self = this;
-        self._stopped = true;
-        for (const c of self._stopCallbacks) {
-            c();
-        }
-        // Release any write fences that are waiting on us.
-        for (const w of self._pendingWrites) {
-            w.committed();
-        }
-    }
-}
-
-function throttle(func: Function, wait: number, options?) {
-    var timeout, context, args, result;
-    var previous = 0;
-    if (!options) options = {};
-
-    var later = function () {
-        previous = options.leading === false ? 0 : Date.now();
-        timeout = null;
-        result = func.apply(context, args);
-        if (!timeout) context = args = null;
-    };
-
-    var throttled = function () {
-        var _now = Date.now();
-        if (!previous && options.leading === false) previous = _now;
-        var remaining = wait - (_now - previous);
-        context = this;
-        args = arguments;
-        if (remaining <= 0 || remaining > wait) {
-            if (timeout) {
-                clearTimeout(timeout);
-                timeout = null;
-            }
-            previous = _now;
-            result = func.apply(context, args);
-            if (!timeout) context = args = null;
-        } else if (!timeout && options.trailing !== false) {
-            timeout = setTimeout(later, remaining);
-        }
-        return result;
-    };
-
-    (throttled as any).cancel = function () {
-        clearTimeout(timeout);
-        previous = 0;
-        timeout = context = args = null;
-    };
-
-    return throttled;
-}

package/meteor/mongo/synchronous-cursor.ts

@@ -1,293 +0,0 @@
-import MongoDB from "mongodb";
-import { equals } from "../ejson/ejson";
-import { OPLOG_COLLECTION } from "./oplog_tailing";
-import { CursorDescription } from "./live_cursor";
-
-export function _createSynchronousCursor(db: MongoDB.Db, cursorDescription: CursorDescription<any>, options?) {
-    const { useTransform } = options || {};
-
-    var collection = db.collection(cursorDescription.collectionName);
-    var cursorOptions = cursorDescription.options;
-    var mongoOptions = {
-        sort: cursorOptions.sort,
-        limit: cursorOptions.limit,
-        skip: cursorOptions.skip,
-        projection: cursorOptions.projection,
-        readPreference: cursorOptions.readPreference,
-        numberOfRetries: undefined
-    };
-
-    // Do we want a tailable cursor (which only works on capped collections)?
-    if (cursorOptions.tailable) {
-        mongoOptions.numberOfRetries = -1;
-    }
-
-    var dbCursor = collection.find(cursorDescription.selector, mongoOptions);
-
-    // Do we want a tailable cursor (which only works on capped collections)?
-    if (cursorOptions.tailable) {
-        // We want a tailable cursor...
-        dbCursor.addCursorFlag("tailable", true)
-        // ... and for the server to wait a bit if any getMore has no data (rather
-        // than making us put the relevant sleeps in the client)...
-        dbCursor.addCursorFlag("awaitData", true)
-
-        // And if this is on the oplog collection and the cursor specifies a 'ts',
-        // then set the undocumented oplog replay flag, which does a special scan to
-        // find the first document (instead of creating an index on ts). This is a
-        // very hard-coded Mongo flag which only works on the oplog collection and
-        // only works with the ts field.
-        if (cursorDescription.collectionName === OPLOG_COLLECTION &&
-            cursorDescription.selector.ts) {
-            dbCursor.addCursorFlag("oplogReplay", true)
-        }
-    }
-
-    if (typeof cursorOptions.maxTimeMs !== 'undefined') {
-        dbCursor = dbCursor.maxTimeMS(cursorOptions.maxTimeMs);
-    }
-    if (typeof cursorOptions.hint !== 'undefined') {
-        dbCursor = dbCursor.hint(cursorOptions.hint);
-    }
-
-    return new SynchronousCursor(dbCursor, cursorDescription, { useTransform });
-};
-
-export class SynchronousCursor {
-
-    private _transform: any;
-    private _visitedIds: Set<string>;
-
-    constructor(
-        private _dbCursor: MongoDB.FindCursor,
-        private _cursorDescription: CursorDescription<any>,
-        options: { useTransform?: boolean }
-    ) {
-        var self = this;
-
-        if (options.useTransform && _cursorDescription.options.transform) {
-            self._transform = wrapTransform(_cursorDescription.options.transform);
-        } else {
-            self._transform = null;
-        }
-
-        self._visitedIds = new Set();
-    }
-
-    // Returns a Promise for the next object from the cursor, skipping those whose
-    // IDs we've already seen and replacing Mongo atoms with Meteor atoms.
-    async _nextObjectPromise() {
-        var self = this;
-
-        while (true) {
-            var doc = await this._dbCursor.next();
-
-            if (!doc) return null;
-
-            if (!self._cursorDescription.options.tailable && doc.hasOwnProperty('_id')) {
-                // Did Mongo give us duplicate documents in the same cursor? If so,
-                // ignore this one. (Do this before the transform, since transform might
-                // return some unrelated value.) We don't do this for tailable cursors,
-                // because we want to maintain O(1) memory usage. And if there isn't _id
-                // for some reason (maybe it's the oplog), then we don't do this either.
-                // (Be careful to do this for falsey but existing _id, though.)
-                if (self._visitedIds.has(doc._id)) continue;
-                self._visitedIds.add(doc._id);
-            }
-
-            if (self._transform)
-                doc = self._transform(doc);
-
-            return doc;
-        }
-    }
-
-    // Returns a promise which is resolved with the next object (like with
-    // _nextObjectPromise) or rejected if the cursor doesn't return within
-    // timeoutMS ms.
-    async _nextObjectPromiseWithTimeout(timeoutMS: number) {
-        if (!timeoutMS) {
-            return this._nextObjectPromise();
-        }
-        const nextObjectPromise = this._nextObjectPromise();
-        const timeoutErr = new Error('Client-side timeout waiting for next object');
-        const timeoutPromise = new Promise<void>((_resolve, reject) => {
-            setTimeout(() => {
-                reject(timeoutErr);
-            }, timeoutMS);
-        });
-        return Promise.race([nextObjectPromise, timeoutPromise])
-            .catch((err) => {
-                if (err === timeoutErr) {
-                    this._dbCursor.close();
-                }
-                throw err;
-            });
-    }
-
-    close() {
-        this._dbCursor.close();
-    }
-
-    async forEach(callback: (doc: any, index: number, cursor: SynchronousCursor) => void, thisArg?) {
-        var self = this;
-
-        // Get back to the beginning.
-        self._rewind();
-
-        // We implement the loop ourself instead of using self._dbCursor.each,
-        // because "each" will call its callback outside of a fiber which makes it
-        // much more complex to make this function synchronous.
-        var index = 0;
-        while (true) {
-            var doc = await self._nextObjectPromise();
-            if (!doc) return;
-            callback.call(thisArg, doc, index++, self);
-        }
-    }
-
-    _rewind() {
-        var self = this;
-
-        // known to be synchronous
-        self._dbCursor.rewind();
-
-        self._visitedIds = new Set();
-    }
-}
-
-// Wrap a transform function to return objects that have the _id field
-// of the untransformed document. This ensures that subsystems such as
-// the observe-sequence package that call `observe` can keep track of
-// the documents identities.
-//
-// - Require that it returns objects
-// - If the return value has an _id field, verify that it matches the
-//   original _id field
-// - If the return value doesn't have an _id field, add it back.
-function wrapTransform(transform: Function & { __wrappedTransform__?: boolean }) {
-    if (!transform)
-        return null;
-
-    // No need to doubly-wrap transforms.
-    if (transform.__wrappedTransform__)
-        return transform;
-
-    const wrapped = doc => {
-        if (!doc.hasOwnProperty('_id')) {
-            // XXX do we ever have a transform on the oplog's collection? because that
-            // collection has no _id.
-            throw new Error('can only transform documents with _id');
-        }
-
-        const id = doc._id;
-
-        const transformed = transform(doc);
-
-        if (transformed.hasOwnProperty('_id')) {
-            if (!equals(transformed._id, id)) {
-                throw new Error('transformed document can\'t have different _id');
-            }
-        } else {
-            transformed._id = id;
-        }
-
-        return transformed;
-    };
-
-    wrapped.__wrappedTransform__ = true;
-
-    return wrapped;
-};
-
-
-/*
-    forEach(callback, thisArg) {
-        var self = this;
-
-        // Get back to the beginning.
-        self._rewind();
-
-        // We implement the loop ourself instead of using self._dbCursor.each,
-        // because "each" will call its callback outside of a fiber which makes it
-        // much more complex to make this function synchronous.
-        var index = 0;
-        while (true) {
-            var doc = self._nextObject();
-            if (!doc) return;
-            callback.call(thisArg, doc, index++, self._selfForIteration);
-        }
-    }
-
-    // XXX Allow overlapping callback executions if callback yields.
-    map(callback, thisArg) {
-        var self = this;
-        var res = [];
-        self.forEach(function (doc, index) {
-            res.push(callback.call(thisArg, doc, index, self._selfForIteration));
-        });
-        return res;
-    },
-
-    _rewind() {
-        var self = this;
-
-        // known to be synchronous
-        self._dbCursor.rewind();
-
-        self._visitedIds = new LocalCollection._IdMap;
-    }
-
-    // Mostly usable for tailable cursors.
-    close() {
-        var self = this;
-
-        self._dbCursor.close();
-    }
-
-    fetch() {
-        var self = this;
-        return self.map(_.identity);
-    }
-
-    // This method is NOT wrapped in Cursor.
-    getRawObjects(ordered) {
-        var self = this;
-        if (ordered) {
-            return self.fetch();
-        } else {
-            var results = new LocalCollection._IdMap;
-            self.forEach(function (doc) {
-                results.set(doc._id, doc);
-            });
-            return results;
-        }
-    }
-
-    [Symbol.iterator]() {
-        var self = this;
-
-        // Get back to the beginning.
-        self._rewind();
-
-        return {
-            next() {
-                const doc = self._nextObject();
-                return doc ? {
-                    value: doc
-                } : {
-                    done: true
-                };
-            }
-        };
-    };
-
-    [Symbol.asyncIterator]() {
-        const syncResult = this[Symbol.iterator]();
-        return {
-            async next() {
-                return Promise.resolve(syncResult.next());
-            }
-        };
-    }
-*/

package/meteor/mongo/synchronous-queue.ts

@@ -1,119 +0,0 @@
-import DoubleEndedQueue from "double-ended-queue";
-
-type AsyncFunction = (...args: any[]) => Promise<any>;
-type TaskHandle = { task: AsyncFunction, name: string, runTaskResolve?: Function, runTaskReject?: Function };
-
-export class _SynchronousQueue {
-    // List of tasks to run (not including a currently-running task if any). Each
-    // is an object with field 'task' (the task function to run) and 'future' (the
-    // Future associated with the blocking runTask call that queued it, or null if
-    // called from queueTask).
-    private _taskHandles = new DoubleEndedQueue<TaskHandle>();
-
-    // This is true if self._run() is either currently executing or scheduled to
-    // do so soon.
-    private _runningOrRunScheduled = false;
-
-    // This is true if we're currently draining.  While we're draining, a further
-    // drain is a noop, to prevent infinite loops.  "drain" is a heuristic type
-    // operation, that has a meaning like unto "what a naive person would expect
-    // when modifying a table from an observe"
-    private _draining = false;
-
-    constructor() { }
-
-    async runTask(task: AsyncFunction) {
-        return new Promise((resolve, reject) => {
-            var handle: TaskHandle = {
-                task: task,
-                name: task.name,
-                runTaskResolve: resolve,
-                runTaskReject: reject
-            };
-            this._taskHandles.push(handle);
-            this._scheduleRun();
-            // Yield. We'll get back here after the task is run (and will throw if the
-            // task throws).
-                
-        })
-    }
-
-    queueTask(task: AsyncFunction) {
-        var self = this;
-        self._taskHandles.push({
-            task: task,
-            name: task.name
-        });
-        self._scheduleRun();
-        // No need to block.
-    }
-
-    async flush() {
-        var self = this;
-        await self.runTask(async () => { });
-    }
-
-    async drain() {
-        var self = this;
-        if (self._draining)
-            return;
-        self._draining = true;
-        while (!self._taskHandles.isEmpty()) {
-            await self.flush();
-        }
-        self._draining = false;
-    };
-
-    _scheduleRun() {
-        // Already running or scheduled? Do nothing.
-        if (this._runningOrRunScheduled)
-            return;
-
-        this._runningOrRunScheduled = true;
-        setImmediate(async () => {
-            await this._run();
-        });
-    };
-
-    async _run() {
-        var self = this;
-
-        if (!self._runningOrRunScheduled)
-            throw new Error("expected to be _runningOrRunScheduled");
-
-        if (self._taskHandles.isEmpty()) {
-            // Done running tasks! Don't immediately schedule another run, but
-            // allow future tasks to do so.
-            self._runningOrRunScheduled = false;
-            return;
-        }
-        var taskHandle = self._taskHandles.shift();
-
-        // Run the task.
-        var exception = undefined;
-        try {
-            await taskHandle.task();
-        } catch (err) {
-            if (taskHandle.runTaskReject) {
-                // We'll throw this exception through runTask.
-                exception = err;
-            } else {
-                console.error("Exception in queued task", err);
-            }
-        }
-
-        // Soon, run the next task, if there is any.
-        self._runningOrRunScheduled = false;
-        self._scheduleRun();
-
-        // If this was queued with runTask, let the runTask call return (throwing if
-        // the task threw).
-        if (taskHandle.runTaskReject) {
-            if (exception)
-                taskHandle.runTaskReject(exception);
-            else
-                taskHandle.runTaskResolve();
-        }
-    }
-
-}