mongodb-livedata-server 0.0.4 → 0.0.6

package/meteor/mongo/live_connection.ts

@@ -1,302 +0,0 @@
-import { Hook } from "../callback-hook/hook";
-import { MongoClient, Db } from "mongodb";
-import { CursorDescription } from "./live_cursor";
-import { stringify } from "../ejson/ejson";
-import { _createSynchronousCursor } from "./synchronous-cursor";
-import { OplogHandle } from "./oplog_tailing";
-import { DocFetcher } from "./doc_fetcher";
-import { ObserveHandle, ObserveMultiplexer } from "./observe_multiplexer";
-import { PollingObserveDriver } from "./polling_observe_driver";
-import { OplogObserveDriver } from "./oplog-observe-driver";
-import { MinimongoMatcher } from "./minimongo_matcher";
-import MinimongoSorter from "./minimongo_sorter";
-
-export class LiveMongoConnection {
-
-    private client: MongoClient;
-    public db: Db;
-
-    public _oplogHandle: OplogHandle = null;
-    public _docFetcher: DocFetcher = null;
-
-    private _observeMultiplexers: Record<string, any> = {};
-    private _onFailoverHook: Hook = new Hook();
-
-    constructor(url: string, options?) {
-        options = options || {};
-
-        var mongoOptions = {
-            ignoreUndefined: true,
-            maxPoolSize: undefined
-        };
-
-        // Internally the oplog connections specify their own maxPoolSize
-        // which we don't want to overwrite with any user defined value
-        if (options.hasOwnProperty("maxPoolSize")) {
-            // If we just set this for "server", replSet will override it. If we just
-            // set it for replSet, it will be ignored if we're not using a replSet.
-            mongoOptions.maxPoolSize = options.maxPoolSize;
-        }
-
-        this.client = new MongoClient(url, mongoOptions);
-        this.db = this.client.db();
-
-        this.client.on('serverDescriptionChanged', event => {
-            // When the connection is no longer against the primary node, execute all
-            // failover hooks. This is important for the driver as it has to re-pool the
-            // query when it happens.
-            if (
-                event.previousDescription.type !== 'RSPrimary' &&
-                event.newDescription.type === 'RSPrimary'
-            ) {
-                this._onFailoverHook.each(callback => {
-                    callback();
-                    return true;
-                });
-            }
-        });
-
-        if (options.oplogUrl) {
-            this._oplogHandle = new OplogHandle(options.oplogUrl, this.db.databaseName);
-            this._docFetcher = new DocFetcher(this.db);
-        }
-    }
-
-    close() {
-        var self = this;
-
-        if (!self.db)
-            throw Error("close called before Connection created?");
-
-        // XXX probably untested
-        var oplogHandle = self._oplogHandle;
-        self._oplogHandle = null;
-        if (oplogHandle)
-            oplogHandle.stop();
-
-        // Use Future.wrap so that errors get thrown. This happens to
-        // work even outside a fiber since the 'close' method is not
-        // actually asynchronous.
-        self.client.close(true);
-    }
-
-    // Tails the cursor described by cursorDescription, most likely on the
-    // oplog. Calls docCallback with each document found. Ignores errors and just
-    // restarts the tail on error.
-    //
-    // If timeoutMS is set, then if we don't get a new document every timeoutMS,
-    // kill and restart the cursor. This is primarily a workaround for #8598.
-    tail<T>(cursorDescription: CursorDescription<T>, docCallback: (doc: T) => void, timeoutMS?: number) {
-        if (!cursorDescription.options.tailable)
-            throw new Error("Can only tail a tailable cursor");
-
-        var cursor = _createSynchronousCursor(this.db, cursorDescription);
-
-        var stopped = false;
-        var lastTS: number;
-        const loop = async () => {
-            var doc = null;
-            while (true) {
-                if (stopped)
-                    return;
-                try {
-                    doc = await cursor._nextObjectPromiseWithTimeout(timeoutMS);
-                } catch (err) {
-                    // There's no good way to figure out if this was actually an error from
-                    // Mongo, or just client-side (including our own timeout error). Ah
-                    // well. But either way, we need to retry the cursor (unless the failure
-                    // was because the observe got stopped).
-                    doc = null;
-                }
-                // Since we awaited a promise above, we need to check again to see if
-                // we've been stopped before calling the callback.
-                if (stopped)
-                    return;
-                if (doc) {
-                    // If a tailable cursor contains a "ts" field, use it to recreate the
-                    // cursor on error. ("ts" is a standard that Mongo uses internally for
-                    // the oplog, and there's a special flag that lets you do binary search
-                    // on it instead of needing to use an index.)
-                    lastTS = doc.ts;
-                    docCallback(doc);
-                } else {
-                    const newSelector = { ...cursorDescription.selector };
-                    if (lastTS) {
-                        (newSelector as any).ts = { $gt: lastTS };
-                    }
-                    const newDescription = new CursorDescription(
-                        cursorDescription.collectionName,
-                        newSelector,
-                        cursorDescription.options
-                    );
-                    cursor = _createSynchronousCursor(this.db, newDescription);
-                    // Mongo failover takes many seconds.  Retry in a bit.  (Without this
-                    // setTimeout, we peg the CPU at 100% and never notice the actual
-                    // failover.
-                    setTimeout(loop, 100);
-                    break;
-                }
-            }
-        };
-
-        setImmediate(loop);
-
-        return {
-            stop: function () {
-                stopped = true;
-                cursor.close();
-            }
-        };
-    }
-
-    _observeChanges(cursorDescription: CursorDescription<any>, ordered: boolean, callbacks: any, nonMutatingCallbacks: boolean) {
-        var self = this;
-
-        if (cursorDescription.options.tailable) {
-            return self._observeChangesTailable(cursorDescription, ordered, callbacks);
-        }
-
-        // You may not filter out _id when observing changes, because the id is a core
-        // part of the observeChanges API.
-        const fieldsOptions = cursorDescription.options.projection;
-        if (fieldsOptions && (fieldsOptions._id === 0 || fieldsOptions._id === false))
-            throw Error("You may not observe a cursor with {fields: {_id: 0}}");
-
-        var observeKey = stringify(Object.assign({ ordered: ordered }, cursorDescription));
-
-        var multiplexer: ObserveMultiplexer, observeDriver;
-        var firstHandle = false;
-
-        // Find a matching ObserveMultiplexer, or create a new one. This next block is
-        // guaranteed to not yield (and it doesn't call anything that can observe a
-        // new query), so no other calls to this function can interleave with it.
-        //Meteor._noYieldsAllowed(function () {
-        if (self._observeMultiplexers.hasOwnProperty(observeKey)) {
-            multiplexer = self._observeMultiplexers[observeKey];
-        } else {
-            firstHandle = true;
-            // Create a new ObserveMultiplexer.
-            multiplexer = new ObserveMultiplexer({
-                ordered: ordered,
-                onStop: function () {
-                    delete self._observeMultiplexers[observeKey];
-                    observeDriver.stop();
-                }
-            });
-            self._observeMultiplexers[observeKey] = multiplexer;
-        }
-        //});
-
-        var observeHandle = new ObserveHandle(
-            multiplexer,
-            callbacks,
-            nonMutatingCallbacks
-        );
-
-        if (firstHandle) {
-            let matcher: MinimongoMatcher;
-            let sorter: MinimongoSorter;
-
-            // At a bare minimum, using the oplog requires us to have an oplog, to
-            // want unordered callbacks, and to not want a callback on the polls
-            // that won't happen.
-            const basicPrerequisites = self._oplogHandle && !ordered && !callbacks._testOnlyPollCallback;
-
-            let selectorIsCompilable = false;
-            // We need to be able to compile the selector. Fall back to polling for
-            // some newfangled $selector that minimongo doesn't support yet.
-            try {
-                matcher = new MinimongoMatcher(cursorDescription.selector);
-                selectorIsCompilable = true;
-            } catch (e) {
-            }
-
-            const supportedByOplog = OplogObserveDriver.cursorSupported(cursorDescription, matcher);
-
-            let cursorIsSortable = false;
-            // And we need to be able to compile the sort, if any.  eg, can't be
-            // {$natural: 1}.
-            if (!cursorDescription.options.sort)
-                cursorIsSortable = true;
-            try {
-                sorter = new MinimongoSorter(cursorDescription.options.sort);
-                cursorIsSortable = true;
-            } catch (e) {
-            }
-
-            const canUseOplog = basicPrerequisites && selectorIsCompilable && cursorIsSortable && supportedByOplog;
-
-            var driverClass = canUseOplog ? OplogObserveDriver : PollingObserveDriver;
-            observeDriver = new driverClass({
-                cursorDescription: cursorDescription,
-                mongoHandle: self,
-                multiplexer: multiplexer,
-                ordered: ordered,
-                matcher,  // ignored by polling
-                sorter  // ignored by polling
-            });
-        }
-
-        // Blocks until the initial adds have been sent.
-        multiplexer.addHandleAndSendInitialAdds(observeHandle);
-
-        return observeHandle;
-    }
-
-    // observeChanges for tailable cursors on capped collections.
-    //
-    // Some differences from normal cursors:
-    //   - Will never produce anything other than 'added' or 'addedBefore'. If you
-    //     do update a document that has already been produced, this will not notice
-    //     it.
-    //   - If you disconnect and reconnect from Mongo, it will essentially restart
-    //     the query, which will lead to duplicate results. This is pretty bad,
-    //     but if you include a field called 'ts' which is inserted as
-    //     new MongoInternals.MongoTimestamp(0, 0) (which is initialized to the
-    //     current Mongo-style timestamp), we'll be able to find the place to
-    //     restart properly. (This field is specifically understood by Mongo with an
-    //     optimization which allows it to find the right place to start without
-    //     an index on ts. It's how the oplog works.)
-    //   - No callbacks are triggered synchronously with the call (there's no
-    //     differentiation between "initial data" and "later changes"; everything
-    //     that matches the query gets sent asynchronously).
-    //   - De-duplication is not implemented.
-    //   - Does not yet interact with the write fence. Probably, this should work by
-    //     ignoring removes (which don't work on capped collections) and updates
-    //     (which don't affect tailable cursors), and just keeping track of the ID
-    //     of the inserted object, and closing the write fence once you get to that
-    //     ID (or timestamp?).  This doesn't work well if the document doesn't match
-    //     the query, though.  On the other hand, the write fence can close
-    //     immediately if it does not match the query. So if we trust minimongo
-    //     enough to accurately evaluate the query against the write fence, we
-    //     should be able to do this...  Of course, minimongo doesn't even support
-    //     Mongo Timestamps yet.
-    _observeChangesTailable(cursorDescription: CursorDescription<any>, ordered: boolean, callbacks: any) {
-        var self = this;
-
-        // Tailable cursors only ever call added/addedBefore callbacks, so it's an
-        // error if you didn't provide them.
-        if ((ordered && !callbacks.addedBefore) ||
-            (!ordered && !callbacks.added)) {
-            throw new Error("Can't observe an " + (ordered ? "ordered" : "unordered")
-                + " tailable cursor without a "
-                + (ordered ? "addedBefore" : "added") + " callback");
-        }
-
-        return self.tail(cursorDescription, function (doc) {
-            var id = doc._id;
-            delete doc._id;
-            // The ts is an implementation detail. Hide it.
-            delete doc.ts;
-            if (ordered) {
-                callbacks.addedBefore(id, doc, null);
-            } else {
-                callbacks.added(id, doc);
-            }
-        });
-    }
-
-    _onFailover(callback: Function) {
-        return this._onFailoverHook.register(callback);
-    }
-}

package/meteor/mongo/live_cursor.ts

@@ -1,79 +0,0 @@
-import { Subscription } from "../ddp/subscription";
-import MongoDB, { WithId } from "mongodb";
-import { LiveMongoConnection } from "./live_connection";
-import { Random } from "../random/main";
-
-interface CustomFindOptions<T> extends MongoDB.FindOptions<WithId<T>> {
-    pollingThrottleMs?: number;
-    pollingIntervalMs?: number;
-    transform?: (doc: T) => T;
-    maxTimeMs?: number;
-    disableOplog?: boolean;
-}
-
-export class CursorDescription<T> {
-    public selector: MongoDB.Filter<WithId<T>>;
-    public options: CustomFindOptions<WithId<T>>;
-    constructor(public collectionName: string, selector: MongoDB.Filter<WithId<T>>, options?: CustomFindOptions<WithId<T>>) {
-        var self = this;
-        self.collectionName = collectionName;
-        self.selector = _rewriteSelector(selector);
-        self.options = options || {};
-    }
-}
-
-export class LiveCursor<T> {
-    public cursorDescription: CursorDescription<T>;
-
-    constructor (public mongo: LiveMongoConnection, collectionName: string, selector: MongoDB.Filter<WithId<T>>, options: CustomFindOptions<WithId<T>>) {
-        this.cursorDescription = new CursorDescription(collectionName, selector, options);
-    }
-
-    _publishCursor(sub: Subscription) {
-        const observeHandle = this.mongo._observeChanges(
-            this.cursorDescription,
-            false,
-            {
-                added: (id: string, fields: Partial<T>) => {
-                    sub.added(this.cursorDescription.collectionName, id, fields);
-                },
-                changed: (id: string, fields: Partial<T>) => {
-                    sub.changed(this.cursorDescription.collectionName, id, fields);
-                },
-                removed: (id: string) => {
-                    sub.removed(this.cursorDescription.collectionName, id);
-                },
-            },
-            // Publications don't mutate the documents
-            // This is tested by the `livedata - publish callbacks clone` test
-            true
-        );
-    
-        // We don't call sub.ready() here: it gets called in livedata_server, after
-        // possibly calling _publishCursor on multiple returned cursors.
-    
-        // register stop callback (expects lambda w/ no args).
-        sub.onStop(function() {
-            observeHandle.stop();
-        });
-    
-        // return the observeHandle in case it needs to be stopped early
-        return observeHandle;
-    }
-}
-
-
-function _rewriteSelector<T>(selector: MongoDB.Filter<WithId<T>>) {
-    if (Array.isArray(selector)) {
-      // This is consistent with the Mongo console itself; if we don't do this
-      // check passing an empty array ends up selecting all items
-      throw new Error("Mongo selector can't be an array.");
-    }
-
-    if (!selector || ('_id' in selector && !selector._id)) {
-      // can't match anything
-      return { _id: Random.id() } as MongoDB.Filter<WithId<T>>;
-    }
-
-    return selector;
-}