mongodb-livedata-server 0.0.4 → 0.0.6

package/meteor/mongo/oplog_tailing.ts

@@ -1,414 +0,0 @@
-import { Filter, Long, Timestamp } from "mongodb";
-import { Hook } from "../callback-hook/hook";
-import { _Crossbar } from "../ddp/crossbar";
-import DoubleEndedQueue from "double-ended-queue";
-import { stringify } from "../ejson/ejson";
-import { LiveMongoConnection } from "./live_connection";
-import { CursorDescription } from "./live_cursor";
-
-export const OPLOG_COLLECTION = 'oplog.rs';
-
-var TOO_FAR_BEHIND = process.env.METEOR_OPLOG_TOO_FAR_BEHIND || 2000;
-var TAIL_TIMEOUT = +process.env.METEOR_OPLOG_TAIL_TIMEOUT || 30000;
-
-var showTS = function (ts) {
-    return "Timestamp(" + ts.getHighBits() + ", " + ts.getLowBits() + ")";
-};
-
-type OplogRecord = {
-    ts: Timestamp,
-    op: "d" | "i" | "u" | "c",
-    ns: string,
-    o: { drop: any, dropDatabase: number, applyOps: any }
-};
-
-export function idForOp(op) {
-    if (op.op === 'd')
-        return op.o._id;
-    else if (op.op === 'i')
-        return op.o._id;
-    else if (op.op === 'u')
-        return op.o2._id;
-    else if (op.op === 'c')
-        throw Error("Operator 'c' doesn't supply an object with id: " +
-            stringify(op));
-    else
-        throw Error("Unknown op: " + stringify(op));
-};
-
-export class OplogHandle {
-    private _oplogUrl: string;
-    private _dbName: string;
-
-    private _oplogLastEntryConnection: LiveMongoConnection;
-    private _oplogTailConnection: LiveMongoConnection;
-    private _stopped: boolean;
-    private _tailHandle: { stop: () => void } | null;
-    private _crossbar: _Crossbar;
-    private _baseOplogSelector: Filter<OplogRecord>;
-    private _catchingUpFutures: { ts: Timestamp, future: any }[];
-    private _onSkippedEntriesHook: Hook;
-    private _entryQueue: DoubleEndedQueue<any>;
-    private _workerActive: boolean;
-    private _lastProcessedTS: Timestamp;
-    private _readyFuture: { promise: Promise<void>, resolve: Function };
-
-    constructor (oplogUrl: string, dbName: string) {
-        var self = this;
-        self._readyFuture = { promise: undefined, resolve: undefined };
-        self._readyFuture.promise = new Promise(r => self._readyFuture.resolve = r);
-        self._oplogUrl = oplogUrl;
-        self._dbName = dbName;
-
-        self._oplogLastEntryConnection = null;
-        self._oplogTailConnection = null;
-        self._stopped = false;
-        self._tailHandle = null;
-        self._crossbar = new _Crossbar({
-            factPackage: "mongo-livedata", factName: "oplog-watchers"
-        });
-        self._baseOplogSelector = {
-            ns: new RegExp("^(?:" + _escapeRegExp(self._dbName) + "\\.|admin\\.$cmd)"),
-
-            $or: [
-                { op: { $in: ['i', 'u', 'd'] } },
-                // drop collection
-                { op: 'c', 'o.drop': { $exists: true } },
-                { op: 'c', 'o.dropDatabase': 1 },
-                { op: 'c', 'o.applyOps': { $exists: true } },
-            ]
-        };
-
-        // Data structures to support waitUntilCaughtUp(). Each oplog entry has a
-        // MongoTimestamp object on it (which is not the same as a Date --- it's a
-        // combination of time and an incrementing counter; see
-        // http://docs.mongodb.org/manual/reference/bson-types/#timestamps).
-        //
-        // _catchingUpFutures is an array of {ts: MongoTimestamp, future: Future}
-        // objects, sorted by ascending timestamp. _lastProcessedTS is the
-        // MongoTimestamp of the last oplog entry we've processed.
-        //
-        // Each time we call waitUntilCaughtUp, we take a peek at the final oplog
-        // entry in the db.  If we've already processed it (ie, it is not greater than
-        // _lastProcessedTS), waitUntilCaughtUp immediately returns. Otherwise,
-        // waitUntilCaughtUp makes a new Future and inserts it along with the final
-        // timestamp entry that it read, into _catchingUpFutures. waitUntilCaughtUp
-        // then waits on that future, which is resolved once _lastProcessedTS is
-        // incremented to be past its timestamp by the worker fiber.
-        //
-        // XXX use a priority queue or something else that's faster than an array
-        self._catchingUpFutures = [];
-        self._lastProcessedTS = null;
-
-        self._onSkippedEntriesHook = new Hook({
-            debugPrintExceptions: "onSkippedEntries callback"
-        });
-
-        self._entryQueue = new DoubleEndedQueue();
-        self._workerActive = false;
-
-        self._startTailing();
-    };
-
-    stop() {
-        var self = this;
-        if (self._stopped)
-            return;
-        self._stopped = true;
-        if (self._tailHandle)
-            self._tailHandle.stop();
-        // XXX should close connections too
-    }
-
-    async onOplogEntry(trigger: Record<string, any>, callback: (notification: Record<string, any>) => void) {
-        var self = this;
-        if (self._stopped)
-            throw new Error("Called onOplogEntry on stopped handle!");
-
-        // Calling onOplogEntry requires us to wait for the tailing to be ready.
-        await self._readyFuture.promise;
-
-        var originalCallback = callback;
-        callback = function (notification) {
-            try {
-                originalCallback(notification);
-            } catch (err) {
-                console.error("Error in oplog callback", err);
-            }
-        }
-        var listenHandle = self._crossbar.listen(trigger, callback);
-        return {
-            stop: function () {
-                listenHandle.stop();
-            }
-        };
-    }
-
-    // Register a callback to be invoked any time we skip oplog entries (eg,
-    // because we are too far behind).
-    onSkippedEntries(callback) {
-        var self = this;
-        if (self._stopped)
-            throw new Error("Called onSkippedEntries on stopped handle!");
-        return self._onSkippedEntriesHook.register(callback);
-    }
-    // Calls `callback` once the oplog has been processed up to a point that is
-    // roughly "now": specifically, once we've processed all ops that are
-    // currently visible.
-    // XXX become convinced that this is actually safe even if oplogConnection
-    // is some kind of pool
-    async waitUntilCaughtUp() {
-        var self = this;
-        if (self._stopped)
-            throw new Error("Called waitUntilCaughtUp on stopped handle!");
-
-        // Calling waitUntilCaughtUp requries us to wait for the oplog connection to
-        // be ready.
-        await self._readyFuture.promise;
-        var lastEntry;
-
-        while (!self._stopped) {
-            // We need to make the selector at least as restrictive as the actual
-            // tailing selector (ie, we need to specify the DB name) or else we might
-            // find a TS that won't show up in the actual tail stream.
-            try {
-                lastEntry = await self._oplogLastEntryConnection.db.collection<OplogRecord>(OPLOG_COLLECTION).findOne(
-                    self._baseOplogSelector,
-                    { projection: { ts: 1 }, sort: { $natural: -1 } }
-                );
-                break;
-            } catch (e) {
-                // During failover (eg) if we get an exception we should log and retry
-                // instead of crashing.
-                console.warn("Got exception while reading last entry", e);
-                await _sleepForMs(100);
-            }
-        }
-
-        if (self._stopped)
-            return;
-
-        if (!lastEntry) {
-            // Really, nothing in the oplog? Well, we've processed everything.
-            return;
-        }
-
-        var ts = lastEntry.ts;
-        if (!ts)
-            throw Error("oplog entry without ts: " + stringify(lastEntry));
-
-        if (self._lastProcessedTS && ts.lessThanOrEqual(self._lastProcessedTS)) {
-            // We've already caught up to here.
-            return;
-        }
-
-        // Insert the future into our list. Almost always, this will be at the end,
-        // but it's conceivable that if we fail over from one primary to another,
-        // the oplog entries we see will go backwards.
-        var insertAfter = self._catchingUpFutures.length;
-        while (insertAfter - 1 > 0 && self._catchingUpFutures[insertAfter - 1].ts.greaterThan(ts)) {
-            insertAfter--;
-        }
-        var f = { promise: null, resolve: null };
-        f.promise = new Promise((r) => f.resolve = r);
-        self._catchingUpFutures.splice(insertAfter, 0, { ts: ts, future: f });
-        await f.promise;
-    }
-
-    async _startTailing() {
-        var self = this;
-        // First, make sure that we're talking to the local database.
-        if (self._oplogUrl.indexOf("/local?") === -1 && !self._oplogUrl.endsWith("/local")) {
-            throw Error("$MONGO_OPLOG_URL must be set to the 'local' database of " +
-                "a Mongo replica set");
-        }
-
-        // We make two separate connections to Mongo. The Node Mongo driver
-        // implements a naive round-robin connection pool: each "connection" is a
-        // pool of several (5 by default) TCP connections, and each request is
-        // rotated through the pools. Tailable cursor queries block on the server
-        // until there is some data to return (or until a few seconds have
-        // passed). So if the connection pool used for tailing cursors is the same
-        // pool used for other queries, the other queries will be delayed by seconds
-        // 1/5 of the time.
-        //
-        // The tail connection will only ever be running a single tail command, so
-        // it only needs to make one underlying TCP connection.
-        self._oplogTailConnection = new LiveMongoConnection(self._oplogUrl, { maxPoolSize: 1 });
-        // XXX better docs, but: it's to get monotonic results
-        // XXX is it safe to say "if there's an in flight query, just use its
-        //     results"? I don't think so but should consider that
-        self._oplogLastEntryConnection = new LiveMongoConnection(self._oplogUrl, { maxPoolSize: 1 });
-
-        // Now, make sure that there actually is a repl set here. If not, oplog
-        // tailing won't ever find anything!
-        // More on the isMasterDoc
-        // https://docs.mongodb.com/manual/reference/command/isMaster/
-        const isMasterDoc = await self._oplogLastEntryConnection.db.admin().command({ ismaster: 1 });
-
-        if (!(isMasterDoc && isMasterDoc.setName)) {
-            throw Error("$MONGO_OPLOG_URL must be set to the 'local' database of " +
-                "a Mongo replica set");
-        }
-
-        // Find the last oplog entry.
-        var lastOplogEntry = await self._oplogLastEntryConnection.db.collection<OplogRecord>(OPLOG_COLLECTION).findOne(
-            {},
-            { sort: { $natural: -1 }, projection: { ts: 1 } }
-        );
-
-        var oplogSelector = { ...self._baseOplogSelector };
-        if (lastOplogEntry) {
-            // Start after the last entry that currently exists.
-            oplogSelector.ts = { $gt: lastOplogEntry.ts };
-            // If there are any calls to callWhenProcessedLatest before any other
-            // oplog entries show up, allow callWhenProcessedLatest to call its
-            // callback immediately.
-            self._lastProcessedTS = lastOplogEntry.ts;
-        }
-
-        var cursorDescription = new CursorDescription(OPLOG_COLLECTION, oplogSelector, { tailable: true });
-
-        // Start tailing the oplog.
-        //
-        // We restart the low-level oplog query every 30 seconds if we didn't get a
-        // doc. This is a workaround for #8598: the Node Mongo driver has at least
-        // one bug that can lead to query callbacks never getting called (even with
-        // an error) when leadership failover occur.
-        self._tailHandle = self._oplogTailConnection.tail(
-            cursorDescription,
-            function (doc) {
-                self._entryQueue.push(doc);
-                self._maybeStartWorker();
-            },
-            TAIL_TIMEOUT
-        );
-        self._readyFuture.resolve();
-    }
-
-    _maybeStartWorker() {
-        var self = this;
-        if (self._workerActive) return;
-        self._workerActive = true;
-
-        setImmediate(function () {
-            // May be called recursively in case of transactions.
-            function handleDoc(doc: OplogRecord) {
-                if (doc.ns === "admin.$cmd") {
-                    if (doc.o.applyOps) {
-                        // This was a successful transaction, so we need to apply the
-                        // operations that were involved.
-                        let nextTimestamp: Omit<Long, "toExtendedJSON" | "_bsontype"> = doc.ts;
-                        doc.o.applyOps.forEach(op => {
-                            // See https://github.com/meteor/meteor/issues/10420.
-                            if (!op.ts) {
-                                op.ts = nextTimestamp;
-                                nextTimestamp = nextTimestamp.add(Long.ONE);
-                            }
-                            handleDoc(op);
-                        });
-                        return;
-                    }
-                    throw new Error("Unknown command " + stringify(doc));
-                }
-
-                const trigger = {
-                    dropCollection: false,
-                    dropDatabase: false,
-                    op: doc,
-                    collection: undefined,
-                    id: undefined
-                };
-
-                if (typeof doc.ns === "string" &&
-                    doc.ns.startsWith(self._dbName + ".")) {
-                    trigger.collection = doc.ns.slice(self._dbName.length + 1);
-                }
-
-                // Is it a special command and the collection name is hidden
-                // somewhere in operator?
-                if (trigger.collection === "$cmd") {
-                    if (doc.o.dropDatabase) {
-                        delete trigger.collection;
-                        trigger.dropDatabase = true;
-                    } else if (doc.o.hasOwnProperty("drop")) {
-                        trigger.collection = doc.o.drop;
-                        trigger.dropCollection = true;
-                        trigger.id = null;
-                    } else {
-                        throw Error("Unknown command " + stringify(doc));
-                    }
-
-                } else {
-                    // All other ops have an id.
-                    trigger.id = idForOp(doc);
-                }
-
-                self._crossbar.fire(trigger);
-            }
-
-            try {
-                while (!self._stopped &&
-                    !self._entryQueue.isEmpty()) {
-                    // Are we too far behind? Just tell our observers that they need to
-                    // repoll, and drop our queue.
-                    if (self._entryQueue.length > TOO_FAR_BEHIND) {
-                        var lastEntry = self._entryQueue.pop();
-                        self._entryQueue.clear();
-
-                        self._onSkippedEntriesHook.each(function (callback) {
-                            callback();
-                            return true;
-                        });
-
-                        // Free any waitUntilCaughtUp() calls that were waiting for us to
-                        // pass something that we just skipped.
-                        self._setLastProcessedTS(lastEntry.ts);
-                        continue;
-                    }
-
-                    const doc = self._entryQueue.shift();
-
-                    // Fire trigger(s) for this doc.
-                    handleDoc(doc);
-
-                    // Now that we've processed this operation, process pending
-                    // sequencers.
-                    if (doc.ts) {
-                        self._setLastProcessedTS(doc.ts);
-                    } else {
-                        throw Error("oplog entry without ts: " + stringify(doc));
-                    }
-                }
-            } finally {
-                self._workerActive = false;
-            }
-        });
-    }
-
-    _setLastProcessedTS(ts) {
-        var self = this;
-        self._lastProcessedTS = ts;
-        while (self._catchingUpFutures.length > 0 && self._catchingUpFutures[0].ts.lessThanOrEqual(self._lastProcessedTS)) {
-            var sequencer = self._catchingUpFutures.shift();
-            sequencer.future.return();
-        }
-    }
-
-    //Methods used on tests to dinamically change TOO_FAR_BEHIND
-    _defineTooFarBehind(value) {
-        TOO_FAR_BEHIND = value;
-    }
-    _resetTooFarBehind() {
-        TOO_FAR_BEHIND = process.env.METEOR_OPLOG_TOO_FAR_BEHIND || 2000;
-    }
-
-}
-
-export async function _sleepForMs(ms: number) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-export function _escapeRegExp(s: string) {
-    return s.replace(/[\.*+?^$\\|(){}[\]]/g, '\\$&');
-}

package/meteor/mongo/oplog_v2_converter.ts

@@ -1,124 +0,0 @@
-// Converter of the new MongoDB Oplog format (>=5.0) to the one that Meteor
-// handles well, i.e., `$set` and `$unset`. The new format is completely new,
-// and looks as follows:
-//
-//   { $v: 2, diff: Diff }
-//
-// where `Diff` is a recursive structure:
-//
-//   {
-//     // Nested updates (sometimes also represented with an s-field).
-//     // Example: `{ $set: { 'foo.bar': 1 } }`.
-//     i: { <key>: <value>, ... },
-//
-//     // Top-level updates.
-//     // Example: `{ $set: { foo: { bar: 1 } } }`.
-//     u: { <key>: <value>, ... },
-//
-//     // Unsets.
-//     // Example: `{ $unset: { foo: '' } }`.
-//     d: { <key>: false, ... },
-//
-//     // Array operations.
-//     // Example: `{ $push: { foo: 'bar' } }`.
-//     s<key>: { a: true, u<index>: <value>, ... },
-//     ...
-//
-//     // Nested operations (sometimes also represented in the `i` field).
-//     // Example: `{ $set: { 'foo.bar': 1 } }`.
-//     s<key>: Diff,
-//     ...
-//   }
-//
-// (all fields are optional).
-
-function join(prefix, key) {
-    return prefix ? `${prefix}.${key}` : key;
-  }
-  
-  const arrayOperatorKeyRegex = /^(a|u\d+)$/;
-  
-  function isArrayOperatorKey(field) {
-    return arrayOperatorKeyRegex.test(field);
-  }
-  
-  function isArrayOperator(operator) {
-    return operator.a === true && Object.keys(operator).every(isArrayOperatorKey);
-  }
-  
-  function flattenObjectInto(target, source, prefix) {
-    if (Array.isArray(source) || typeof source !== 'object' || source === null) {
-      target[prefix] = source;
-    } else {
-      const entries = Object.entries(source);
-      if (entries.length) {
-        entries.forEach(([key, value]) => {
-          flattenObjectInto(target, value, join(prefix, key));
-        });
-      } else {
-        target[prefix] = source;
-      }
-    }
-  }
-  
-  const logDebugMessages = !!process.env.OPLOG_CONVERTER_DEBUG;
-  
-  function convertOplogDiff(oplogEntry, diff, prefix) {
-    if (logDebugMessages) {
-      console.log(`convertOplogDiff(${JSON.stringify(oplogEntry)}, ${JSON.stringify(diff)}, ${JSON.stringify(prefix)})`);
-    }
-  
-    Object.entries(diff).forEach(([diffKey, value]) => {
-      if (diffKey === 'd') {
-        // Handle `$unset`s.
-        oplogEntry.$unset ??= {};
-        Object.keys(value).forEach(key => {
-          oplogEntry.$unset[join(prefix, key)] = true;
-        });
-      } else if (diffKey === 'i') {
-        // Handle (potentially) nested `$set`s.
-        oplogEntry.$set ??= {};
-        flattenObjectInto(oplogEntry.$set, value, prefix);
-      } else if (diffKey === 'u') {
-        // Handle flat `$set`s.
-        oplogEntry.$set ??= {};
-        Object.entries(value).forEach(([key, value]) => {
-          oplogEntry.$set[join(prefix, key)] = value;
-        });
-      } else {
-        // Handle s-fields.
-        const key = diffKey.slice(1);
-        if (isArrayOperator(value)) {
-          // Array operator.
-          Object.entries(value).forEach(([position, value]) => {
-            if (position === 'a') {
-              return;
-            }
-  
-            const positionKey = join(join(prefix, key), position.slice(1));
-            if (value === null) {
-              oplogEntry.$unset ??= {};
-              oplogEntry.$unset[positionKey] = true;
-            } else {
-              oplogEntry.$set ??= {};
-              oplogEntry.$set[positionKey] = value;
-            }
-          });
-        } else if (key) {
-          // Nested object.
-          convertOplogDiff(oplogEntry, value, join(prefix, key));
-        }
-      }
-    });
-  }
-  
-  export function oplogV2V1Converter(oplogEntry) {
-    // Pass-through v1 and (probably) invalid entries.
-    if (oplogEntry.$v !== 2 || !oplogEntry.diff) {
-      return oplogEntry;
-    }
-  
-    const convertedOplogEntry = { $v: 2 };
-    convertOplogDiff(convertedOplogEntry, oplogEntry.diff, '');
-    return convertedOplogEntry;
-  }