mongodb-livedata-server 0.0.1

package/meteor/mongo/oplog_tailing.ts

@@ -0,0 +1,414 @@
+import { Filter, Long, Timestamp } from "mongodb";
+import { Hook } from "../callback-hook/hook";
+import { _Crossbar } from "../ddp/crossbar";
+import DoubleEndedQueue from "double-ended-queue";
+import { stringify } from "../ejson/ejson";
+import { LiveMongoConnection } from "./live_connection";
+import { CursorDescription } from "./live_cursor";
+
+export const OPLOG_COLLECTION = 'oplog.rs';
+
+var TOO_FAR_BEHIND = process.env.METEOR_OPLOG_TOO_FAR_BEHIND || 2000;
+var TAIL_TIMEOUT = +process.env.METEOR_OPLOG_TAIL_TIMEOUT || 30000;
+
+var showTS = function (ts) {
+    return "Timestamp(" + ts.getHighBits() + ", " + ts.getLowBits() + ")";
+};
+
+type OplogRecord = {
+    ts: Timestamp,
+    op: "d" | "i" | "u" | "c",
+    ns: string,
+    o: { drop: any, dropDatabase: number, applyOps: any }
+};
+
+export function idForOp(op) {
+    if (op.op === 'd')
+        return op.o._id;
+    else if (op.op === 'i')
+        return op.o._id;
+    else if (op.op === 'u')
+        return op.o2._id;
+    else if (op.op === 'c')
+        throw Error("Operator 'c' doesn't supply an object with id: " +
+            stringify(op));
+    else
+        throw Error("Unknown op: " + stringify(op));
+};
+
+export class OplogHandle {
+    private _oplogUrl: string;
+    private _dbName: string;
+
+    private _oplogLastEntryConnection: LiveMongoConnection;
+    private _oplogTailConnection: LiveMongoConnection;
+    private _stopped: boolean;
+    private _tailHandle: { stop: () => void } | null;
+    private _crossbar: _Crossbar;
+    private _baseOplogSelector: Filter<OplogRecord>;
+    private _catchingUpFutures: { ts: Timestamp, future: any }[];
+    private _onSkippedEntriesHook: Hook;
+    private _entryQueue: DoubleEndedQueue<any>;
+    private _workerActive: boolean;
+    private _lastProcessedTS: Timestamp;
+    private _readyFuture: { promise: Promise<void>, resolve: Function };
+
+    constructor (oplogUrl: string, dbName: string) {
+        var self = this;
+        self._readyFuture = { promise: undefined, resolve: undefined };
+        self._readyFuture.promise = new Promise(r => self._readyFuture.resolve = r);
+        self._oplogUrl = oplogUrl;
+        self._dbName = dbName;
+
+        self._oplogLastEntryConnection = null;
+        self._oplogTailConnection = null;
+        self._stopped = false;
+        self._tailHandle = null;
+        self._crossbar = new _Crossbar({
+            factPackage: "mongo-livedata", factName: "oplog-watchers"
+        });
+        self._baseOplogSelector = {
+            ns: new RegExp("^(?:" + _escapeRegExp(self._dbName) + "\\.|admin\\.$cmd)"),
+
+            $or: [
+                { op: { $in: ['i', 'u', 'd'] } },
+                // drop collection
+                { op: 'c', 'o.drop': { $exists: true } },
+                { op: 'c', 'o.dropDatabase': 1 },
+                { op: 'c', 'o.applyOps': { $exists: true } },
+            ]
+        };
+
+        // Data structures to support waitUntilCaughtUp(). Each oplog entry has a
+        // MongoTimestamp object on it (which is not the same as a Date --- it's a
+        // combination of time and an incrementing counter; see
+        // http://docs.mongodb.org/manual/reference/bson-types/#timestamps).
+        //
+        // _catchingUpFutures is an array of {ts: MongoTimestamp, future: Future}
+        // objects, sorted by ascending timestamp. _lastProcessedTS is the
+        // MongoTimestamp of the last oplog entry we've processed.
+        //
+        // Each time we call waitUntilCaughtUp, we take a peek at the final oplog
+        // entry in the db.  If we've already processed it (ie, it is not greater than
+        // _lastProcessedTS), waitUntilCaughtUp immediately returns. Otherwise,
+        // waitUntilCaughtUp makes a new Future and inserts it along with the final
+        // timestamp entry that it read, into _catchingUpFutures. waitUntilCaughtUp
+        // then waits on that future, which is resolved once _lastProcessedTS is
+        // incremented to be past its timestamp by the worker fiber.
+        //
+        // XXX use a priority queue or something else that's faster than an array
+        self._catchingUpFutures = [];
+        self._lastProcessedTS = null;
+
+        self._onSkippedEntriesHook = new Hook({
+            debugPrintExceptions: "onSkippedEntries callback"
+        });
+
+        self._entryQueue = new DoubleEndedQueue();
+        self._workerActive = false;
+
+        self._startTailing();
+    };
+
+    stop() {
+        var self = this;
+        if (self._stopped)
+            return;
+        self._stopped = true;
+        if (self._tailHandle)
+            self._tailHandle.stop();
+        // XXX should close connections too
+    }
+
+    async onOplogEntry(trigger: Record<string, any>, callback: (notification: Record<string, any>) => void) {
+        var self = this;
+        if (self._stopped)
+            throw new Error("Called onOplogEntry on stopped handle!");
+
+        // Calling onOplogEntry requires us to wait for the tailing to be ready.
+        await self._readyFuture.promise;
+
+        var originalCallback = callback;
+        callback = function (notification) {
+            try {
+                originalCallback(notification);
+            } catch (err) {
+                console.error("Error in oplog callback", err);
+            }
+        }
+        var listenHandle = self._crossbar.listen(trigger, callback);
+        return {
+            stop: function () {
+                listenHandle.stop();
+            }
+        };
+    }
+
+    // Register a callback to be invoked any time we skip oplog entries (eg,
+    // because we are too far behind).
+    onSkippedEntries(callback) {
+        var self = this;
+        if (self._stopped)
+            throw new Error("Called onSkippedEntries on stopped handle!");
+        return self._onSkippedEntriesHook.register(callback);
+    }
+    // Calls `callback` once the oplog has been processed up to a point that is
+    // roughly "now": specifically, once we've processed all ops that are
+    // currently visible.
+    // XXX become convinced that this is actually safe even if oplogConnection
+    // is some kind of pool
+    async waitUntilCaughtUp() {
+        var self = this;
+        if (self._stopped)
+            throw new Error("Called waitUntilCaughtUp on stopped handle!");
+
+        // Calling waitUntilCaughtUp requries us to wait for the oplog connection to
+        // be ready.
+        await self._readyFuture.promise;
+        var lastEntry;
+
+        while (!self._stopped) {
+            // We need to make the selector at least as restrictive as the actual
+            // tailing selector (ie, we need to specify the DB name) or else we might
+            // find a TS that won't show up in the actual tail stream.
+            try {
+                lastEntry = await self._oplogLastEntryConnection.db.collection<OplogRecord>(OPLOG_COLLECTION).findOne(
+                    self._baseOplogSelector,
+                    { projection: { ts: 1 }, sort: { $natural: -1 } }
+                );
+                break;
+            } catch (e) {
+                // During failover (eg) if we get an exception we should log and retry
+                // instead of crashing.
+                console.warn("Got exception while reading last entry", e);
+                await _sleepForMs(100);
+            }
+        }
+
+        if (self._stopped)
+            return;
+
+        if (!lastEntry) {
+            // Really, nothing in the oplog? Well, we've processed everything.
+            return;
+        }
+
+        var ts = lastEntry.ts;
+        if (!ts)
+            throw Error("oplog entry without ts: " + stringify(lastEntry));
+
+        if (self._lastProcessedTS && ts.lessThanOrEqual(self._lastProcessedTS)) {
+            // We've already caught up to here.
+            return;
+        }
+
+        // Insert the future into our list. Almost always, this will be at the end,
+        // but it's conceivable that if we fail over from one primary to another,
+        // the oplog entries we see will go backwards.
+        var insertAfter = self._catchingUpFutures.length;
+        while (insertAfter - 1 > 0 && self._catchingUpFutures[insertAfter - 1].ts.greaterThan(ts)) {
+            insertAfter--;
+        }
+        var f = { promise: null, resolve: null };
+        f.promise = new Promise((r) => f.resolve = r);
+        self._catchingUpFutures.splice(insertAfter, 0, { ts: ts, future: f });
+        await f.promise;
+    }
+
+    async _startTailing() {
+        var self = this;
+        // First, make sure that we're talking to the local database.
+        if (self._oplogUrl.indexOf("/local?") === -1 && !self._oplogUrl.endsWith("/local")) {
+            throw Error("$MONGO_OPLOG_URL must be set to the 'local' database of " +
+                "a Mongo replica set");
+        }
+
+        // We make two separate connections to Mongo. The Node Mongo driver
+        // implements a naive round-robin connection pool: each "connection" is a
+        // pool of several (5 by default) TCP connections, and each request is
+        // rotated through the pools. Tailable cursor queries block on the server
+        // until there is some data to return (or until a few seconds have
+        // passed). So if the connection pool used for tailing cursors is the same
+        // pool used for other queries, the other queries will be delayed by seconds
+        // 1/5 of the time.
+        //
+        // The tail connection will only ever be running a single tail command, so
+        // it only needs to make one underlying TCP connection.
+        self._oplogTailConnection = new LiveMongoConnection(self._oplogUrl, { maxPoolSize: 1 });
+        // XXX better docs, but: it's to get monotonic results
+        // XXX is it safe to say "if there's an in flight query, just use its
+        //     results"? I don't think so but should consider that
+        self._oplogLastEntryConnection = new LiveMongoConnection(self._oplogUrl, { maxPoolSize: 1 });
+
+        // Now, make sure that there actually is a repl set here. If not, oplog
+        // tailing won't ever find anything!
+        // More on the isMasterDoc
+        // https://docs.mongodb.com/manual/reference/command/isMaster/
+        const isMasterDoc = await self._oplogLastEntryConnection.db.admin().command({ ismaster: 1 });
+
+        if (!(isMasterDoc && isMasterDoc.setName)) {
+            throw Error("$MONGO_OPLOG_URL must be set to the 'local' database of " +
+                "a Mongo replica set");
+        }
+
+        // Find the last oplog entry.
+        var lastOplogEntry = await self._oplogLastEntryConnection.db.collection<OplogRecord>(OPLOG_COLLECTION).findOne(
+            {},
+            { sort: { $natural: -1 }, projection: { ts: 1 } }
+        );
+
+        var oplogSelector = { ...self._baseOplogSelector };
+        if (lastOplogEntry) {
+            // Start after the last entry that currently exists.
+            oplogSelector.ts = { $gt: lastOplogEntry.ts };
+            // If there are any calls to callWhenProcessedLatest before any other
+            // oplog entries show up, allow callWhenProcessedLatest to call its
+            // callback immediately.
+            self._lastProcessedTS = lastOplogEntry.ts;
+        }
+
+        var cursorDescription = new CursorDescription(OPLOG_COLLECTION, oplogSelector, { tailable: true });
+
+        // Start tailing the oplog.
+        //
+        // We restart the low-level oplog query every 30 seconds if we didn't get a
+        // doc. This is a workaround for #8598: the Node Mongo driver has at least
+        // one bug that can lead to query callbacks never getting called (even with
+        // an error) when leadership failover occur.
+        self._tailHandle = self._oplogTailConnection.tail(
+            cursorDescription,
+            function (doc) {
+                self._entryQueue.push(doc);
+                self._maybeStartWorker();
+            },
+            TAIL_TIMEOUT
+        );
+        self._readyFuture.resolve();
+    }
+
+    _maybeStartWorker() {
+        var self = this;
+        if (self._workerActive) return;
+        self._workerActive = true;
+
+        setImmediate(function () {
+            // May be called recursively in case of transactions.
+            function handleDoc(doc: OplogRecord) {
+                if (doc.ns === "admin.$cmd") {
+                    if (doc.o.applyOps) {
+                        // This was a successful transaction, so we need to apply the
+                        // operations that were involved.
+                        let nextTimestamp: Omit<Long, "toExtendedJSON" | "_bsontype"> = doc.ts;
+                        doc.o.applyOps.forEach(op => {
+                            // See https://github.com/meteor/meteor/issues/10420.
+                            if (!op.ts) {
+                                op.ts = nextTimestamp;
+                                nextTimestamp = nextTimestamp.add(Long.ONE);
+                            }
+                            handleDoc(op);
+                        });
+                        return;
+                    }
+                    throw new Error("Unknown command " + stringify(doc));
+                }
+
+                const trigger = {
+                    dropCollection: false,
+                    dropDatabase: false,
+                    op: doc,
+                    collection: undefined,
+                    id: undefined
+                };
+
+                if (typeof doc.ns === "string" &&
+                    doc.ns.startsWith(self._dbName + ".")) {
+                    trigger.collection = doc.ns.slice(self._dbName.length + 1);
+                }
+
+                // Is it a special command and the collection name is hidden
+                // somewhere in operator?
+                if (trigger.collection === "$cmd") {
+                    if (doc.o.dropDatabase) {
+                        delete trigger.collection;
+                        trigger.dropDatabase = true;
+                    } else if (doc.o.hasOwnProperty("drop")) {
+                        trigger.collection = doc.o.drop;
+                        trigger.dropCollection = true;
+                        trigger.id = null;
+                    } else {
+                        throw Error("Unknown command " + stringify(doc));
+                    }
+
+                } else {
+                    // All other ops have an id.
+                    trigger.id = idForOp(doc);
+                }
+
+                self._crossbar.fire(trigger);
+            }
+
+            try {
+                while (!self._stopped &&
+                    !self._entryQueue.isEmpty()) {
+                    // Are we too far behind? Just tell our observers that they need to
+                    // repoll, and drop our queue.
+                    if (self._entryQueue.length > TOO_FAR_BEHIND) {
+                        var lastEntry = self._entryQueue.pop();
+                        self._entryQueue.clear();
+
+                        self._onSkippedEntriesHook.each(function (callback) {
+                            callback();
+                            return true;
+                        });
+
+                        // Free any waitUntilCaughtUp() calls that were waiting for us to
+                        // pass something that we just skipped.
+                        self._setLastProcessedTS(lastEntry.ts);
+                        continue;
+                    }
+
+                    const doc = self._entryQueue.shift();
+
+                    // Fire trigger(s) for this doc.
+                    handleDoc(doc);
+
+                    // Now that we've processed this operation, process pending
+                    // sequencers.
+                    if (doc.ts) {
+                        self._setLastProcessedTS(doc.ts);
+                    } else {
+                        throw Error("oplog entry without ts: " + stringify(doc));
+                    }
+                }
+            } finally {
+                self._workerActive = false;
+            }
+        });
+    }
+
+    _setLastProcessedTS(ts) {
+        var self = this;
+        self._lastProcessedTS = ts;
+        while (self._catchingUpFutures.length > 0 && self._catchingUpFutures[0].ts.lessThanOrEqual(self._lastProcessedTS)) {
+            var sequencer = self._catchingUpFutures.shift();
+            sequencer.future.return();
+        }
+    }
+
+    //Methods used on tests to dinamically change TOO_FAR_BEHIND
+    _defineTooFarBehind(value) {
+        TOO_FAR_BEHIND = value;
+    }
+    _resetTooFarBehind() {
+        TOO_FAR_BEHIND = process.env.METEOR_OPLOG_TOO_FAR_BEHIND || 2000;
+    }
+
+}
+
+export async function _sleepForMs(ms: number) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export function _escapeRegExp(s: string) {
+    return s.replace(/[\.*+?^$\\|(){}[\]]/g, '\\$&');
+}

package/meteor/mongo/oplog_v2_converter.ts

@@ -0,0 +1,124 @@
+// Converter of the new MongoDB Oplog format (>=5.0) to the one that Meteor
+// handles well, i.e., `$set` and `$unset`. The new format is completely new,
+// and looks as follows:
+//
+//   { $v: 2, diff: Diff }
+//
+// where `Diff` is a recursive structure:
+//
+//   {
+//     // Nested updates (sometimes also represented with an s-field).
+//     // Example: `{ $set: { 'foo.bar': 1 } }`.
+//     i: { <key>: <value>, ... },
+//
+//     // Top-level updates.
+//     // Example: `{ $set: { foo: { bar: 1 } } }`.
+//     u: { <key>: <value>, ... },
+//
+//     // Unsets.
+//     // Example: `{ $unset: { foo: '' } }`.
+//     d: { <key>: false, ... },
+//
+//     // Array operations.
+//     // Example: `{ $push: { foo: 'bar' } }`.
+//     s<key>: { a: true, u<index>: <value>, ... },
+//     ...
+//
+//     // Nested operations (sometimes also represented in the `i` field).
+//     // Example: `{ $set: { 'foo.bar': 1 } }`.
+//     s<key>: Diff,
+//     ...
+//   }
+//
+// (all fields are optional).
+
+function join(prefix, key) {
+    return prefix ? `${prefix}.${key}` : key;
+  }
+  
+  const arrayOperatorKeyRegex = /^(a|u\d+)$/;
+  
+  function isArrayOperatorKey(field) {
+    return arrayOperatorKeyRegex.test(field);
+  }
+  
+  function isArrayOperator(operator) {
+    return operator.a === true && Object.keys(operator).every(isArrayOperatorKey);
+  }
+  
+  function flattenObjectInto(target, source, prefix) {
+    if (Array.isArray(source) || typeof source !== 'object' || source === null) {
+      target[prefix] = source;
+    } else {
+      const entries = Object.entries(source);
+      if (entries.length) {
+        entries.forEach(([key, value]) => {
+          flattenObjectInto(target, value, join(prefix, key));
+        });
+      } else {
+        target[prefix] = source;
+      }
+    }
+  }
+  
+  const logDebugMessages = !!process.env.OPLOG_CONVERTER_DEBUG;
+  
+  function convertOplogDiff(oplogEntry, diff, prefix) {
+    if (logDebugMessages) {
+      console.log(`convertOplogDiff(${JSON.stringify(oplogEntry)}, ${JSON.stringify(diff)}, ${JSON.stringify(prefix)})`);
+    }
+  
+    Object.entries(diff).forEach(([diffKey, value]) => {
+      if (diffKey === 'd') {
+        // Handle `$unset`s.
+        oplogEntry.$unset ??= {};
+        Object.keys(value).forEach(key => {
+          oplogEntry.$unset[join(prefix, key)] = true;
+        });
+      } else if (diffKey === 'i') {
+        // Handle (potentially) nested `$set`s.
+        oplogEntry.$set ??= {};
+        flattenObjectInto(oplogEntry.$set, value, prefix);
+      } else if (diffKey === 'u') {
+        // Handle flat `$set`s.
+        oplogEntry.$set ??= {};
+        Object.entries(value).forEach(([key, value]) => {
+          oplogEntry.$set[join(prefix, key)] = value;
+        });
+      } else {
+        // Handle s-fields.
+        const key = diffKey.slice(1);
+        if (isArrayOperator(value)) {
+          // Array operator.
+          Object.entries(value).forEach(([position, value]) => {
+            if (position === 'a') {
+              return;
+            }
+  
+            const positionKey = join(join(prefix, key), position.slice(1));
+            if (value === null) {
+              oplogEntry.$unset ??= {};
+              oplogEntry.$unset[positionKey] = true;
+            } else {
+              oplogEntry.$set ??= {};
+              oplogEntry.$set[positionKey] = value;
+            }
+          });
+        } else if (key) {
+          // Nested object.
+          convertOplogDiff(oplogEntry, value, join(prefix, key));
+        }
+      }
+    });
+  }
+  
+  export function oplogV2V1Converter(oplogEntry) {
+    // Pass-through v1 and (probably) invalid entries.
+    if (oplogEntry.$v !== 2 || !oplogEntry.diff) {
+      return oplogEntry;
+    }
+  
+    const convertedOplogEntry = { $v: 2 };
+    convertOplogDiff(convertedOplogEntry, oplogEntry.diff, '');
+    return convertedOplogEntry;
+  }