mongodb-livedata-server 0.0.14 → 0.1.1

package/README.md

@@ -1,69 +1,75 @@
-MongoDB Live Data Server
-========================
-
-This project is essentially a MongoDB live data driver (based either on polling or on Oplog tailing) combined with a DDP server, extracted
-out of [Meteor](https://github.com/meteor/meteor), with **Fibers** and **underscore** dependencies removed and code converted to Typescript.
-
-Live data is one of the root concepts of Meteor. Data is served via WebSockets via the DDP protocol and updated automatically whenever something changes in the database. Also, calling server methods via WebSocket is supported.
-
-Using Meteor locks you into the Meteor ecosystem, which has some problems (mostly for historical reasons). Using live data as a separate npm package might be preferable in many scenarios. Also, people who are trying to migrate from Meteor, might find this package useful as an intermediate step.
-
-### Installation
-
-```
-npm i mongodb-livedata-server
-```
-
-### Usage
-
-As a most common example, this is how you can use livedata with Express.js:
-
-```js
-const { DDPServer, LiveCursor, LiveMongoConnection } = require('mongodb-livedata-server')
-const express = require('express')
-const app = express()
-const port = 3000
-
-app.get('/', (req, res) => {
-  res.send('Hello World!')
-})
-
-const httpServer = app.listen(port, () => {
-  console.log(`Example app listening on port ${port}`)
-})
-
-const liveMongoConnection = new LiveMongoConnection(process.env.MONGO_URL, {
-    oplogUrl: process.env.MONGO_OPLOG_URL
-});
-const liveDataServer = new DDPServer({}, httpServer);
-
-liveDataServer.methods({
-    "test-method": async (msg) => {
-        console.log("Test msg: ", msg);
-        return "hello! Current timestamp is: " + Date.now()
-    }
-})
-
-liveDataServer.publish({
-    "test-subscription": async () => {
-        return new LiveCursor(liveMongoConnection, "test-collection", { category: "apples" });
-    }
-})
-
-```
-
-`liveDataServer.methods` and `liveDataServer.publish` have exactly same interface as [Meteor.methods](https://docs.meteor.com/api/methods.html#Meteor-methods) and [Meteor.publish](https://docs.meteor.com/api/pubsub.html#Meteor-publish) respectively, notice however that when publishing subscriptions, you must use `LiveCursor` rather than a normal MongoDB cursor.
-
-### Important notes
-
-- The project is in alpha. Use at your own risk.
-- Neither method context nor subscription context have the `unblock` method anymore (because this package doesn't use Fibers)
-- Meteor syntax for MongoDB queries is not supported. Please always use MongoDB Node.js driver syntax. For example, instead of
-  ```ts
-  const doc = myCollection.findOne(id);
-  ```
-  use
-  ```ts
-  const doc = await myCollection.findOne({ _id: id });
-  ```
-- Neither MongoDB.ObjectId nor it's Meteor.js alternative is supported at the moment. String ids only.
+MongoDB Live Data Server
+========================
+
+This project is essentially a MongoDB live data driver (based either on polling or on Oplog tailing) combined with a DDP server, extracted
+out of [Meteor](https://github.com/meteor/meteor), with **Fibers** and **underscore** dependencies removed and code converted to Typescript.
+
+Live data is one of the root concepts of Meteor. Data is served via WebSockets via the DDP protocol and updated automatically whenever something changes in the database. Also, calling server methods via WebSocket is supported.
+
+Using Meteor locks you into the Meteor ecosystem, which has some problems (mostly for historical reasons). Using live data as a separate npm package might be preferable in many scenarios. Also, people who are trying to migrate from Meteor, might find this package useful as an intermediate step.
+
+### Installation
+
+```
+npm i mongodb-livedata-server
+```
+
+### Usage
+
+As a most common example, this is how you can use livedata with Express.js:
+
+```js
+const { DDPServer, LiveCursor, LiveMongoConnection } = require('mongodb-livedata-server')
+const express = require('express')
+const app = express()
+const port = 3000
+
+app.get('/', (req, res) => {
+  res.send('Hello World!')
+})
+
+const httpServer = app.listen(port, () => {
+  console.log(`Example app listening on port ${port}`)
+})
+
+const liveMongoConnection = new LiveMongoConnection(process.env.MONGO_URL, {
+    oplogUrl: process.env.MONGO_OPLOG_URL
+});
+const liveDataServer = new DDPServer({}, httpServer);
+
+liveDataServer.methods({
+    "test-method": async (msg) => {
+        console.log("Test msg: ", msg);
+        return "hello! Current timestamp is: " + Date.now()
+    }
+})
+
+liveDataServer.publish({
+    "test-subscription": async () => {
+        return new LiveCursor(liveMongoConnection, "test-collection", { category: "apples" });
+    }
+})
+
+```
+
+`liveDataServer.methods` and `liveDataServer.publish` have exactly same interface as [Meteor.methods](https://docs.meteor.com/api/methods.html#Meteor-methods) and [Meteor.publish](https://docs.meteor.com/api/pubsub.html#Meteor-publish) respectively, notice however that when publishing subscriptions, you must use `LiveCursor` rather than a normal MongoDB cursor.
+
+### Important notes
+
+- The project is in alpha. Use at your own risk.
+- Neither method context nor subscription context have the `unblock` method anymore (because this package doesn't use Fibers)
+- Meteor syntax for MongoDB queries is not supported. Please always use MongoDB Node.js driver syntax. For example, instead of
+  ```ts
+  const doc = myCollection.findOne(id);
+  ```
+  use
+  ```ts
+  const doc = await myCollection.findOne({ _id: id });
+  ```
+- Neither MongoDB.ObjectId nor it's Meteor.js alternative is supported at the moment. String ids only.
+
+### DDP Extension
+
+- Starting from 0.1.0, this library extends DDP with `init` message, which is used to avoid initial spam of `added` messages.
+- Starting from 0.1.1, `init` message will only be sent if `version` `1a` of DDP protocol is specified. Additionally, when version `1a`
+is specified, server will not send removes for all documents when stopping a subscription, and rely on the client for the cleanup instead.

package/dist/meteor/binary-heap/max_heap.d.ts

@@ -18,12 +18,12 @@ export declare class MaxHeap {
     _swap(idxA: any, idxB: any): void;
     get(id: any): any;
     set(id: any, value: any): void;
-    remove(id: any): void;
+    delete(id: any): void;
     has(id: any): any;
     empty(): boolean;
     clear(): void;
     forEach(iterator: any): void;
-    size(): number;
+    get size(): number;
     setDefault(id: any, def: any): any;
     clone(): MaxHeap;
     maxElementId(): any;

package/dist/meteor/binary-heap/max_heap.js

@@ -51,14 +51,14 @@ class MaxHeap {
         }
     }
     _downHeap(idx) {
-        while (leftChildIdx(idx) < this.size()) {
+        while (leftChildIdx(idx) < this.size) {
             const left = leftChildIdx(idx);
             const right = rightChildIdx(idx);
             let largest = idx;
-            if (left < this.size()) {
+            if (left < this.size) {
                 largest = this._maxIndex(largest, left);
             }
-            if (right < this.size()) {
+            if (right < this.size) {
                 largest = this._maxIndex(largest, right);
             }
             if (largest === idx) {
@@ -121,7 +121,7 @@ class MaxHeap {
             this._upHeap(this._heap.length - 1);
         }
     }
-    remove(id) {
+    delete(id) {
         if (this.has(id)) {
             const last = this._heap.length - 1;
             const idx = this._heapIdx.get(id);
@@ -143,7 +143,7 @@ class MaxHeap {
         return this._heapIdx.has(id);
     }
     empty() {
-        return !this.size();
+        return this.size === 0;
     }
     clear() {
         this._heap = [];
@@ -153,7 +153,7 @@ class MaxHeap {
     forEach(iterator) {
         this._heap.forEach(obj => iterator(obj.value, obj.id));
     }
-    size() {
+    get size() {
         return this._heap.length;
     }
     setDefault(id, def) {
@@ -168,7 +168,7 @@ class MaxHeap {
         return clone;
     }
     maxElementId() {
-        return this.size() ? this._heap[0].id : null;
+        return this.size > 0 ? this._heap[0].id : null;
     }
     _selfCheck() {
         for (let i = 1; i < this._heap.length; i++) {

package/dist/meteor/binary-heap/min_max_heap.d.ts

@@ -3,7 +3,7 @@ export declare class MinMaxHeap extends MaxHeap {
     private _minHeap;
     constructor(comparator: any, options?: MaxHeapOptions);
     set(id: string, value: any): void;
-    remove(id: string): void;
+    delete(id: string): void;
     clear(): void;
     setDefault(id: string, def: any): any;
     clone(): MinMaxHeap;

package/dist/meteor/binary-heap/min_max_heap.js

@@ -24,9 +24,9 @@ class MinMaxHeap extends max_heap_1.MaxHeap {
         super.set(id, value);
         this._minHeap.set(id, value);
     }
-    remove(id) {
-        super.remove(id);
-        this._minHeap.remove(id);
+    delete(id) {
+        super.delete(id);
+        this._minHeap.delete(id);
     }
     clear() {
         super.clear();

package/dist/meteor/ddp/livedata_server.d.ts

@@ -3,6 +3,7 @@ import { MethodInvocation } from "./method-invocation";
 import { StreamServerSocket } from "./stream_server";
 import { DDPSession, SessionConnectionHandle } from "./session";
 import { Server } from "http";
+import { Hook } from "../callback-hook/hook";
 import { Subscription } from "./subscription";
 export declare const DDP: {
     _CurrentPublicationInvocation: Subscription;
@@ -17,9 +18,9 @@ interface PublicationStrategy {
 export declare class DDPServer {
     private options;
     private onConnectionHook;
-    private onMessageHook;
+    onMessageHook: Hook;
     private publish_handlers;
-    private universal_publish_handlers;
+    universal_publish_handlers: any[];
     private method_handlers;
     private _publicationStrategies;
     private sessions;

package/dist/meteor/ddp/livedata_server.js

@@ -107,7 +107,7 @@ class DDPServer {
      */
     setPublicationStrategy(publicationName, strategy) {
         if (!Object.values(DDPServer.publicationStrategies).includes(strategy)) {
-            throw new Error(`Invalid merge strategy: ${strategy} 
+            throw new Error(`Invalid merge strategy: ${strategy} 
         for collection ${publicationName}`);
         }
         this._publicationStrategies[publicationName] = strategy;
@@ -370,6 +370,7 @@ function wrapInternalException(exception, context) {
         if (exception.sanitizedError.isClientSafe)
             return exception.sanitizedError;
     }
+    console.error("Error " + context + ":", exception);
     return ddpError(500, "Internal server error");
 }
 exports.wrapInternalException = wrapInternalException;

package/dist/meteor/ddp/session-collection-view.js

@@ -3,7 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.SessionCollectionView = void 0;
 const diff_1 = require("../diff-sequence/diff");
 const ejson_1 = require("../ejson/ejson");
-const id_map_1 = require("../id-map/id_map");
 const session_document_view_1 = require("./session-document-view");
 /**
  * Represents a client's view of a single collection
@@ -15,10 +14,10 @@ class SessionCollectionView {
     constructor(collectionName, callbacks) {
         this.collectionName = collectionName;
         this.callbacks = callbacks;
-        this.documents = new id_map_1.IdMap();
+        this.documents = new Map();
     }
     isEmpty() {
-        return this.documents.empty();
+        return this.documents.size === 0;
     }
     diff(previous) {
         diff_1.DiffSequence.diffMaps(previous.documents, this.documents, {
@@ -91,7 +90,7 @@ class SessionCollectionView {
         if (docView.existsIn.size === 0) {
             // it is gone from everyone
             self.callbacks.removed(self.collectionName, id);
-            self.documents.remove(id);
+            self.documents.delete(id);
         }
         else {
             var changed = {};

package/dist/meteor/ddp/session.d.ts

@@ -1,7 +1,9 @@
 import DoubleEndedQueue from "double-ended-queue";
 import { StreamServerSocket } from "./stream_server";
 import { DDPServer } from "./livedata_server";
+import { SessionCollectionView } from "./session-collection-view";
 import { SubscriptionHandle } from "./subscription";
+import { OrderedDict } from "../ordered-dict/ordered_dict";
 export interface SessionConnectionHandle {
     id: string;
     close: Function;
@@ -20,13 +22,13 @@ interface DDPMessage {
 }
 export declare class DDPSession {
     id: string;
-    server: any;
+    server: DDPServer;
     inQueue: DoubleEndedQueue<any>;
     userId: string | null;
     connectionHandle: SessionConnectionHandle;
     _dontStartNewUniversalSubs: boolean;
     _socketUrl: string;
-    private version;
+    version: string;
     private socket;
     private initialized;
     private workerRunning;
@@ -41,6 +43,7 @@ export declare class DDPSession {
     constructor(server: DDPServer, version: string, socket: StreamServerSocket, options: any);
     sendReady(subscriptionIds: string[]): void;
     _canSend(collectionName: any): boolean;
+    sendInitialAdds(collectionName: string, docs: Map<string, any> | OrderedDict): void;
     sendAdded(collectionName: string, id: string, fields: Record<string, any>): void;
     sendChanged(collectionName: string, id: string, fields: Record<string, any>): void;
     sendRemoved(collectionName: string, id: string): void;
@@ -49,7 +52,8 @@ export declare class DDPSession {
         changed: any;
         removed: any;
     };
-    getCollectionView(collectionName: string): any;
+    getCollectionView(collectionName: string): SessionCollectionView;
+    initialAdds(subscriptionHandle: SubscriptionHandle, collectionName: string, docs: Map<string, any> | OrderedDict): void;
     added(subscriptionHandle: SubscriptionHandle, collectionName: string, id: string, fields: Record<string, any>): void;
     removed(subscriptionHandle: SubscriptionHandle, collectionName: string, id: string): void;
     changed(subscriptionHandle: SubscriptionHandle, collectionName: string, id: string, fields: Record<string, any>): void;

package/dist/meteor/ddp/session.js

@@ -14,7 +14,6 @@ const utils_1 = require("./utils");
 const diff_1 = require("../diff-sequence/diff");
 const session_collection_view_1 = require("./session-collection-view");
 const subscription_1 = require("./subscription");
-const id_map_1 = require("../id-map/id_map");
 class DDPSession {
     constructor(server, version, socket, options) {
         this.protocol_handlers = {
@@ -161,7 +160,7 @@ class DDPSession {
         self._namedSubs = new Map();
         self._universalSubs = [];
         self.userId = null;
-        self.collectionViews = new id_map_1.IdMap();
+        self.collectionViews = new Map();
         // Set this to false to not send messages when collectionViews are
         // modified. This is done when rerunning subs in _setUserId and those messages
         // are calculated via a diff instead.
@@ -230,6 +229,13 @@ class DDPSession {
     _canSend(collectionName) {
         return this._isSending || !this.server.getPublicationStrategy(collectionName).useCollectionView;
     }
+    sendInitialAdds(collectionName, docs) {
+        if (this._canSend(collectionName)) {
+            const items = [];
+            docs.forEach(doc => items.push(doc));
+            this.send({ msg: "init", collection: collectionName, items });
+        }
+    }
     sendAdded(collectionName, id, fields) {
         if (this._canSend(collectionName))
             this.send({ msg: "added", collection: collectionName, id, fields });
@@ -266,6 +272,15 @@ class DDPSession {
         }
         return ret;
     }
+    initialAdds(subscriptionHandle, collectionName, docs) {
+        if (this.server.getPublicationStrategy(collectionName).useCollectionView) {
+            const view = this.getCollectionView(collectionName);
+            docs.forEach((doc, id) => view.added(subscriptionHandle, id, doc));
+        }
+        else {
+            this.sendInitialAdds(collectionName, docs);
+        }
+    }
     added(subscriptionHandle, collectionName, id, fields) {
         if (this.server.getPublicationStrategy(collectionName).useCollectionView) {
             const view = this.getCollectionView(collectionName);
@@ -280,7 +295,7 @@ class DDPSession {
             const view = this.getCollectionView(collectionName);
             view.removed(subscriptionHandle, id);
             if (view.isEmpty()) {
-                this.collectionViews.remove(collectionName);
+                this.collectionViews.delete(collectionName);
             }
         }
         else {
@@ -316,7 +331,7 @@ class DDPSession {
             return;
         // Drop the merge box data immediately.
         self.inQueue = null;
-        self.collectionViews = new id_map_1.IdMap();
+        self.collectionViews = new Map();
         if (self.heartbeat) {
             self.heartbeat.stop();
             self.heartbeat = null;
@@ -457,7 +472,7 @@ class DDPSession {
         // update the userId.
         self._isSending = false;
         var beforeCVs = self.collectionViews;
-        self.collectionViews = new id_map_1.IdMap();
+        self.collectionViews = new Map();
         self.userId = userId;
         // _setUserId is normally called from a Meteor method with
         // DDP._CurrentMethodInvocation set. But DDP._CurrentMethodInvocation is not
@@ -507,15 +522,16 @@ class DDPSession {
     }
     // Tear down specified subscription
     _stopSubscription(subId, error) {
-        var self = this;
         var subName = null;
         if (subId) {
-            var maybeSub = self._namedSubs.get(subId);
+            var maybeSub = this._namedSubs.get(subId);
             if (maybeSub) {
                 subName = maybeSub._name;
-                maybeSub._removeAllDocuments();
+                // version 1a doesn't send document deletions and relies on the clients for cleanup
+                if (this.version !== "1a")
+                    maybeSub._removeAllDocuments();
                 maybeSub._deactivate();
-                self._namedSubs.delete(subId);
+                this._namedSubs.delete(subId);
             }
         }
         var response = { msg: 'nosub', id: subId, error: undefined };
@@ -523,7 +539,7 @@ class DDPSession {
             response.error = (0, livedata_server_1.wrapInternalException)(error, subName ? ("from sub " + subName + " id " + subId)
                 : ("from sub id " + subId));
         }
-        self.send(response);
+        this.send(response);
     }
     // Tear down all subscriptions. Note that this does NOT send removed or nosub
     // messages, since we assume the client is gone.

package/dist/meteor/ddp/stream_server.js

@@ -48,7 +48,7 @@ class StreamServer {
         }
         else {
             serverOptions.faye_server_options = {
-                extensions: [permessage_deflate_1.default.configure({})]
+                extensions: [permessage_deflate_1.default.configure({ maxWindowBits: 11, memLevel: 4 })]
             };
         }
         this.server = sockjs_1.default.createServer(serverOptions);

package/dist/meteor/ddp/subscription.d.ts

@@ -1,3 +1,4 @@
+import { OrderedDict } from "../ordered-dict/ordered_dict";
 import { AsyncFunction } from "../types";
 import { DDPSession, SessionConnectionHandle } from "./session";
 export type SubscriptionHandle = `N${string}` | `U${string}`;
@@ -13,7 +14,7 @@ export declare class Subscription {
     private _handler;
     private _subscriptionId;
     private _params;
-    private _name?;
+    _name?: string;
     connection: SessionConnectionHandle;
     private _subscriptionHandle;
     private _deactivated;
@@ -53,6 +54,7 @@ export declare class Subscription {
      */
     onStop(func: () => void): void;
     _isDeactivated(): boolean;
+    initialAdds(collectionName: string, documents: Map<string, any> | OrderedDict): void;
     /**
      * @summary Call inside the publish function.  Informs the subscriber that a document has been added to the record set.
      * @locus Server

package/dist/meteor/ddp/subscription.js

@@ -276,6 +276,22 @@ class Subscription {
         var self = this;
         return self._deactivated || self._session.inQueue === null;
     }
+    initialAdds(collectionName, documents) {
+        if (this._isDeactivated())
+            return;
+        if (this._session.server.getPublicationStrategy(collectionName).doAccountingForCollection) {
+            let ids = this._documents.get(collectionName);
+            if (ids == null) {
+                ids = new Set();
+                this._documents.set(collectionName, ids);
+            }
+            documents.forEach((_doc, id) => ids.add(id));
+        }
+        if (this._session.version === "1a")
+            this._session.initialAdds(this._subscriptionHandle, collectionName, documents);
+        else
+            documents.forEach((doc, id) => this._session.added(this._subscriptionHandle, collectionName, id, doc));
+    }
     /**
      * @summary Call inside the publish function.  Informs the subscriber that a document has been added to the record set.
      * @locus Server

package/dist/meteor/ddp/utils.js

@@ -34,7 +34,7 @@ function last(array, n, guard) {
     return exports.slice.call(array, Math.max(array.length - n, 0));
 }
 exports.last = last;
-exports.SUPPORTED_DDP_VERSIONS = ['1', 'pre2', 'pre1'];
+exports.SUPPORTED_DDP_VERSIONS = ['1a', '1', 'pre2', 'pre1'];
 function parseDDP(stringMessage) {
     try {
         var msg = JSON.parse(stringMessage);

package/dist/meteor/diff-sequence/diff.d.ts

@@ -1,4 +1,3 @@
-import { IdMap } from "../id-map/id_map";
 interface DiffCallbacks {
     both: (key: string, left: any, right: any) => void;
     leftOnly: (key: string, value: any) => void;
@@ -6,11 +5,11 @@ interface DiffCallbacks {
 }
 interface DiffSequence {
     diffQueryChanges(ordered: boolean, oldResults: any[], newResults: any[], observer: any, options?: any): any;
-    diffQueryChanges(ordered: boolean, oldResults: IdMap, newResults: IdMap, observer: any, options?: any): any;
+    diffQueryChanges(ordered: boolean, oldResults: Map<string, any>, newResults: Map<string, any>, observer: any, options?: any): any;
     diffQueryUnorderedChanges(oldResults: any, newResults: any, observer: any, options?: any): any;
     diffQueryOrderedChanges(old_results: any, new_results: any, observer: any, options?: any): any;
     diffObjects(left: Record<string, any>, right: Record<string, any>, callbacks: DiffCallbacks): any;
-    diffMaps(left: IdMap, right: IdMap, callbacks: DiffCallbacks): any;
+    diffMaps(left: Map<string, any>, right: Map<string, any>, callbacks: DiffCallbacks): any;
     makeChangedFields(newDoc: Record<string, any>, oldDoc: Record<string, any>): any;
     applyChanges(doc: Record<string, any>, changeFields: Record<string, any>): void;
 }

package/dist/meteor/mongo/caching_change_observer.d.ts

@@ -1,8 +1,8 @@
-import { IdMap } from "../id-map/id_map";
 import { OrderedDict } from "../ordered-dict/ordered_dict";
 export declare class _CachingChangeObserver {
-    docs: OrderedDict | IdMap;
+    docs: OrderedDict | Map<string, any>;
     applyChange: {
+        initialAdds?: (docs: OrderedDict | Map<string, any>) => void;
         added?: (id: string, fields: any) => void;
         changed?: (id: string, fields: any) => void;
         removed?: (id: string) => void;
@@ -11,7 +11,6 @@ export declare class _CachingChangeObserver {
     };
     private ordered;
     constructor(options?: {
-        callbacks?: any;
         ordered?: boolean;
     });
 }

package/dist/meteor/mongo/caching_change_observer.js

@@ -8,27 +8,11 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports._CachingChangeObserver = void 0;
 const diff_1 = require("../diff-sequence/diff");
-const ejson_1 = require("../ejson/ejson");
-const id_map_1 = require("../id-map/id_map");
 const ordered_dict_1 = require("../ordered-dict/ordered_dict");
 // available as `this` to those callbacks.
 class _CachingChangeObserver {
     constructor(options = {}) {
-        const orderedFromCallbacks = (options.callbacks &&
-            !!(options.callbacks.addedBefore || options.callbacks.movedBefore));
-        if (options.hasOwnProperty('ordered')) {
-            this.ordered = options.ordered;
-            if (options.callbacks && options.ordered !== orderedFromCallbacks) {
-                throw Error('ordered option doesn\'t match callbacks');
-            }
-        }
-        else if (options.callbacks) {
-            this.ordered = orderedFromCallbacks;
-        }
-        else {
-            throw Error('must provide ordered or callbacks');
-        }
-        const callbacks = options.callbacks || {};
+        this.ordered = options.ordered || false;
         if (this.ordered) {
             this.docs = new ordered_dict_1.OrderedDict();
             this.applyChange = {
@@ -36,13 +20,6 @@ class _CachingChangeObserver {
                     // Take a shallow copy since the top-level properties can be changed
                     const doc = Object.assign({}, fields);
                     doc._id = id;
-                    if (callbacks.addedBefore) {
-                        callbacks.addedBefore.call(this, id, (0, ejson_1.clone)(fields), before);
-                    }
-                    // This line triggers if we provide added with movedBefore.
-                    if (callbacks.added) {
-                        callbacks.added.call(this, id, (0, ejson_1.clone)(fields));
-                    }
                     // XXX could `before` be a falsy ID?  Technically
                     // idStringify seems to allow for them -- though
                     // OrderedDict won't call stringify on a falsy arg.
@@ -50,27 +27,24 @@ class _CachingChangeObserver {
                 },
                 movedBefore: (id, before) => {
                     const doc = this.docs.get(id);
-                    if (callbacks.movedBefore) {
-                        callbacks.movedBefore.call(this, id, before);
-                    }
                     this.docs.moveBefore(id, before || null);
                 },
             };
         }
         else {
-            this.docs = new id_map_1.IdMap();
+            this.docs = new Map();
             this.applyChange = {
                 added: (id, fields) => {
                     // Take a shallow copy since the top-level properties can be changed
                     const doc = Object.assign({}, fields);
-                    if (callbacks.added) {
-                        callbacks.added.call(this, id, (0, ejson_1.clone)(fields));
-                    }
                     doc._id = id;
                     this.docs.set(id, doc);
                 },
             };
         }
+        this.applyChange.initialAdds = (docs) => {
+            this.docs = docs;
+        };
         // The methods in _IdMap and OrderedDict used by these callbacks are
         // identical.
         this.applyChange.changed = (id, fields) => {
@@ -78,16 +52,10 @@ class _CachingChangeObserver {
             if (!doc) {
                 throw new Error(`Unknown id for changed: ${id}`);
             }
-            if (callbacks.changed) {
-                callbacks.changed.call(this, id, (0, ejson_1.clone)(fields));
-            }
             diff_1.DiffSequence.applyChanges(doc, fields);
         };
         this.applyChange.removed = id => {
-            if (callbacks.removed) {
-                callbacks.removed.call(this, id);
-            }
-            this.docs.remove(id);
+            this.docs.delete(id);
         };
     }
 }

package/dist/meteor/mongo/live_cursor.js

@@ -19,6 +19,9 @@ class LiveCursor {
     }
     async _publishCursor(sub) {
         const observeHandle = await this.mongo._observeChanges(this.cursorDescription, false, {
+            initialAdds: (docs) => {
+                sub.initialAdds(this.cursorDescription.collectionName, docs);
+            },
             added: (id, fields) => {
                 sub.added(this.cursorDescription.collectionName, id, fields);
             },

package/dist/meteor/mongo/observe_multiplexer.d.ts

@@ -1,9 +1,11 @@
+import { OrderedDict } from "../ordered-dict/ordered_dict";
 export interface ObserveCallbacks {
     added: (id: string, fields: Record<string, any>) => void;
     changed: (id: string, fields: Record<string, any>) => void;
     removed: (id: string) => void;
     addedBefore?: (id: string, fields: Record<string, any>, before?: any) => void;
     movedBefore?: (id: string, fields: Record<string, any>, before?: any) => void;
+    initialAdds: (docs: Map<string, any> | OrderedDict) => void;
     _testOnlyPollCallback?: any;
 }
 export declare class ObserveMultiplexer {
@@ -27,12 +29,12 @@ export declare class ObserveMultiplexer {
     callbackNames(): string[];
     _ready(): boolean;
     _applyCallback(callbackName: string, args: any): Promise<void>;
-    _sendAdds(handle: ObserveHandle): void;
 }
 export declare class ObserveHandle {
     private _multiplexer;
     nonMutatingCallbacks: boolean;
     _id: number;
+    _initialAdds: ObserveCallbacks["initialAdds"];
     _addedBefore: ObserveCallbacks["addedBefore"];
     _movedBefore: ObserveCallbacks["movedBefore"];
     _added: ObserveCallbacks["added"];

package/dist/meteor/mongo/observe_multiplexer.js

@@ -1,15 +1,4 @@
 "use strict";
-var __rest = (this && this.__rest) || function (s, e) {
-    var t = {};
-    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
-        t[p] = s[p];
-    if (s != null && typeof Object.getOwnPropertySymbols === "function")
-        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
-            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
-                t[p[i]] = s[p[i]];
-        }
-    return t;
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ObserveHandle = exports.ObserveMultiplexer = void 0;
 const ejson_1 = require("../ejson/ejson");
@@ -51,9 +40,8 @@ class ObserveMultiplexer {
         ++self._addHandleTasksScheduledButNotPerformed;
         await self._queue.runTask(async () => {
             self._handles[handle._id] = handle;
-            // Send out whatever adds we have so far (whether or not the
-            // multiplexer is ready).
-            self._sendAdds(handle);
+            if (this._ready() && this._cache.docs.size > 0)
+                handle._initialAdds(this._cache.docs);
             --self._addHandleTasksScheduledButNotPerformed;
         });
         // *outside* the task, since otherwise we'd deadlock
@@ -97,6 +85,13 @@ class ObserveMultiplexer {
         this._queue.queueTask(async () => {
             if (this._ready())
                 throw Error("can't make ObserveMultiplex ready twice!");
+            if (this._cache.docs.size > 0) {
+                for (const handleId of Object.keys(this._handles)) {
+                    var handle = this._handles && this._handles[handleId];
+                    if (handle)
+                        handle._initialAdds(this._cache.docs);
+                }
+            }
             this._readyFuture.resolve();
             this._readyFuture.isResolved = true;
         });
@@ -130,9 +125,9 @@ class ObserveMultiplexer {
     callbackNames() {
         var self = this;
         if (self._ordered)
-            return ["addedBefore", "changed", "movedBefore", "removed"];
+            return ["initialAdds", "addedBefore", "changed", "movedBefore", "removed"];
         else
-            return ["added", "changed", "removed"];
+            return ["initialAdds", "added", "changed", "removed"];
     }
     _ready() {
         return this._readyFuture.isResolved;
@@ -151,6 +146,9 @@ class ObserveMultiplexer {
                 (callbackName !== 'added' && callbackName !== 'addedBefore')) {
                 throw new Error("Got " + callbackName + " during initial adds");
             }
+            // don't actually send anything to the handles until initial adds are cached
+            if (!self._ready())
+                return;
             // Now multiplex the callbacks out to all observe handles. It's OK if
             // these calls yield; since we're inside a task, no other use of our queue
             // can continue until these are done. (But we do have to be careful to not
@@ -166,28 +164,6 @@ class ObserveMultiplexer {
             }
         });
     }
-    // Sends initial adds to a handle. It should only be called from within a task
-    // (the task that is processing the addHandleAndSendInitialAdds call). It
-    // synchronously invokes the handle's added or addedBefore; there's no need to
-    // flush the queue afterwards to ensure that the callbacks get out.
-    _sendAdds(handle) {
-        var self = this;
-        //if (self._queue.safeToRunTask())
-        //    throw Error("_sendAdds may only be called from within a task!");
-        var add = self._ordered ? handle._addedBefore : handle._added;
-        if (!add)
-            return;
-        // note: docs may be an _IdMap or an OrderedDict
-        self._cache.docs.forEach((doc, id) => {
-            if (!self._handles.hasOwnProperty(handle._id))
-                throw Error("handle got removed before sending initial adds!");
-            const _a = handle.nonMutatingCallbacks ? doc : (0, ejson_1.clone)(doc), { _id } = _a, fields = __rest(_a, ["_id"]);
-            if (self._ordered)
-                add(id, fields, null); // we're going in order, so add at end
-            else
-                add(id, fields);
-        });
-    }
 }
 exports.ObserveMultiplexer = ObserveMultiplexer;
 let nextObserveHandleId = 1;

package/dist/meteor/mongo/oplog-observe-driver.d.ts

@@ -3,7 +3,6 @@ import { MinimongoMatcher } from "./minimongo_matcher";
 import MinimongoSorter from "./minimongo_sorter";
 import { ObserveMultiplexer } from "./observe_multiplexer";
 import { CursorDescription } from "./live_cursor";
-import { IdMap } from "../id-map/id_map";
 import { SynchronousCursor } from "./synchronous-cursor";
 interface OplogObserveDriverOptions {
     cursorDescription: CursorDescription<any>;
@@ -53,13 +52,15 @@ export declare class OplogObserveDriver {
     _handleOplogEntrySteadyOrFetching(op: any): void;
     _runInitialQuery(): Promise<void>;
     _pollQuery(): void;
-    _runQuery(options?: any): Promise<void>;
+    _runQuery(options?: {
+        initial?: boolean;
+    }): Promise<void>;
     _needToPollQuery(): void;
     _doneQuerying(): Promise<void>;
     _cursorForQuery(optionsOverwrite: {
         limit: number;
     }): SynchronousCursor;
-    _publishNewResults(newResults: IdMap, newBuffer: IdMap): void;
+    _publishNewResults(newResults: Map<string, any>, newBuffer: Map<string, any>): void;
     stop(): void;
     _registerPhaseChange(phase: any): void;
     static cursorSupported(cursorDescription: CursorDescription<any>, matcher: MinimongoMatcher): boolean;

package/dist/meteor/mongo/oplog-observe-driver.js

@@ -11,7 +11,6 @@ const max_heap_1 = require("../binary-heap/max_heap");
 const minimongo_common_1 = require("./minimongo_common");
 const ejson_1 = require("../ejson/ejson");
 const oplog_tailing_1 = require("./oplog_tailing");
-const id_map_1 = require("../id-map/id_map");
 const synchronous_cursor_1 = require("./synchronous-cursor");
 var PHASE;
 (function (PHASE) {
@@ -77,7 +76,7 @@ class OplogObserveDriver {
             self._comparator = null;
             self._sorter = null;
             self._unpublishedBuffer = null;
-            self._published = new id_map_1.IdMap();
+            self._published = new Map();
         }
         // Indicates if it is safe to insert a new document at the end of the buffer
         // for this query. i.e. it is known that there are no documents matching the
@@ -187,11 +186,11 @@ class OplogObserveDriver {
         // (exceeding capacity specified by limit). If so, push the maximum
         // element to the buffer, we might want to save it in memory to reduce the
         // amount of Mongo lookups in the future.
-        if (self._limit && self._published.size() > self._limit) {
+        if (self._limit && self._published.size > self._limit) {
             // XXX in theory the size of published is no more than limit+1
-            if (self._published.size() !== self._limit + 1) {
+            if (self._published.size !== self._limit + 1) {
                 throw new Error("After adding to published, " +
-                    (self._published.size() - self._limit) +
+                    (self._published.size - self._limit) +
                     " documents are overflowing the set");
             }
             var overflowingDocId = self._published.maxElementId();
@@ -199,7 +198,7 @@ class OplogObserveDriver {
             if ((0, ejson_1.equals)(overflowingDocId, id)) {
                 throw new Error("The document just added is overflowing the published set");
             }
-            self._published.remove(overflowingDocId);
+            self._published.delete(overflowingDocId);
             self._multiplexer.removed(overflowingDocId);
             self._addBuffered(overflowingDocId, overflowingDoc);
         }
@@ -208,11 +207,11 @@ class OplogObserveDriver {
     _removePublished(id) {
         var self = this;
         //Meteor._noYieldsAllowed(function () {
-        self._published.remove(id);
+        self._published.delete(id);
         self._multiplexer.removed(id);
-        if (!self._limit || self._published.size() === self._limit)
+        if (!self._limit || self._published.size === self._limit)
             return;
-        if (self._published.size() > self._limit)
+        if (self._published.size > self._limit)
             throw Error("self._published got too big");
         // OK, we are publishing less than the limit. Maybe we should look in the
         // buffer to find the next element past what we were publishing before.
@@ -264,9 +263,9 @@ class OplogObserveDriver {
         //Meteor._noYieldsAllowed(function () {
         self._unpublishedBuffer.set(id, self._sharedProjectionFn(doc));
         // If something is overflowing the buffer, we just remove it from cache
-        if (self._unpublishedBuffer.size() > self._limit) {
+        if (self._unpublishedBuffer.size > self._limit) {
             var maxBufferedId = self._unpublishedBuffer.maxElementId();
-            self._unpublishedBuffer.remove(maxBufferedId);
+            self._unpublishedBuffer.delete(maxBufferedId);
             // Since something matching is removed from cache (both published set and
             // buffer), set flag to false
             self._safeAppendToBuffer = false;
@@ -278,11 +277,11 @@ class OplogObserveDriver {
     _removeBuffered(id) {
         var self = this;
         //Meteor._noYieldsAllowed(function () {
-        self._unpublishedBuffer.remove(id);
+        self._unpublishedBuffer.delete(id);
         // To keep the contract "buffer is never empty in STEADY phase unless the
         // everything matching fits into published" true, we poll everything as
         // soon as we see the buffer becoming empty.
-        if (!self._unpublishedBuffer.size() && !self._safeAppendToBuffer)
+        if (!self._unpublishedBuffer.size && !self._safeAppendToBuffer)
             self._needToPollQuery();
         //});
     }
@@ -299,22 +298,22 @@ class OplogObserveDriver {
             throw Error("tried to add something already existed in buffer " + id);
         var limit = self._limit;
         var comparator = self._comparator;
-        var maxPublished = (limit && self._published.size() > 0)
+        var maxPublished = (limit && self._published.size > 0)
             ? self._published.get(self._published.maxElementId()) // published is MaxHeap because limit is defined
             : null;
-        var maxBuffered = (limit && self._unpublishedBuffer.size() > 0)
+        var maxBuffered = (limit && self._unpublishedBuffer.size > 0)
             ? self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId())
             : null;
         // The query is unlimited or didn't publish enough documents yet or the
         // new document would fit into published set pushing the maximum element
         // out, then we need to publish the doc.
-        var toPublish = !limit || self._published.size() < limit ||
+        var toPublish = !limit || self._published.size < limit ||
             comparator(doc, maxPublished) < 0;
         // Otherwise we might need to buffer it (only in case of limited query).
         // Buffering is allowed if the buffer is not filled up yet and all
         // matching docs are either in the published set or in the buffer.
         var canAppendToBuffer = !toPublish && self._safeAppendToBuffer &&
-            self._unpublishedBuffer.size() < limit;
+            self._unpublishedBuffer.size < limit;
         // Or if it is small enough to be safely inserted to the middle or the
         // beginning of the buffer.
         var canInsertIntoBuffer = !toPublish && maxBuffered &&
@@ -364,7 +363,7 @@ class OplogObserveDriver {
         else if (cachedBefore && matchesNow) {
             var oldDoc = self._published.get(id);
             var comparator = self._comparator;
-            var minBuffered = self._limit && self._unpublishedBuffer.size() &&
+            var minBuffered = self._limit && self._unpublishedBuffer.size &&
                 self._unpublishedBuffer.get(self._unpublishedBuffer.minElementId());
             var maxBuffered;
             if (publishedBefore) {
@@ -378,7 +377,7 @@ class OplogObserveDriver {
                 // published. Notably, we don't want to schedule repoll and continue
                 // relying on this property.
                 var staysInPublished = !self._limit ||
-                    self._unpublishedBuffer.size() === 0 ||
+                    self._unpublishedBuffer.size === 0 ||
                     comparator(newDoc, minBuffered) <= 0;
                 if (staysInPublished) {
                     self._changePublished(id, oldDoc, newDoc);
@@ -405,10 +404,10 @@ class OplogObserveDriver {
                 // we don't trigger the querying immediately.  if we end this block
                 // with the buffer empty, we will need to trigger the query poll
                 // manually too.
-                self._unpublishedBuffer.remove(id);
+                self._unpublishedBuffer.delete(id);
                 // published is MaxHeap because bufferedBefore is only set when limit is defined
                 var maxPublished = self._published.get(self._published.maxElementId());
-                maxBuffered = self._unpublishedBuffer.size() && self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId());
+                maxBuffered = self._unpublishedBuffer.size && self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId());
                 // the buffered doc was updated, it could move to published
                 var toPublish = comparator(newDoc, maxPublished) < 0;
                 // or stays in buffer even after the change
@@ -427,7 +426,7 @@ class OplogObserveDriver {
                     self._safeAppendToBuffer = false;
                     // Normally this check would have been done in _removeBuffered but
                     // we didn't use it, so we need to do it ourself now.
-                    if (!self._unpublishedBuffer.size()) {
+                    if (!self._unpublishedBuffer.size) {
                         self._needToPollQuery();
                     }
                 }
@@ -667,8 +666,8 @@ class OplogObserveDriver {
             // If we've been stopped, we don't have to run anything any more.
             if (self._stopped)
                 return;
-            newResults = new id_map_1.IdMap();
-            newBuffer = new id_map_1.IdMap();
+            newResults = new Map();
+            newBuffer = new Map();
             // Query 2x documents as the half excluded from the original query will go
             // into unpublished buffer to reduce additional Mongo lookups in cases
             // when documents are removed from the published set and need a
@@ -809,7 +808,7 @@ class OplogObserveDriver {
         // Sanity-check that everything we tried to put into _published ended up
         // there.
         // XXX if this is slow, remove it later
-        if (self._published.size() !== newResults.size()) {
+        if (self._published.size !== newResults.size) {
             console.error('The Mongo server and the Meteor query disagree on how ' +
                 'many documents match your query. Cursor description: ', self._cursorDescription);
             throw Error("The Mongo server and the Meteor query disagree on how " +
@@ -825,7 +824,7 @@ class OplogObserveDriver {
         newBuffer.forEach(function (doc, id) {
             self._addBuffered(id, doc);
         });
-        self._safeAppendToBuffer = newBuffer.size() < self._limit;
+        self._safeAppendToBuffer = newBuffer.size < self._limit;
         //});
     }
     // This stop function is invoked from the onStop of the ObserveMultiplexer, so

package/dist/meteor/mongo/polling_observe_driver.js

@@ -1,11 +1,17 @@
 "use strict";
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PollingObserveDriver = void 0;
 const writefence_1 = require("../ddp/writefence");
 const diff_1 = require("../diff-sequence/diff");
 const synchronous_queue_1 = require("./synchronous-queue");
 const observe_driver_utils_1 = require("./observe_driver_utils");
-const id_map_1 = require("../id-map/id_map");
 var POLLING_THROTTLE_MS = +process.env.METEOR_POLLING_THROTTLE_MS || 50;
 var POLLING_INTERVAL_MS = +process.env.METEOR_POLLING_INTERVAL_MS || 10 * 1000;
 class PollingObserveDriver {
@@ -69,6 +75,7 @@ class PollingObserveDriver {
         self._taskQueue.queueTask(async () => await self._pollMongo());
     }
     async _pollMongo() {
+        var _a, e_1, _b, _c;
         var self = this;
         --self._pollsScheduledButNotStarted;
         if (self._stopped)
@@ -79,7 +86,7 @@ class PollingObserveDriver {
         if (!oldResults) {
             first = true;
             // XXX maybe use OrderedDict instead?
-            oldResults = self._ordered ? [] : new id_map_1.IdMap();
+            oldResults = self._ordered ? [] : new Map();
         }
         // Save the list of pending writes which this round will commit.
         var writesForCycle = self._pendingWrites;
@@ -88,8 +95,27 @@ class PollingObserveDriver {
         try {
             const cursor = self._mongoHandle.db.collection(self._cursorDescription.collectionName).find(self._cursorDescription.selector);
             if (!self._ordered) {
-                newResults = new id_map_1.IdMap();
-                await cursor.forEach((doc) => newResults.set(doc._id, doc));
+                newResults = new Map();
+                try {
+                    for (var _d = true, cursor_1 = __asyncValues(cursor), cursor_1_1; cursor_1_1 = await cursor_1.next(), _a = cursor_1_1.done, !_a;) {
+                        _c = cursor_1_1.value;
+                        _d = false;
+                        try {
+                            const doc = _c;
+                            newResults.set(doc._id, doc);
+                        }
+                        finally {
+                            _d = true;
+                        }
+                    }
+                }
+                catch (e_1_1) { e_1 = { error: e_1_1 }; }
+                finally {
+                    try {
+                        if (!_d && !_a && (_b = cursor_1.return)) await _b.call(cursor_1);
+                    }
+                    finally { if (e_1) throw e_1.error; }
+                }
             }
             else
                 newResults = await cursor.toArray();

package/dist/meteor/ordered-dict/ordered_dict.d.ts

@@ -10,12 +10,12 @@ export declare class OrderedDict {
     constructor(...args: any[]);
     _k(key: any): string;
     empty(): boolean;
-    size(): number;
+    get size(): number;
     _linkEltIn(elt: any): void;
     _linkEltOut(elt: any): void;
     putBefore(key: any, item: any, before: any): void;
     append(key: any, item: any): void;
-    remove(key: any): any;
+    delete(key: any): any;
     get(key: any): any;
     has(key: any): any;
     forEach(iter: (doc: any, key: string, index: number) => any, context?: any): void;

package/dist/meteor/ordered-dict/ordered_dict.js

@@ -38,7 +38,7 @@ class OrderedDict {
     empty() {
         return !this._first;
     }
-    size() {
+    get size() {
         return this._size;
     }
     _linkEltIn(elt) {
@@ -82,7 +82,7 @@ class OrderedDict {
     append(key, item) {
         this.putBefore(key, item, null);
     }
-    remove(key) {
+    delete(key) {
         var elt = this._dict[this._k(key)];
         if (typeof elt === "undefined")
             throw new Error("Item " + key + " not present in OrderedDict");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mongodb-livedata-server",
-  "version": "0.0.14",
+  "version": "0.1.1",
   "description": "MongoDB live data server, extracted from Meteor, Fibers removed and converted to TypeScript",
   "main": "dist/livedata_server.js",
   "types": "dist/livedata_server.d.ts",