@connecttomahdi/rxdb 17.0.0-beta.17

package/dist/cjs/plugins/replication-firestore/index.js

@@ -0,0 +1,266 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+var _exportNames = {
+  RxFirestoreReplicationState: true,
+  replicateFirestore: true
+};
+exports.RxFirestoreReplicationState = void 0;
+exports.replicateFirestore = replicateFirestore;
+var _inheritsLoose2 = _interopRequireDefault(require("@babel/runtime/helpers/inheritsLoose"));
+var _index = require("../../plugins/utils/index.js");
+var _firestore = require("firebase/firestore");
+var _index2 = require("../leader-election/index.js");
+var _index3 = require("../replication/index.js");
+var _index4 = require("../../index.js");
+var _rxjs = require("rxjs");
+var _firestoreHelper = require("./firestore-helper.js");
+Object.keys(_firestoreHelper).forEach(function (key) {
+  if (key === "default" || key === "__esModule") return;
+  if (Object.prototype.hasOwnProperty.call(_exportNames, key)) return;
+  if (key in exports && exports[key] === _firestoreHelper[key]) return;
+  Object.defineProperty(exports, key, {
+    enumerable: true,
+    get: function () {
+      return _firestoreHelper[key];
+    }
+  });
+});
+var _firestoreTypes = require("./firestore-types.js");
+Object.keys(_firestoreTypes).forEach(function (key) {
+  if (key === "default" || key === "__esModule") return;
+  if (Object.prototype.hasOwnProperty.call(_exportNames, key)) return;
+  if (key in exports && exports[key] === _firestoreTypes[key]) return;
+  Object.defineProperty(exports, key, {
+    enumerable: true,
+    get: function () {
+      return _firestoreTypes[key];
+    }
+  });
+});
+var RxFirestoreReplicationState = exports.RxFirestoreReplicationState = /*#__PURE__*/function (_RxReplicationState) {
+  function RxFirestoreReplicationState(firestore, replicationIdentifierHash, collection, pull, push, live = true, retryTime = 1000 * 5, autoStart = true) {
+    var _this;
+    _this = _RxReplicationState.call(this, replicationIdentifierHash, collection, '_deleted', pull, push, live, retryTime, autoStart) || this;
+    _this.firestore = firestore;
+    _this.replicationIdentifierHash = replicationIdentifierHash;
+    _this.collection = collection;
+    _this.pull = pull;
+    _this.push = push;
+    _this.live = live;
+    _this.retryTime = retryTime;
+    _this.autoStart = autoStart;
+    return _this;
+  }
+  (0, _inheritsLoose2.default)(RxFirestoreReplicationState, _RxReplicationState);
+  return RxFirestoreReplicationState;
+}(_index3.RxReplicationState);
+function replicateFirestore(options) {
+  var collection = options.collection;
+  (0, _index4.addRxPlugin)(_index2.RxDBLeaderElectionPlugin);
+  var pullStream$ = new _rxjs.Subject();
+  var replicationPrimitivesPull;
+  options.live = typeof options.live === 'undefined' ? true : options.live;
+  options.waitForLeadership = typeof options.waitForLeadership === 'undefined' ? true : options.waitForLeadership;
+  var serverTimestampField = typeof options.serverTimestampField === 'undefined' ? 'serverTimestamp' : options.serverTimestampField;
+  options.serverTimestampField = serverTimestampField;
+  var primaryPath = collection.schema.primaryPath;
+
+  /**
+   * The serverTimestampField MUST NOT be part of the collections RxJsonSchema.
+   */
+  var schemaPart = (0, _index4.getSchemaByObjectPath)(collection.schema.jsonSchema, serverTimestampField);
+  if (schemaPart ||
+  // also must not be nested.
+  serverTimestampField.includes('.')) {
+    throw (0, _index4.newRxError)('RC6', {
+      field: serverTimestampField,
+      schema: collection.schema.jsonSchema
+    });
+  }
+  var pullFilters = options.pull?.filter !== undefined ? (0, _index.toArray)(options.pull.filter) : [];
+  var pullQuery = (0, _firestore.query)(options.firestore.collection, ...pullFilters);
+  if (options.pull) {
+    replicationPrimitivesPull = {
+      async handler(lastPulledCheckpoint, batchSize) {
+        var newerQuery;
+        var sameTimeQuery;
+        if (lastPulledCheckpoint) {
+          var lastServerTimestamp = (0, _firestoreHelper.isoStringToServerTimestamp)(lastPulledCheckpoint.serverTimestamp);
+          newerQuery = (0, _firestore.query)(pullQuery, (0, _firestore.where)(serverTimestampField, '>', lastServerTimestamp), (0, _firestore.orderBy)(serverTimestampField, 'asc'), (0, _firestore.limit)(batchSize));
+          sameTimeQuery = (0, _firestore.query)(pullQuery, (0, _firestore.where)(serverTimestampField, '==', lastServerTimestamp), (0, _firestore.where)((0, _firestore.documentId)(), '>', lastPulledCheckpoint.id), (0, _firestore.orderBy)((0, _firestore.documentId)(), 'asc'), (0, _firestore.limit)(batchSize));
+        } else {
+          newerQuery = (0, _firestore.query)(pullQuery, (0, _firestore.orderBy)(serverTimestampField, 'asc'), (0, _firestore.limit)(batchSize));
+        }
+        var mustsReRun = true;
+        var useDocs = [];
+        while (mustsReRun) {
+          /**
+           * Local writes that have not been persisted to the server
+           * are in pending state and do not have a correct serverTimestamp set.
+           * We have to ensure we only use document states that are in sync with the server.
+           * @link https://medium.com/firebase-developers/the-secrets-of-firestore-fieldvalue-servertimestamp-revealed-29dd7a38a82b
+           */
+          await (0, _firestore.waitForPendingWrites)(options.firestore.database);
+          await (0, _firestore.runTransaction)(options.firestore.database, async _tx => {
+            useDocs = [];
+            var [newerQueryResult, sameTimeQueryResult] = await Promise.all([(0, _firestore.getDocs)(newerQuery), sameTimeQuery ? (0, _firestore.getDocs)(sameTimeQuery) : undefined]);
+            if (newerQueryResult.metadata.hasPendingWrites || sameTimeQuery && (0, _index.ensureNotFalsy)(sameTimeQueryResult).metadata.hasPendingWrites) {
+              return;
+            } else {
+              mustsReRun = false;
+              if (sameTimeQuery) {
+                useDocs = (0, _index.ensureNotFalsy)(sameTimeQueryResult).docs;
+              }
+              var missingAmount = batchSize - useDocs.length;
+              if (missingAmount > 0) {
+                var additionalDocs = newerQueryResult.docs.slice(0, missingAmount).filter(x => !!x);
+                useDocs = useDocs.concat(additionalDocs);
+              }
+            }
+          });
+        }
+        if (useDocs.length === 0) {
+          return {
+            checkpoint: lastPulledCheckpoint ?? undefined,
+            documents: []
+          };
+        }
+        var lastDoc = (0, _index.ensureNotFalsy)((0, _index.lastOfArray)(useDocs));
+        var documents = useDocs.map(row => (0, _firestoreHelper.firestoreRowToDocData)(serverTimestampField, primaryPath, row));
+        var newCheckpoint = {
+          id: lastDoc.id,
+          serverTimestamp: (0, _firestoreHelper.serverTimestampToIsoString)(serverTimestampField, lastDoc.data())
+        };
+        var ret = {
+          documents: documents,
+          checkpoint: newCheckpoint
+        };
+        return ret;
+      },
+      batchSize: (0, _index.ensureNotFalsy)(options.pull).batchSize,
+      modifier: (0, _index.ensureNotFalsy)(options.pull).modifier,
+      stream$: pullStream$.asObservable(),
+      initialCheckpoint: options.pull.initialCheckpoint
+    };
+  }
+  var replicationPrimitivesPush;
+  if (options.push) {
+    var pushFilter = options.push?.filter;
+    replicationPrimitivesPush = {
+      async handler(rows) {
+        if (pushFilter !== undefined) {
+          rows = await (0, _index.asyncFilter)(rows, row => pushFilter(row.newDocumentState));
+        }
+        var writeRowsById = {};
+        var docIds = rows.map(row => {
+          var docId = row.newDocumentState[primaryPath];
+          writeRowsById[docId] = row;
+          return docId;
+        });
+        await (0, _firestore.waitForPendingWrites)(options.firestore.database);
+        var conflicts = [];
+
+        /**
+         * Everything must run INSIDE of the transaction
+         * because on tx-errors, firebase will re-run the transaction on some cases.
+         * @link https://firebase.google.com/docs/firestore/manage-data/transactions#transaction_failure
+         * @link https://firebase.google.com/docs/firestore/manage-data/transactions
+         */
+        await (0, _firestore.runTransaction)(options.firestore.database, async _tx => {
+          conflicts = []; // reset in case the tx has re-run.
+          /**
+           * @link https://stackoverflow.com/a/48423626/3443137
+           */
+
+          var getQuery = ids => {
+            return (0, _firestore.getDocs)((0, _firestore.query)(options.firestore.collection, (0, _firestore.where)((0, _firestore.documentId)(), 'in', ids))).then(result => result.docs).catch(error => {
+              if (error?.code && error.code === 'permission-denied') {
+                // Query may fail due to rules using 'resource' with non existing ids
+                // So try to get the docs one by one
+                return Promise.all(ids.map(id => (0, _firestore.getDoc)((0, _firestore.doc)(options.firestore.collection, id)))).then(docs => docs.filter(doc => doc.exists()));
+              }
+              throw error;
+            });
+          };
+          var docsInDbResult = await (0, _firestoreHelper.getContentByIds)(docIds, getQuery);
+          var docsInDbById = {};
+          docsInDbResult.forEach(row => {
+            var docDataInDb = (0, _firestoreHelper.stripServerTimestampField)(serverTimestampField, row.data());
+            var docId = row.id;
+            docDataInDb[primaryPath] = docId;
+            docsInDbById[docId] = docDataInDb;
+          });
+
+          /**
+           * @link https://firebase.google.com/docs/firestore/manage-data/transactions#batched-writes
+           */
+          var batch = (0, _firestore.writeBatch)(options.firestore.database);
+          var hasWrite = false;
+          await Promise.all(Object.entries(writeRowsById).map(async ([docId, writeRow]) => {
+            var docInDb = docsInDbById[docId];
+            if (docInDb && (!writeRow.assumedMasterState || collection.conflictHandler.isEqual(docInDb, writeRow.assumedMasterState, 'replication-firestore-push') === false)) {
+              // conflict
+              conflicts.push(docInDb);
+            } else {
+              // no conflict
+              hasWrite = true;
+              var docRef = (0, _firestore.doc)(options.firestore.collection, docId);
+              var writeDocData = (0, _index.flatClone)(writeRow.newDocumentState);
+              writeDocData[serverTimestampField] = (0, _firestore.serverTimestamp)();
+              if (!docInDb) {
+                // insert
+                batch.set(docRef, (0, _firestoreHelper.stripPrimaryKey)(primaryPath, writeDocData));
+              } else {
+                // update
+                batch.update(docRef, (0, _firestoreHelper.stripPrimaryKey)(primaryPath, writeDocData));
+              }
+            }
+          }));
+          if (hasWrite) {
+            await batch.commit();
+          }
+        });
+        await (0, _firestore.waitForPendingWrites)(options.firestore.database);
+        return conflicts;
+      },
+      batchSize: options.push.batchSize,
+      modifier: options.push.modifier
+    };
+  }
+  var replicationState = new RxFirestoreReplicationState(options.firestore, options.replicationIdentifier, collection, replicationPrimitivesPull, replicationPrimitivesPush, options.live, options.retryTime, options.autoStart);
+
+  /**
+   * Use long polling to get live changes for the pull.stream$
+   */
+  if (options.live && options.pull) {
+    var startBefore = replicationState.start.bind(replicationState);
+    var cancelBefore = replicationState.cancel.bind(replicationState);
+    replicationState.start = () => {
+      var lastChangeQuery = (0, _firestore.query)(pullQuery, (0, _firestore.orderBy)(serverTimestampField, 'desc'), (0, _firestore.limit)(1));
+      var unsubscribe = (0, _firestore.onSnapshot)(lastChangeQuery, _querySnapshot => {
+        /**
+         * There is no good way to observe the event stream in firestore.
+         * So instead we listen to any write to the collection
+         * and then emit a 'RESYNC' flag.
+         */
+        replicationState.reSync();
+      }, error => {
+        replicationState.subjects.error.next((0, _index4.newRxError)('RC_STREAM', {
+          error: (0, _index.errorToPlainJson)(error)
+        }));
+      });
+      replicationState.cancel = () => {
+        unsubscribe();
+        return cancelBefore();
+      };
+      return startBefore();
+    };
+  }
+  (0, _index3.startReplicationOnLeaderShip)(options.waitForLeadership, replicationState);
+  return replicationState;
+}
+//# sourceMappingURL=index.js.map

package/dist/cjs/plugins/replication-firestore/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":["_index","require","_firestore","_index2","_index3","_index4","_rxjs","_firestoreHelper","Object","keys","forEach","key","prototype","hasOwnProperty","call","_exportNames","exports","defineProperty","enumerable","get","_firestoreTypes","RxFirestoreReplicationState","_RxReplicationState","firestore","replicationIdentifierHash","collection","pull","push","live","retryTime","autoStart","_this","_inheritsLoose2","default","RxReplicationState","replicateFirestore","options","addRxPlugin","RxDBLeaderElectionPlugin","pullStream$","Subject","replicationPrimitivesPull","waitForLeadership","serverTimestampField","primaryPath","schema","schemaPart","getSchemaByObjectPath","jsonSchema","includes","newRxError","field","pullFilters","filter","undefined","toArray","pullQuery","query","handler","lastPulledCheckpoint","batchSize","newerQuery","sameTimeQuery","lastServerTimestamp","isoStringToServerTimestamp","serverTimestamp","where","orderBy","limit","documentId","id","mustsReRun","useDocs","waitForPendingWrites","database","runTransaction","_tx","newerQueryResult","sameTimeQueryResult","Promise","all","getDocs","metadata","hasPendingWrites","ensureNotFalsy","docs","missingAmount","length","additionalDocs","slice","x","concat","checkpoint","documents","lastDoc","lastOfArray","map","row","firestoreRowToDocData","newCheckpoint","serverTimestampToIsoString","data","ret","modifier","stream$","asObservable","initialCheckpoint","replicationPrimitivesPush","pushFilter","rows","asyncFilter","newDocumentState","writeRowsById","docIds","docId","conflicts","getQuery","ids","then","result","catch","error","code","getDoc","doc","exists","docsInDbResult","getContentByIds","docsInDbById","docDataInDb","stripServerTimestampField","batch","writeBatch","hasWrite","entries","writeRow","docInDb","assumedMasterState","conflictHandler","isEqual","docRef","writeDocData","flatClone","set","stripPrimaryKey","update","commit","replicationState","replicationIdentifier","startBefore","start","bind","cancelBefore","cancel","lastChangeQuery","unsubscribe","onSnapshot","_querySnapshot","reSync","subjects","next","errorToPlainJson","startReplicationOnLeaderShip"],"sources":["../../../../src/plugins/replication-firestore/index.ts"],"sourcesContent":["import {\r\n    asyncFilter,\r\n    ensureNotFalsy,\r\n    errorToPlainJson,\r\n    flatClone,\r\n    lastOfArray,\r\n    toArray\r\n} from '../../plugins/utils/index.ts';\r\n\r\nimport {\r\n    doc,\r\n    query,\r\n    where,\r\n    orderBy,\r\n    limit,\r\n    getDocs,\r\n    getDoc,\r\n    onSnapshot,\r\n    runTransaction,\r\n    writeBatch,\r\n    serverTimestamp,\r\n    QueryDocumentSnapshot,\r\n    waitForPendingWrites,\r\n    documentId,\r\n    FirestoreError,\r\n    QueryConstraint\r\n} from 'firebase/firestore';\r\n\r\nimport { RxDBLeaderElectionPlugin } from '../leader-election/index.ts';\r\nimport type {\r\n    RxCollection,\r\n    ReplicationPullOptions,\r\n    ReplicationPushOptions,\r\n    RxReplicationWriteToMasterRow,\r\n    RxReplicationPullStreamItem\r\n} from '../../types/index.d.ts';\r\nimport {\r\n    RxReplicationState,\r\n    startReplicationOnLeaderShip\r\n} from '../replication/index.ts';\r\nimport {\r\n    addRxPlugin,\r\n    ById,\r\n    getSchemaByObjectPath,\r\n    newRxError,\r\n    WithDeleted\r\n} from '../../index.ts';\r\n\r\nimport type {\r\n    FirestoreCheckpointType,\r\n    FirestoreOptions,\r\n    SyncOptionsFirestore\r\n} from './firestore-types.ts';\r\nimport { Subject } from 'rxjs';\r\nimport {\r\n    firestoreRowToDocData,\r\n    getContentByIds,\r\n    isoStringToServerTimestamp,\r\n    serverTimestampToIsoString,\r\n    stripPrimaryKey,\r\n    stripServerTimestampField\r\n} from './firestore-helper.ts';\r\n\r\nexport * from './firestore-helper.ts';\r\nexport * from './firestore-types.ts';\r\n\r\nexport class RxFirestoreReplicationState<RxDocType> extends RxReplicationState<RxDocType, FirestoreCheckpointType> {\r\n    constructor(\r\n        public readonly firestore: FirestoreOptions<RxDocType>,\r\n        public readonly replicationIdentifierHash: string,\r\n        public readonly collection: RxCollection<RxDocType>,\r\n        public readonly pull?: ReplicationPullOptions<RxDocType, FirestoreCheckpointType>,\r\n        public readonly push?: ReplicationPushOptions<RxDocType>,\r\n        public readonly live: boolean = true,\r\n        public retryTime: number = 1000 * 5,\r\n        public autoStart: boolean = true\r\n    ) {\r\n        super(\r\n            replicationIdentifierHash,\r\n            collection,\r\n            '_deleted',\r\n            pull,\r\n            push,\r\n            live,\r\n            retryTime,\r\n            autoStart\r\n        );\r\n    }\r\n}\r\n\r\nexport function replicateFirestore<RxDocType>(\r\n    options: SyncOptionsFirestore<RxDocType>\r\n): RxFirestoreReplicationState<RxDocType> {\r\n    const collection: RxCollection<RxDocType, any, any> = options.collection;\r\n    addRxPlugin(RxDBLeaderElectionPlugin);\r\n    const pullStream$: Subject<RxReplicationPullStreamItem<RxDocType, FirestoreCheckpointType>> = new Subject();\r\n    let replicationPrimitivesPull: ReplicationPullOptions<RxDocType, FirestoreCheckpointType> | undefined;\r\n    options.live = typeof options.live === 'undefined' ? true : options.live;\r\n    options.waitForLeadership = typeof options.waitForLeadership === 'undefined' ? true : options.waitForLeadership;\r\n    const serverTimestampField = typeof options.serverTimestampField === 'undefined' ? 'serverTimestamp' : options.serverTimestampField;\r\n    options.serverTimestampField = serverTimestampField;\r\n    const primaryPath = collection.schema.primaryPath;\r\n\r\n    /**\r\n     * The serverTimestampField MUST NOT be part of the collections RxJsonSchema.\r\n     */\r\n    const schemaPart = getSchemaByObjectPath(collection.schema.jsonSchema, serverTimestampField);\r\n    if (\r\n        schemaPart ||\r\n        // also must not be nested.\r\n        serverTimestampField.includes('.')\r\n    ) {\r\n        throw newRxError('RC6', {\r\n            field: serverTimestampField,\r\n            schema: collection.schema.jsonSchema\r\n        });\r\n    }\r\n\r\n    const pullFilters: QueryConstraint[] = options.pull?.filter !== undefined\r\n        ? toArray(options.pull.filter) as QueryConstraint[]\r\n        : [];\r\n\r\n    const pullQuery = query(options.firestore.collection, ...pullFilters);\r\n\r\n    if (options.pull) {\r\n        replicationPrimitivesPull = {\r\n            async handler(\r\n                lastPulledCheckpoint: FirestoreCheckpointType | undefined,\r\n                batchSize: number\r\n            ) {\r\n                let newerQuery: ReturnType<typeof query>;\r\n                let sameTimeQuery: ReturnType<typeof query> | undefined;\r\n\r\n                if (lastPulledCheckpoint) {\r\n                    const lastServerTimestamp = isoStringToServerTimestamp(lastPulledCheckpoint.serverTimestamp);\r\n                    newerQuery = query(pullQuery,\r\n                        where(serverTimestampField, '>', lastServerTimestamp),\r\n                        orderBy(serverTimestampField, 'asc'),\r\n                        limit(batchSize)\r\n                    );\r\n                    sameTimeQuery = query(pullQuery,\r\n                        where(serverTimestampField, '==', lastServerTimestamp),\r\n                        where(documentId(), '>', lastPulledCheckpoint.id),\r\n                        orderBy(documentId(), 'asc'),\r\n                        limit(batchSize)\r\n                    );\r\n                } else {\r\n                    newerQuery = query(pullQuery,\r\n                        orderBy(serverTimestampField, 'asc'),\r\n                        limit(batchSize)\r\n                    );\r\n                }\r\n\r\n                let mustsReRun = true;\r\n                let useDocs: QueryDocumentSnapshot<RxDocType>[] = [];\r\n                while (mustsReRun) {\r\n                    /**\r\n                     * Local writes that have not been persisted to the server\r\n                     * are in pending state and do not have a correct serverTimestamp set.\r\n                     * We have to ensure we only use document states that are in sync with the server.\r\n                     * @link https://medium.com/firebase-developers/the-secrets-of-firestore-fieldvalue-servertimestamp-revealed-29dd7a38a82b\r\n                     */\r\n                    await waitForPendingWrites(options.firestore.database);\r\n                    await runTransaction(options.firestore.database, async (_tx: any) => {\r\n                        useDocs = [];\r\n                        const [\r\n                            newerQueryResult,\r\n                            sameTimeQueryResult\r\n                        ] = await Promise.all([\r\n                            getDocs(newerQuery),\r\n                            sameTimeQuery ? getDocs(sameTimeQuery) : undefined\r\n                        ]);\r\n\r\n                        if (\r\n                            newerQueryResult.metadata.hasPendingWrites ||\r\n                            (sameTimeQuery && ensureNotFalsy(sameTimeQueryResult).metadata.hasPendingWrites)\r\n                        ) {\r\n                            return;\r\n                        } else {\r\n                            mustsReRun = false;\r\n\r\n                            if (sameTimeQuery) {\r\n                                useDocs = ensureNotFalsy(sameTimeQueryResult).docs as any;\r\n                            }\r\n                            const missingAmount = batchSize - useDocs.length;\r\n                            if (missingAmount > 0) {\r\n                                const additionalDocs = newerQueryResult.docs.slice(0, missingAmount).filter((x: any) => !!x);\r\n                                useDocs = useDocs.concat(additionalDocs as any);\r\n                            }\r\n                        }\r\n                    });\r\n                }\r\n\r\n                if (useDocs.length === 0) {\r\n                    return {\r\n                        checkpoint: lastPulledCheckpoint ?? undefined,\r\n                        documents: []\r\n                    };\r\n                }\r\n                const lastDoc = ensureNotFalsy(lastOfArray(useDocs));\r\n                const documents: WithDeleted<RxDocType>[] = useDocs\r\n                    .map(row => firestoreRowToDocData(\r\n                        serverTimestampField,\r\n                        primaryPath,\r\n                        row\r\n                    ));\r\n                const newCheckpoint: FirestoreCheckpointType = {\r\n                    id: lastDoc.id,\r\n                    serverTimestamp: serverTimestampToIsoString(serverTimestampField, lastDoc.data())\r\n                };\r\n                const ret = {\r\n                    documents: documents,\r\n                    checkpoint: newCheckpoint\r\n                };\r\n                return ret;\r\n            },\r\n            batchSize: ensureNotFalsy(options.pull).batchSize,\r\n            modifier: ensureNotFalsy(options.pull).modifier,\r\n            stream$: pullStream$.asObservable(),\r\n            initialCheckpoint: options.pull.initialCheckpoint\r\n        };\r\n    }\r\n\r\n    let replicationPrimitivesPush: ReplicationPushOptions<RxDocType> | undefined;\r\n    if (options.push) {\r\n        const pushFilter = options.push?.filter;\r\n        replicationPrimitivesPush = {\r\n            async handler(\r\n                rows: RxReplicationWriteToMasterRow<RxDocType>[]\r\n            ) {\r\n                if (pushFilter !== undefined) {\r\n                    rows = await asyncFilter(rows, (row) => pushFilter(row.newDocumentState));\r\n                }\r\n\r\n                const writeRowsById: ById<RxReplicationWriteToMasterRow<RxDocType>> = {};\r\n                const docIds: string[] = rows.map(row => {\r\n                    const docId = (row.newDocumentState as any)[primaryPath];\r\n                    writeRowsById[docId] = row;\r\n                    return docId;\r\n                });\r\n                await waitForPendingWrites(options.firestore.database);\r\n                let conflicts: WithDeleted<RxDocType>[] = [];\r\n\r\n                /**\r\n                 * Everything must run INSIDE of the transaction\r\n                 * because on tx-errors, firebase will re-run the transaction on some cases.\r\n                 * @link https://firebase.google.com/docs/firestore/manage-data/transactions#transaction_failure\r\n                 * @link https://firebase.google.com/docs/firestore/manage-data/transactions\r\n                 */\r\n                await runTransaction(options.firestore.database, async (_tx: any) => {\r\n                    conflicts = []; // reset in case the tx has re-run.\r\n                    /**\r\n                     * @link https://stackoverflow.com/a/48423626/3443137\r\n                     */\r\n\r\n                    const getQuery = (ids: string[]) => {\r\n                        return getDocs(\r\n                            query(\r\n                                options.firestore.collection,\r\n                                where(documentId(), 'in', ids)\r\n                            )\r\n                        )\r\n                        .then((result: any) => result.docs)\r\n                        .catch((error: any) => {\r\n                            if (error?.code && (error as FirestoreError).code === 'permission-denied') {\r\n                                // Query may fail due to rules using 'resource' with non existing ids\r\n                                // So try to get the docs one by one\r\n                                return Promise.all(\r\n                                    ids.map(\r\n                                        id => getDoc(doc(options.firestore.collection, id))\r\n                                    )\r\n                                )\r\n                                .then((docs: any[]) => docs.filter((doc: any) => doc.exists()));\r\n                            }\r\n                            throw error;\r\n                        });\r\n                    };\r\n\r\n                    const docsInDbResult = await getContentByIds<RxDocType>(docIds, getQuery);\r\n\r\n                    const docsInDbById: ById<RxDocType> = {};\r\n                    docsInDbResult.forEach(row => {\r\n                        const docDataInDb = stripServerTimestampField(serverTimestampField, row.data());\r\n                        const docId = row.id;\r\n                        (docDataInDb as any)[primaryPath] = docId;\r\n                        docsInDbById[docId] = docDataInDb;\r\n                    });\r\n\r\n                    /**\r\n                     * @link https://firebase.google.com/docs/firestore/manage-data/transactions#batched-writes\r\n                     */\r\n                    const batch = writeBatch(options.firestore.database);\r\n                    let hasWrite = false;\r\n                    await Promise.all(\r\n                        Object.entries(writeRowsById).map(async ([docId, writeRow]) => {\r\n                            const docInDb: RxDocType | undefined = docsInDbById[docId];\r\n\r\n                            if (\r\n                                docInDb &&\r\n                                (\r\n                                    !writeRow.assumedMasterState ||\r\n                                    collection.conflictHandler.isEqual(docInDb as any, writeRow.assumedMasterState, 'replication-firestore-push') === false\r\n                                )\r\n                            ) {\r\n                                // conflict\r\n                                conflicts.push(docInDb as any);\r\n                            } else {\r\n                                // no conflict\r\n                                hasWrite = true;\r\n                                const docRef = doc(options.firestore.collection, docId);\r\n                                const writeDocData = flatClone(writeRow.newDocumentState);\r\n                                (writeDocData as any)[serverTimestampField] = serverTimestamp();\r\n                                if (!docInDb) {\r\n                                    // insert\r\n                                    batch.set(docRef, stripPrimaryKey(primaryPath, writeDocData));\r\n                                } else {\r\n                                    // update\r\n                                    batch.update(docRef, stripPrimaryKey(primaryPath, writeDocData));\r\n                                }\r\n                            }\r\n                        })\r\n                    );\r\n\r\n                    if (hasWrite) {\r\n                        await batch.commit();\r\n                    }\r\n                });\r\n                await waitForPendingWrites(options.firestore.database);\r\n                return conflicts;\r\n            },\r\n            batchSize: options.push.batchSize,\r\n            modifier: options.push.modifier\r\n        };\r\n    }\r\n\r\n\r\n    const replicationState = new RxFirestoreReplicationState<RxDocType>(\r\n        options.firestore,\r\n        options.replicationIdentifier,\r\n        collection,\r\n        replicationPrimitivesPull,\r\n        replicationPrimitivesPush,\r\n        options.live,\r\n        options.retryTime,\r\n        options.autoStart\r\n    );\r\n\r\n    /**\r\n     * Use long polling to get live changes for the pull.stream$\r\n     */\r\n    if (options.live && options.pull) {\r\n        const startBefore = replicationState.start.bind(replicationState);\r\n        const cancelBefore = replicationState.cancel.bind(replicationState);\r\n        replicationState.start = () => {\r\n            const lastChangeQuery = query(\r\n                pullQuery,\r\n                orderBy(serverTimestampField, 'desc'),\r\n                limit(1)\r\n            );\r\n            const unsubscribe = onSnapshot(\r\n                lastChangeQuery,\r\n                (_querySnapshot: any) => {\r\n                    /**\r\n                     * There is no good way to observe the event stream in firestore.\r\n                     * So instead we listen to any write to the collection\r\n                     * and then emit a 'RESYNC' flag.\r\n                     */\r\n                    replicationState.reSync();\r\n                },\r\n                (error: FirestoreError) => {\r\n                    replicationState.subjects.error.next(\r\n                        newRxError('RC_STREAM', { error: errorToPlainJson(error) })\r\n                    );\r\n                }\r\n            );\r\n            replicationState.cancel = () => {\r\n                unsubscribe();\r\n                return cancelBefore();\r\n            };\r\n            return startBefore();\r\n        };\r\n    }\r\n\r\n    startReplicationOnLeaderShip(options.waitForLeadership, replicationState);\r\n\r\n    return replicationState;\r\n}\r\n"],"mappings":";;;;;;;;;;;;;AAAA,IAAAA,MAAA,GAAAC,OAAA;AASA,IAAAC,UAAA,GAAAD,OAAA;AAmBA,IAAAE,OAAA,GAAAF,OAAA;AAQA,IAAAG,OAAA,GAAAH,OAAA;AAIA,IAAAI,OAAA,GAAAJ,OAAA;AAaA,IAAAK,KAAA,GAAAL,OAAA;AACA,IAAAM,gBAAA,GAAAN,OAAA;AASAO,MAAA,CAAAC,IAAA,CAAAF,gBAAA,EAAAG,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAH,MAAA,CAAAI,SAAA,CAAAC,cAAA,CAAAC,IAAA,CAAAC,YAAA,EAAAJ,GAAA;EAAA,IAAAA,GAAA,IAAAK,OAAA,IAAAA,OAAA,CAAAL,GAAA,MAAAJ,gBAAA,CAAAI,GAAA;EAAAH,MAAA,CAAAS,cAAA,CAAAD,OAAA,EAAAL,GAAA;IAAAO,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAZ,gBAAA,CAAAI,GAAA;IAAA;EAAA;AAAA;AACA,IAAAS,eAAA,GAAAnB,OAAA;AAAAO,MAAA,CAAAC,IAAA,CAAAW,eAAA,EAAAV,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAH,MAAA,CAAAI,SAAA,CAAAC,cAAA,CAAAC,IAAA,CAAAC,YAAA,EAAAJ,GAAA;EAAA,IAAAA,GAAA,IAAAK,OAAA,IAAAA,OAAA,CAAAL,GAAA,MAAAS,eAAA,CAAAT,GAAA;EAAAH,MAAA,CAAAS,cAAA,CAAAD,OAAA,EAAAL,GAAA;IAAAO,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAC,eAAA,CAAAT,GAAA;IAAA;EAAA;AAAA;AAAqC,IAExBU,2BAA2B,GAAAL,OAAA,CAAAK,2BAAA,0BAAAC,mBAAA;EACpC,SAAAD,4BACoBE,SAAsC,EACtCC,yBAAiC,EACjCC,UAAmC,EACnCC,IAAiE,EACjEC,IAAwC,EACxCC,IAAa,GAAG,IAAI,EAC7BC,SAAiB,GAAG,IAAI,GAAG,CAAC,EAC5BC,SAAkB,GAAG,IAAI,EAClC;IAAA,IAAAC,KAAA;IACEA,KAAA,GAAAT,mBAAA,CAAAR,IAAA,OACIU,yBAAyB,EACzBC,UAAU,EACV,UAAU,EACVC,IAAI,EACJC,IAAI,EACJC,IAAI,EACJC,SAAS,EACTC,SACJ,CAAC;IAACC,KAAA,CAlBcR,SAAsC,GAAtCA,SAAsC;IAAAQ,KAAA,CACtCP,yBAAiC,GAAjCA,yBAAiC;IAAAO,KAAA,CACjCN,UAAmC,GAAnCA,UAAmC;IAAAM,KAAA,CACnCL,IAAiE,GAAjEA,IAAiE;IAAAK,KAAA,CACjEJ,IAAwC,GAAxCA,IAAwC;IAAAI,KAAA,CACxCH,IAAa,GAAbA,IAAa;IAAAG,KAAA,CACtBF,SAAiB,GAAjBA,SAAiB;IAAAE,KAAA,CACjBD,SAAkB,GAAlBA,SAAkB;IAAA,OAAAC,KAAA;EAY7B;EAAC,IAAAC,eAAA,CAAAC,OAAA,EAAAZ,2BAAA,EAAAC,mBAAA;EAAA,OAAAD,2BAAA;AAAA,EArBuDa,0BAAkB;AAwBvE,SAASC,kBAAkBA,CAC9BC,OAAwC,EACF;EACtC,IAAMX,UAA6C,GAAGW,OAAO,CAACX,UAAU;EACxE,IAAAY,mBAAW,EAACC,gCAAwB,CAAC;EACrC,IAAMC,WAAqF,GAAG,IAAIC,aAAO,CAAC,CAAC;EAC3G,IAAIC,yBAAiG;EACrGL,OAAO,CAACR,IAAI,GAAG,OAAOQ,OAAO,CAACR,IAAI,KAAK,WAAW,GAAG,IAAI,GAAGQ,OAAO,CAACR,IAAI;EACxEQ,OAAO,CAACM,iBAAiB,GAAG,OAAON,OAAO,CAACM,iBAAiB,KAAK,WAAW,GAAG,IAAI,GAAGN,OAAO,CAACM,iBAAiB;EAC/G,IAAMC,oBAAoB,GAAG,OAAOP,OAAO,CAACO,oBAAoB,KAAK,WAAW,GAAG,iBAAiB,GAAGP,OAAO,CAACO,oBAAoB;EACnIP,OAAO,CAACO,oBAAoB,GAAGA,oBAAoB;EACnD,IAAMC,WAAW,GAAGnB,UAAU,CAACoB,MAAM,CAACD,WAAW;;EAEjD;AACJ;AACA;EACI,IAAME,UAAU,GAAG,IAAAC,6BAAqB,EAACtB,UAAU,CAACoB,MAAM,CAACG,UAAU,EAAEL,oBAAoB,CAAC;EAC5F,IACIG,UAAU;EACV;EACAH,oBAAoB,CAACM,QAAQ,CAAC,GAAG,CAAC,EACpC;IACE,MAAM,IAAAC,kBAAU,EAAC,KAAK,EAAE;MACpBC,KAAK,EAAER,oBAAoB;MAC3BE,MAAM,EAAEpB,UAAU,CAACoB,MAAM,CAACG;IAC9B,CAAC,CAAC;EACN;EAEA,IAAMI,WAA8B,GAAGhB,OAAO,CAACV,IAAI,EAAE2B,MAAM,KAAKC,SAAS,GACnE,IAAAC,cAAO,EAACnB,OAAO,CAACV,IAAI,CAAC2B,MAAM,CAAC,GAC5B,EAAE;EAER,IAAMG,SAAS,GAAG,IAAAC,gBAAK,EAACrB,OAAO,CAACb,SAAS,CAACE,UAAU,EAAE,GAAG2B,WAAW,CAAC;EAErE,IAAIhB,OAAO,CAACV,IAAI,EAAE;IACde,yBAAyB,GAAG;MACxB,MAAMiB,OAAOA,CACTC,oBAAyD,EACzDC,SAAiB,EACnB;QACE,IAAIC,UAAoC;QACxC,IAAIC,aAAmD;QAEvD,IAAIH,oBAAoB,EAAE;UACtB,IAAMI,mBAAmB,GAAG,IAAAC,2CAA0B,EAACL,oBAAoB,CAACM,eAAe,CAAC;UAC5FJ,UAAU,GAAG,IAAAJ,gBAAK,EAACD,SAAS,EACxB,IAAAU,gBAAK,EAACvB,oBAAoB,EAAE,GAAG,EAAEoB,mBAAmB,CAAC,EACrD,IAAAI,kBAAO,EAACxB,oBAAoB,EAAE,KAAK,CAAC,EACpC,IAAAyB,gBAAK,EAACR,SAAS,CACnB,CAAC;UACDE,aAAa,GAAG,IAAAL,gBAAK,EAACD,SAAS,EAC3B,IAAAU,gBAAK,EAACvB,oBAAoB,EAAE,IAAI,EAAEoB,mBAAmB,CAAC,EACtD,IAAAG,gBAAK,EAAC,IAAAG,qBAAU,EAAC,CAAC,EAAE,GAAG,EAAEV,oBAAoB,CAACW,EAAE,CAAC,EACjD,IAAAH,kBAAO,EAAC,IAAAE,qBAAU,EAAC,CAAC,EAAE,KAAK,CAAC,EAC5B,IAAAD,gBAAK,EAACR,SAAS,CACnB,CAAC;QACL,CAAC,MAAM;UACHC,UAAU,GAAG,IAAAJ,gBAAK,EAACD,SAAS,EACxB,IAAAW,kBAAO,EAACxB,oBAAoB,EAAE,KAAK,CAAC,EACpC,IAAAyB,gBAAK,EAACR,SAAS,CACnB,CAAC;QACL;QAEA,IAAIW,UAAU,GAAG,IAAI;QACrB,IAAIC,OAA2C,GAAG,EAAE;QACpD,OAAOD,UAAU,EAAE;UACf;AACpB;AACA;AACA;AACA;AACA;UACoB,MAAM,IAAAE,+BAAoB,EAACrC,OAAO,CAACb,SAAS,CAACmD,QAAQ,CAAC;UACtD,MAAM,IAAAC,yBAAc,EAACvC,OAAO,CAACb,SAAS,CAACmD,QAAQ,EAAE,MAAOE,GAAQ,IAAK;YACjEJ,OAAO,GAAG,EAAE;YACZ,IAAM,CACFK,gBAAgB,EAChBC,mBAAmB,CACtB,GAAG,MAAMC,OAAO,CAACC,GAAG,CAAC,CAClB,IAAAC,kBAAO,EAACpB,UAAU,CAAC,EACnBC,aAAa,GAAG,IAAAmB,kBAAO,EAACnB,aAAa,CAAC,GAAGR,SAAS,CACrD,CAAC;YAEF,IACIuB,gBAAgB,CAACK,QAAQ,CAACC,gBAAgB,IACzCrB,aAAa,IAAI,IAAAsB,qBAAc,EAACN,mBAAmB,CAAC,CAACI,QAAQ,CAACC,gBAAiB,EAClF;cACE;YACJ,CAAC,MAAM;cACHZ,UAAU,GAAG,KAAK;cAElB,IAAIT,aAAa,EAAE;gBACfU,OAAO,GAAG,IAAAY,qBAAc,EAACN,mBAAmB,CAAC,CAACO,IAAW;cAC7D;cACA,IAAMC,aAAa,GAAG1B,SAAS,GAAGY,OAAO,CAACe,MAAM;cAChD,IAAID,aAAa,GAAG,CAAC,EAAE;gBACnB,IAAME,cAAc,GAAGX,gBAAgB,CAACQ,IAAI,CAACI,KAAK,CAAC,CAAC,EAAEH,aAAa,CAAC,CAACjC,MAAM,CAAEqC,CAAM,IAAK,CAAC,CAACA,CAAC,CAAC;gBAC5FlB,OAAO,GAAGA,OAAO,CAACmB,MAAM,CAACH,cAAqB,CAAC;cACnD;YACJ;UACJ,CAAC,CAAC;QACN;QAEA,IAAIhB,OAAO,CAACe,MAAM,KAAK,CAAC,EAAE;UACtB,OAAO;YACHK,UAAU,EAAEjC,oBAAoB,IAAIL,SAAS;YAC7CuC,SAAS,EAAE;UACf,CAAC;QACL;QACA,IAAMC,OAAO,GAAG,IAAAV,qBAAc,EAAC,IAAAW,kBAAW,EAACvB,OAAO,CAAC,CAAC;QACpD,IAAMqB,SAAmC,GAAGrB,OAAO,CAC9CwB,GAAG,CAACC,GAAG,IAAI,IAAAC,sCAAqB,EAC7BvD,oBAAoB,EACpBC,WAAW,EACXqD,GACJ,CAAC,CAAC;QACN,IAAME,aAAsC,GAAG;UAC3C7B,EAAE,EAAEwB,OAAO,CAACxB,EAAE;UACdL,eAAe,EAAE,IAAAmC,2CAA0B,EAACzD,oBAAoB,EAAEmD,OAAO,CAACO,IAAI,CAAC,CAAC;QACpF,CAAC;QACD,IAAMC,GAAG,GAAG;UACRT,SAAS,EAAEA,SAAS;UACpBD,UAAU,EAAEO;QAChB,CAAC;QACD,OAAOG,GAAG;MACd,CAAC;MACD1C,SAAS,EAAE,IAAAwB,qBAAc,EAAChD,OAAO,CAACV,IAAI,CAAC,CAACkC,SAAS;MACjD2C,QAAQ,EAAE,IAAAnB,qBAAc,EAAChD,OAAO,CAACV,IAAI,CAAC,CAAC6E,QAAQ;MAC/CC,OAAO,EAAEjE,WAAW,CAACkE,YAAY,CAAC,CAAC;MACnCC,iBAAiB,EAAEtE,OAAO,CAACV,IAAI,CAACgF;IACpC,CAAC;EACL;EAEA,IAAIC,yBAAwE;EAC5E,IAAIvE,OAAO,CAACT,IAAI,EAAE;IACd,IAAMiF,UAAU,GAAGxE,OAAO,CAACT,IAAI,EAAE0B,MAAM;IACvCsD,yBAAyB,GAAG;MACxB,MAAMjD,OAAOA,CACTmD,IAAgD,EAClD;QACE,IAAID,UAAU,KAAKtD,SAAS,EAAE;UAC1BuD,IAAI,GAAG,MAAM,IAAAC,kBAAW,EAACD,IAAI,EAAGZ,GAAG,IAAKW,UAAU,CAACX,GAAG,CAACc,gBAAgB,CAAC,CAAC;QAC7E;QAEA,IAAMC,aAA6D,GAAG,CAAC,CAAC;QACxE,IAAMC,MAAgB,GAAGJ,IAAI,CAACb,GAAG,CAACC,GAAG,IAAI;UACrC,IAAMiB,KAAK,GAAIjB,GAAG,CAACc,gBAAgB,CAASnE,WAAW,CAAC;UACxDoE,aAAa,CAACE,KAAK,CAAC,GAAGjB,GAAG;UAC1B,OAAOiB,KAAK;QAChB,CAAC,CAAC;QACF,MAAM,IAAAzC,+BAAoB,EAACrC,OAAO,CAACb,SAAS,CAACmD,QAAQ,CAAC;QACtD,IAAIyC,SAAmC,GAAG,EAAE;;QAE5C;AAChB;AACA;AACA;AACA;AACA;QACgB,MAAM,IAAAxC,yBAAc,EAACvC,OAAO,CAACb,SAAS,CAACmD,QAAQ,EAAE,MAAOE,GAAQ,IAAK;UACjEuC,SAAS,GAAG,EAAE,CAAC,CAAC;UAChB;AACpB;AACA;;UAEoB,IAAMC,QAAQ,GAAIC,GAAa,IAAK;YAChC,OAAO,IAAApC,kBAAO,EACV,IAAAxB,gBAAK,EACDrB,OAAO,CAACb,SAAS,CAACE,UAAU,EAC5B,IAAAyC,gBAAK,EAAC,IAAAG,qBAAU,EAAC,CAAC,EAAE,IAAI,EAAEgD,GAAG,CACjC,CACJ,CAAC,CACAC,IAAI,CAAEC,MAAW,IAAKA,MAAM,CAAClC,IAAI,CAAC,CAClCmC,KAAK,CAAEC,KAAU,IAAK;cACnB,IAAIA,KAAK,EAAEC,IAAI,IAAKD,KAAK,CAAoBC,IAAI,KAAK,mBAAmB,EAAE;gBACvE;gBACA;gBACA,OAAO3C,OAAO,CAACC,GAAG,CACdqC,GAAG,CAACrB,GAAG,CACH1B,EAAE,IAAI,IAAAqD,iBAAM,EAAC,IAAAC,cAAG,EAACxF,OAAO,CAACb,SAAS,CAACE,UAAU,EAAE6C,EAAE,CAAC,CACtD,CACJ,CAAC,CACAgD,IAAI,CAAEjC,IAAW,IAAKA,IAAI,CAAChC,MAAM,CAAEuE,GAAQ,IAAKA,GAAG,CAACC,MAAM,CAAC,CAAC,CAAC,CAAC;cACnE;cACA,MAAMJ,KAAK;YACf,CAAC,CAAC;UACN,CAAC;UAED,IAAMK,cAAc,GAAG,MAAM,IAAAC,gCAAe,EAAYd,MAAM,EAAEG,QAAQ,CAAC;UAEzE,IAAMY,YAA6B,GAAG,CAAC,CAAC;UACxCF,cAAc,CAACpH,OAAO,CAACuF,GAAG,IAAI;YAC1B,IAAMgC,WAAW,GAAG,IAAAC,0CAAyB,EAACvF,oBAAoB,EAAEsD,GAAG,CAACI,IAAI,CAAC,CAAC,CAAC;YAC/E,IAAMa,KAAK,GAAGjB,GAAG,CAAC3B,EAAE;YACnB2D,WAAW,CAASrF,WAAW,CAAC,GAAGsE,KAAK;YACzCc,YAAY,CAACd,KAAK,CAAC,GAAGe,WAAW;UACrC,CAAC,CAAC;;UAEF;AACpB;AACA;UACoB,IAAME,KAAK,GAAG,IAAAC,qBAAU,EAAChG,OAAO,CAACb,SAAS,CAACmD,QAAQ,CAAC;UACpD,IAAI2D,QAAQ,GAAG,KAAK;UACpB,MAAMtD,OAAO,CAACC,GAAG,CACbxE,MAAM,CAAC8H,OAAO,CAACtB,aAAa,CAAC,CAAChB,GAAG,CAAC,OAAO,CAACkB,KAAK,EAAEqB,QAAQ,CAAC,KAAK;YAC3D,IAAMC,OAA8B,GAAGR,YAAY,CAACd,KAAK,CAAC;YAE1D,IACIsB,OAAO,KAEH,CAACD,QAAQ,CAACE,kBAAkB,IAC5BhH,UAAU,CAACiH,eAAe,CAACC,OAAO,CAACH,OAAO,EAASD,QAAQ,CAACE,kBAAkB,EAAE,4BAA4B,CAAC,KAAK,KAAK,CAC1H,EACH;cACE;cACAtB,SAAS,CAACxF,IAAI,CAAC6G,OAAc,CAAC;YAClC,CAAC,MAAM;cACH;cACAH,QAAQ,GAAG,IAAI;cACf,IAAMO,MAAM,GAAG,IAAAhB,cAAG,EAACxF,OAAO,CAACb,SAAS,CAACE,UAAU,EAAEyF,KAAK,CAAC;cACvD,IAAM2B,YAAY,GAAG,IAAAC,gBAAS,EAACP,QAAQ,CAACxB,gBAAgB,CAAC;cACxD8B,YAAY,CAASlG,oBAAoB,CAAC,GAAG,IAAAsB,0BAAe,EAAC,CAAC;cAC/D,IAAI,CAACuE,OAAO,EAAE;gBACV;gBACAL,KAAK,CAACY,GAAG,CAACH,MAAM,EAAE,IAAAI,gCAAe,EAACpG,WAAW,EAAEiG,YAAY,CAAC,CAAC;cACjE,CAAC,MAAM;gBACH;gBACAV,KAAK,CAACc,MAAM,CAACL,MAAM,EAAE,IAAAI,gCAAe,EAACpG,WAAW,EAAEiG,YAAY,CAAC,CAAC;cACpE;YACJ;UACJ,CAAC,CACL,CAAC;UAED,IAAIR,QAAQ,EAAE;YACV,MAAMF,KAAK,CAACe,MAAM,CAAC,CAAC;UACxB;QACJ,CAAC,CAAC;QACF,MAAM,IAAAzE,+BAAoB,EAACrC,OAAO,CAACb,SAAS,CAACmD,QAAQ,CAAC;QACtD,OAAOyC,SAAS;MACpB,CAAC;MACDvD,SAAS,EAAExB,OAAO,CAACT,IAAI,CAACiC,SAAS;MACjC2C,QAAQ,EAAEnE,OAAO,CAACT,IAAI,CAAC4E;IAC3B,CAAC;EACL;EAGA,IAAM4C,gBAAgB,GAAG,IAAI9H,2BAA2B,CACpDe,OAAO,CAACb,SAAS,EACjBa,OAAO,CAACgH,qBAAqB,EAC7B3H,UAAU,EACVgB,yBAAyB,EACzBkE,yBAAyB,EACzBvE,OAAO,CAACR,IAAI,EACZQ,OAAO,CAACP,SAAS,EACjBO,OAAO,CAACN,SACZ,CAAC;;EAED;AACJ;AACA;EACI,IAAIM,OAAO,CAACR,IAAI,IAAIQ,OAAO,CAACV,IAAI,EAAE;IAC9B,IAAM2H,WAAW,GAAGF,gBAAgB,CAACG,KAAK,CAACC,IAAI,CAACJ,gBAAgB,CAAC;IACjE,IAAMK,YAAY,GAAGL,gBAAgB,CAACM,MAAM,CAACF,IAAI,CAACJ,gBAAgB,CAAC;IACnEA,gBAAgB,CAACG,KAAK,GAAG,MAAM;MAC3B,IAAMI,eAAe,GAAG,IAAAjG,gBAAK,EACzBD,SAAS,EACT,IAAAW,kBAAO,EAACxB,oBAAoB,EAAE,MAAM,CAAC,EACrC,IAAAyB,gBAAK,EAAC,CAAC,CACX,CAAC;MACD,IAAMuF,WAAW,GAAG,IAAAC,qBAAU,EAC1BF,eAAe,EACdG,cAAmB,IAAK;QACrB;AACpB;AACA;AACA;AACA;QACoBV,gBAAgB,CAACW,MAAM,CAAC,CAAC;MAC7B,CAAC,EACArC,KAAqB,IAAK;QACvB0B,gBAAgB,CAACY,QAAQ,CAACtC,KAAK,CAACuC,IAAI,CAChC,IAAA9G,kBAAU,EAAC,WAAW,EAAE;UAAEuE,KAAK,EAAE,IAAAwC,uBAAgB,EAACxC,KAAK;QAAE,CAAC,CAC9D,CAAC;MACL,CACJ,CAAC;MACD0B,gBAAgB,CAACM,MAAM,GAAG,MAAM;QAC5BE,WAAW,CAAC,CAAC;QACb,OAAOH,YAAY,CAAC,CAAC;MACzB,CAAC;MACD,OAAOH,WAAW,CAAC,CAAC;IACxB,CAAC;EACL;EAEA,IAAAa,oCAA4B,EAAC9H,OAAO,CAACM,iBAAiB,EAAEyG,gBAAgB,CAAC;EAEzE,OAAOA,gBAAgB;AAC3B","ignoreList":[]}

package/dist/cjs/plugins/replication-google-drive/document-handling.js

@@ -0,0 +1,150 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.batchGetFilesMetadata = batchGetFilesMetadata;
+exports.fetchDocumentContents = fetchDocumentContents;
+exports.getDocumentFiles = getDocumentFiles;
+exports.insertDocumentFiles = insertDocumentFiles;
+exports.updateDocumentFiles = updateDocumentFiles;
+var _rxError = require("../../rx-error.js");
+var _index = require("../utils/index.js");
+var _googleDriveHelper = require("./google-drive-helper.js");
+var MAX_DRIVE_PAGE_SIZE = 1000;
+async function getDocumentFiles(googleDriveOptions, init, docIds) {
+  var fileNames = docIds.map(id => id + '.json');
+  var q = fileNames.map(name => "name = '" + name.replace("'", "\\'") + "'").join(' or ');
+  q = '(' + q + ') and trashed = false';
+  q += ' and \'' + init.docsFolderId + '\' in parents';
+  if (docIds.length >= MAX_DRIVE_PAGE_SIZE) {
+    throw (0, _rxError.newRxError)('SNH');
+  }
+  var params = new URLSearchParams({
+    q,
+    pageSize: MAX_DRIVE_PAGE_SIZE + '',
+    fields: "nextPageToken, files(id,name,mimeType,parents,modifiedTime,size)",
+    // Shared drives support (safe to include always)
+    includeItemsFromAllDrives: "true",
+    supportsAllDrives: "true"
+  });
+  var url = googleDriveOptions.apiEndpoint + '/drive/v3/files?' + params.toString();
+  var res = await fetch(url, {
+    method: "GET",
+    headers: {
+      Authorization: "Bearer " + googleDriveOptions.authToken
+    }
+  });
+  if (!res.ok) {
+    throw await (0, _rxError.newRxFetchError)(res, {
+      ids: docIds
+    });
+  }
+  var data = await res.json();
+  return data;
+}
+async function batchGetFilesMetadata(googleDriveOptions, fileIds) {
+  var boundary = "batch_" + Math.random().toString(16).slice(2);
+  var parts = fileIds.map((id, i) => {
+    return "--" + boundary + "\r\n" + "Content-Type: application/http\r\n" + ("Content-ID: <item-" + i + ">\r\n\r\n") + ("GET /drive/v3/files/" + encodeURIComponent(id) + "?") + "fields=id,name,mimeType,parents,modifiedTime,size&supportsAllDrives=true HTTP/1.1\r\n\r\n";
+  });
+  var batchBody = parts.join("") + ("--" + boundary + "--");
+  var res = await fetch(googleDriveOptions.apiEndpoint + "/batch/drive/v3", {
+    method: "POST",
+    headers: {
+      Authorization: "Bearer " + googleDriveOptions.authToken,
+      "Content-Type": "multipart/mixed; boundary=" + boundary
+    },
+    body: batchBody
+  });
+  if (!res.ok) {
+    throw await (0, _rxError.newRxFetchError)(res);
+  }
+
+  // multipart/mixed; you can parse it, or just return raw.
+  return await res.text();
+}
+async function insertDocumentFiles(googleDriveOptions, init, primaryPath, docs) {
+  // Run uploads in parallel
+  await Promise.all(docs.map(async doc => {
+    var id = doc[primaryPath];
+    await (0, _googleDriveHelper.insertMultipartFile)(googleDriveOptions, init.docsFolderId, id + '.json', doc);
+  }));
+}
+async function updateDocumentFiles(googleDriveOptions, primaryPath, docs, fileIdByDocId, concurrency = 5) {
+  var queue = docs.slice(0);
+  var results = {};
+  async function worker() {
+    while (queue.length) {
+      var doc = queue.shift();
+      var docId = doc[primaryPath];
+      var fileId = (0, _index.ensureNotFalsy)(fileIdByDocId[docId]);
+      var url = googleDriveOptions.apiEndpoint + ("/upload/drive/v3/files/" + encodeURIComponent(fileId)) + "?uploadType=media&supportsAllDrives=true&fields=id";
+      var res = await fetch(url, {
+        method: "PATCH",
+        headers: {
+          Authorization: "Bearer " + googleDriveOptions.authToken,
+          "Content-Type": "application/json; charset=UTF-8"
+        },
+        body: JSON.stringify(doc)
+      });
+      if (!res.ok) {
+        throw await (0, _rxError.newRxFetchError)(res, {
+          args: {
+            docId,
+            fileId
+          }
+        });
+      }
+      results[docId] = await res.json(); // { id }
+    }
+  }
+  await Promise.all(Array.from({
+    length: concurrency
+  }, () => worker()));
+  return results;
+}
+async function fetchDocumentContents(googleDriveOptions, fileIds, concurrency = 5) {
+  var byId = {};
+  var ordered = new Array(fileIds.length);
+  var nextIndex = 0;
+  var sleep = ms => new Promise(r => setTimeout(r, ms));
+  async function fetchOne(fileId, attempt = 0) {
+    var url = googleDriveOptions.apiEndpoint + ("/drive/v3/files/" + encodeURIComponent(fileId) + "?alt=media&supportsAllDrives=true");
+    var res = await fetch(url, {
+      headers: {
+        Authorization: "Bearer " + googleDriveOptions.authToken
+      }
+    });
+    if ([429, 500, 502, 503, 504].includes(res.status) && attempt < 4) {
+      var backoffMs = 250 * Math.pow(2, attempt) + Math.floor(Math.random() * 200);
+      await sleep(backoffMs);
+      return fetchOne(fileId, attempt + 1);
+    }
+    if (!res.ok) throw await (0, _rxError.newRxFetchError)(res, {
+      args: {
+        fileId
+      }
+    });
+    var text = await res.text();
+    return text ? JSON.parse(text) : undefined;
+  }
+  async function worker() {
+    while (true) {
+      var i = nextIndex++;
+      if (i >= fileIds.length) return;
+      var fileId = fileIds[i];
+      var doc = await fetchOne(fileId);
+      ordered[i] = (0, _index.ensureNotFalsy)(doc);
+      byId[fileId] = doc; // if ById allows undefined, remove `as any`
+    }
+  }
+  await Promise.all(Array.from({
+    length: concurrency
+  }, () => worker()));
+  return {
+    byId,
+    ordered
+  };
+}
+//# sourceMappingURL=document-handling.js.map

package/dist/cjs/plugins/replication-google-drive/document-handling.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-handling.js","names":["_rxError","require","_index","_googleDriveHelper","MAX_DRIVE_PAGE_SIZE","getDocumentFiles","googleDriveOptions","init","docIds","fileNames","map","id","q","name","replace","join","docsFolderId","length","newRxError","params","URLSearchParams","pageSize","fields","includeItemsFromAllDrives","supportsAllDrives","url","apiEndpoint","toString","res","fetch","method","headers","Authorization","authToken","ok","newRxFetchError","ids","data","json","batchGetFilesMetadata","fileIds","boundary","Math","random","slice","parts","i","encodeURIComponent","batchBody","body","text","insertDocumentFiles","primaryPath","docs","Promise","all","doc","insertMultipartFile","updateDocumentFiles","fileIdByDocId","concurrency","queue","results","worker","shift","docId","fileId","ensureNotFalsy","JSON","stringify","args","Array","from","fetchDocumentContents","byId","ordered","nextIndex","sleep","ms","r","setTimeout","fetchOne","attempt","includes","status","backoffMs","pow","floor","parse","undefined"],"sources":["../../../../src/plugins/replication-google-drive/document-handling.ts"],"sourcesContent":["import { newRxError, newRxFetchError } from '../../rx-error.ts';\r\nimport { ById } from '../../types/util';\r\nimport { ensureNotFalsy } from '../utils/index.ts';\r\nimport { insertMultipartFile } from './google-drive-helper.ts';\r\nimport type {\r\n    DriveFileListResponse,\r\n    GoogleDriveOptionsWithDefaults\r\n} from './google-drive-types.ts';\r\nimport { DriveStructure } from './init.ts';\r\n\r\nconst MAX_DRIVE_PAGE_SIZE = 1000;\r\n\r\n\r\n\r\nexport async function getDocumentFiles(\r\n    googleDriveOptions: GoogleDriveOptionsWithDefaults,\r\n    init: DriveStructure,\r\n    docIds: string[]\r\n): Promise<DriveFileListResponse> {\r\n    const fileNames = docIds.map(id => id + '.json');\r\n    let q = fileNames\r\n        .map(name => `name = '${name.replace(\"'\", \"\\\\'\")}'`)\r\n        .join(' or ');\r\n    q = '(' + q + ') and trashed = false';\r\n    q += ' and \\'' + init.docsFolderId + '\\' in parents';\r\n    if (docIds.length >= MAX_DRIVE_PAGE_SIZE) {\r\n        throw newRxError('SNH');\r\n    }\r\n    const params = new URLSearchParams({\r\n        q,\r\n        pageSize: MAX_DRIVE_PAGE_SIZE + '',\r\n        fields: \"nextPageToken, files(id,name,mimeType,parents,modifiedTime,size)\",\r\n        // Shared drives support (safe to include always)\r\n        includeItemsFromAllDrives: \"true\",\r\n        supportsAllDrives: \"true\",\r\n    });\r\n    const url = googleDriveOptions.apiEndpoint + '/drive/v3/files?' + params.toString();\r\n    const res = await fetch(url, {\r\n        method: \"GET\",\r\n        headers: {\r\n            Authorization: `Bearer ${googleDriveOptions.authToken}`,\r\n        },\r\n    });\r\n\r\n    if (!res.ok) {\r\n        throw await newRxFetchError(res, {\r\n            ids: docIds,\r\n        });\r\n    }\r\n\r\n    const data = await res.json();\r\n    return data;\r\n}\r\n\r\nexport async function batchGetFilesMetadata(\r\n    googleDriveOptions: GoogleDriveOptionsWithDefaults,\r\n    fileIds: string[]\r\n) {\r\n    const boundary = \"batch_\" + Math.random().toString(16).slice(2);\r\n\r\n    const parts = fileIds.map((id, i) => {\r\n        return (\r\n            `--${boundary}\\r\\n` +\r\n            `Content-Type: application/http\\r\\n` +\r\n            `Content-ID: <item-${i}>\\r\\n\\r\\n` +\r\n            `GET /drive/v3/files/${encodeURIComponent(id)}?` +\r\n            `fields=id,name,mimeType,parents,modifiedTime,size&supportsAllDrives=true HTTP/1.1\\r\\n\\r\\n`\r\n        );\r\n    });\r\n\r\n    const batchBody = parts.join(\"\") + `--${boundary}--`;\r\n\r\n    const res = await fetch(googleDriveOptions.apiEndpoint + \"/batch/drive/v3\", {\r\n        method: \"POST\",\r\n        headers: {\r\n            Authorization: `Bearer ${googleDriveOptions.authToken}`,\r\n            \"Content-Type\": `multipart/mixed; boundary=${boundary}`,\r\n        },\r\n        body: batchBody,\r\n    });\r\n\r\n    if (!res.ok) {\r\n        throw await newRxFetchError(res);\r\n    }\r\n\r\n    // multipart/mixed; you can parse it, or just return raw.\r\n    return await res.text();\r\n}\r\n\r\nexport async function insertDocumentFiles<RxDocType>(\r\n    googleDriveOptions: GoogleDriveOptionsWithDefaults,\r\n    init: DriveStructure,\r\n    primaryPath: keyof RxDocType,\r\n    docs: RxDocType[]\r\n) {\r\n    // Run uploads in parallel\r\n    await Promise.all(docs.map(async (doc) => {\r\n        const id = (doc as any)[primaryPath];\r\n        await insertMultipartFile(\r\n            googleDriveOptions,\r\n            init.docsFolderId,\r\n            id + '.json',\r\n            doc\r\n        );\r\n    }));\r\n}\r\n\r\nexport async function updateDocumentFiles<DocType>(\r\n    googleDriveOptions: GoogleDriveOptionsWithDefaults,\r\n    primaryPath: keyof DocType,\r\n    docs: DocType[],\r\n    fileIdByDocId: Record<string, string>,\r\n    concurrency = 5\r\n) {\r\n    const queue = docs.slice(0);\r\n    const results: Record<string, { id: string }> = {};\r\n\r\n    async function worker() {\r\n        while (queue.length) {\r\n            const doc = queue.shift()!;\r\n            const docId = (doc as any)[primaryPath] as string;\r\n            const fileId = ensureNotFalsy(fileIdByDocId[docId]);\r\n\r\n            const url =\r\n                googleDriveOptions.apiEndpoint +\r\n                `/upload/drive/v3/files/${encodeURIComponent(fileId)}` +\r\n                `?uploadType=media&supportsAllDrives=true&fields=id`;\r\n\r\n            const res = await fetch(url, {\r\n                method: \"PATCH\",\r\n                headers: {\r\n                    Authorization: `Bearer ${googleDriveOptions.authToken}`,\r\n                    \"Content-Type\": \"application/json; charset=UTF-8\",\r\n                },\r\n                body: JSON.stringify(doc),\r\n            });\r\n\r\n            if (!res.ok) {\r\n                throw await newRxFetchError(res, {\r\n                    args: {\r\n                        docId,\r\n                        fileId\r\n                    }\r\n                });\r\n            }\r\n\r\n            results[docId] = await res.json(); // { id }\r\n        }\r\n    }\r\n\r\n    await Promise.all(Array.from({ length: concurrency }, () => worker()));\r\n    return results;\r\n}\r\n\r\n\r\nexport async function fetchDocumentContents<DocType>(\r\n    googleDriveOptions: GoogleDriveOptionsWithDefaults,\r\n    fileIds: string[],\r\n    concurrency = 5\r\n): Promise<{ byId: ById<DocType>; ordered: (DocType)[] }> {\r\n    const byId: ById<DocType> = {};\r\n    const ordered = new Array<DocType>(fileIds.length);\r\n    let nextIndex = 0;\r\n\r\n    const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));\r\n\r\n    async function fetchOne(fileId: string, attempt = 0): Promise<DocType | undefined> {\r\n        const url =\r\n            googleDriveOptions.apiEndpoint +\r\n            `/drive/v3/files/${encodeURIComponent(fileId)}?alt=media&supportsAllDrives=true`;\r\n\r\n        const res = await fetch(url, {\r\n            headers: { Authorization: `Bearer ${googleDriveOptions.authToken}` },\r\n        });\r\n\r\n        if ([429, 500, 502, 503, 504].includes(res.status) && attempt < 4) {\r\n            const backoffMs = 250 * Math.pow(2, attempt) + Math.floor(Math.random() * 200);\r\n            await sleep(backoffMs);\r\n            return fetchOne(fileId, attempt + 1);\r\n        }\r\n\r\n        if (!res.ok) throw await newRxFetchError(res, { args: { fileId } });\r\n\r\n        const text = await res.text();\r\n        return text ? (JSON.parse(text) as DocType) : undefined;\r\n    }\r\n\r\n    async function worker() {\r\n        while (true) {\r\n            const i = nextIndex++;\r\n            if (i >= fileIds.length) return;\r\n\r\n            const fileId = fileIds[i];\r\n            const doc = await fetchOne(fileId);\r\n\r\n            ordered[i] = ensureNotFalsy(doc);\r\n            byId[fileId] = doc as any; // if ById allows undefined, remove `as any`\r\n        }\r\n    }\r\n\r\n    await Promise.all(Array.from({ length: concurrency }, () => worker()));\r\n    return { byId, ordered };\r\n}\r\n"],"mappings":";;;;;;;;;;AAAA,IAAAA,QAAA,GAAAC,OAAA;AAEA,IAAAC,MAAA,GAAAD,OAAA;AACA,IAAAE,kBAAA,GAAAF,OAAA;AAOA,IAAMG,mBAAmB,GAAG,IAAI;AAIzB,eAAeC,gBAAgBA,CAClCC,kBAAkD,EAClDC,IAAoB,EACpBC,MAAgB,EACc;EAC9B,IAAMC,SAAS,GAAGD,MAAM,CAACE,GAAG,CAACC,EAAE,IAAIA,EAAE,GAAG,OAAO,CAAC;EAChD,IAAIC,CAAC,GAAGH,SAAS,CACZC,GAAG,CAACG,IAAI,iBAAeA,IAAI,CAACC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,MAAG,CAAC,CACnDC,IAAI,CAAC,MAAM,CAAC;EACjBH,CAAC,GAAG,GAAG,GAAGA,CAAC,GAAG,uBAAuB;EACrCA,CAAC,IAAI,SAAS,GAAGL,IAAI,CAACS,YAAY,GAAG,eAAe;EACpD,IAAIR,MAAM,CAACS,MAAM,IAAIb,mBAAmB,EAAE;IACtC,MAAM,IAAAc,mBAAU,EAAC,KAAK,CAAC;EAC3B;EACA,IAAMC,MAAM,GAAG,IAAIC,eAAe,CAAC;IAC/BR,CAAC;IACDS,QAAQ,EAAEjB,mBAAmB,GAAG,EAAE;IAClCkB,MAAM,EAAE,kEAAkE;IAC1E;IACAC,yBAAyB,EAAE,MAAM;IACjCC,iBAAiB,EAAE;EACvB,CAAC,CAAC;EACF,IAAMC,GAAG,GAAGnB,kBAAkB,CAACoB,WAAW,GAAG,kBAAkB,GAAGP,MAAM,CAACQ,QAAQ,CAAC,CAAC;EACnF,IAAMC,GAAG,GAAG,MAAMC,KAAK,CAACJ,GAAG,EAAE;IACzBK,MAAM,EAAE,KAAK;IACbC,OAAO,EAAE;MACLC,aAAa,cAAY1B,kBAAkB,CAAC2B;IAChD;EACJ,CAAC,CAAC;EAEF,IAAI,CAACL,GAAG,CAACM,EAAE,EAAE;IACT,MAAM,MAAM,IAAAC,wBAAe,EAACP,GAAG,EAAE;MAC7BQ,GAAG,EAAE5B;IACT,CAAC,CAAC;EACN;EAEA,IAAM6B,IAAI,GAAG,MAAMT,GAAG,CAACU,IAAI,CAAC,CAAC;EAC7B,OAAOD,IAAI;AACf;AAEO,eAAeE,qBAAqBA,CACvCjC,kBAAkD,EAClDkC,OAAiB,EACnB;EACE,IAAMC,QAAQ,GAAG,QAAQ,GAAGC,IAAI,CAACC,MAAM,CAAC,CAAC,CAAChB,QAAQ,CAAC,EAAE,CAAC,CAACiB,KAAK,CAAC,CAAC,CAAC;EAE/D,IAAMC,KAAK,GAAGL,OAAO,CAAC9B,GAAG,CAAC,CAACC,EAAE,EAAEmC,CAAC,KAAK;IACjC,OACI,OAAKL,QAAQ,gDACuB,2BACfK,CAAC,eAAW,6BACVC,kBAAkB,CAACpC,EAAE,CAAC,OAAG,8FAC2C;EAEnG,CAAC,CAAC;EAEF,IAAMqC,SAAS,GAAGH,KAAK,CAAC9B,IAAI,CAAC,EAAE,CAAC,WAAQ0B,QAAQ,QAAI;EAEpD,IAAMb,GAAG,GAAG,MAAMC,KAAK,CAACvB,kBAAkB,CAACoB,WAAW,GAAG,iBAAiB,EAAE;IACxEI,MAAM,EAAE,MAAM;IACdC,OAAO,EAAE;MACLC,aAAa,cAAY1B,kBAAkB,CAAC2B,SAAW;MACvD,cAAc,iCAA+BQ;IACjD,CAAC;IACDQ,IAAI,EAAED;EACV,CAAC,CAAC;EAEF,IAAI,CAACpB,GAAG,CAACM,EAAE,EAAE;IACT,MAAM,MAAM,IAAAC,wBAAe,EAACP,GAAG,CAAC;EACpC;;EAEA;EACA,OAAO,MAAMA,GAAG,CAACsB,IAAI,CAAC,CAAC;AAC3B;AAEO,eAAeC,mBAAmBA,CACrC7C,kBAAkD,EAClDC,IAAoB,EACpB6C,WAA4B,EAC5BC,IAAiB,EACnB;EACE;EACA,MAAMC,OAAO,CAACC,GAAG,CAACF,IAAI,CAAC3C,GAAG,CAAC,MAAO8C,GAAG,IAAK;IACtC,IAAM7C,EAAE,GAAI6C,GAAG,CAASJ,WAAW,CAAC;IACpC,MAAM,IAAAK,sCAAmB,EACrBnD,kBAAkB,EAClBC,IAAI,CAACS,YAAY,EACjBL,EAAE,GAAG,OAAO,EACZ6C,GACJ,CAAC;EACL,CAAC,CAAC,CAAC;AACP;AAEO,eAAeE,mBAAmBA,CACrCpD,kBAAkD,EAClD8C,WAA0B,EAC1BC,IAAe,EACfM,aAAqC,EACrCC,WAAW,GAAG,CAAC,EACjB;EACE,IAAMC,KAAK,GAAGR,IAAI,CAACT,KAAK,CAAC,CAAC,CAAC;EAC3B,IAAMkB,OAAuC,GAAG,CAAC,CAAC;EAElD,eAAeC,MAAMA,CAAA,EAAG;IACpB,OAAOF,KAAK,CAAC5C,MAAM,EAAE;MACjB,IAAMuC,GAAG,GAAGK,KAAK,CAACG,KAAK,CAAC,CAAE;MAC1B,IAAMC,KAAK,GAAIT,GAAG,CAASJ,WAAW,CAAW;MACjD,IAAMc,MAAM,GAAG,IAAAC,qBAAc,EAACR,aAAa,CAACM,KAAK,CAAC,CAAC;MAEnD,IAAMxC,GAAG,GACLnB,kBAAkB,CAACoB,WAAW,gCACJqB,kBAAkB,CAACmB,MAAM,CAAC,CAAE,uDACF;MAExD,IAAMtC,GAAG,GAAG,MAAMC,KAAK,CAACJ,GAAG,EAAE;QACzBK,MAAM,EAAE,OAAO;QACfC,OAAO,EAAE;UACLC,aAAa,cAAY1B,kBAAkB,CAAC2B,SAAW;UACvD,cAAc,EAAE;QACpB,CAAC;QACDgB,IAAI,EAAEmB,IAAI,CAACC,SAAS,CAACb,GAAG;MAC5B,CAAC,CAAC;MAEF,IAAI,CAAC5B,GAAG,CAACM,EAAE,EAAE;QACT,MAAM,MAAM,IAAAC,wBAAe,EAACP,GAAG,EAAE;UAC7B0C,IAAI,EAAE;YACFL,KAAK;YACLC;UACJ;QACJ,CAAC,CAAC;MACN;MAEAJ,OAAO,CAACG,KAAK,CAAC,GAAG,MAAMrC,GAAG,CAACU,IAAI,CAAC,CAAC,CAAC,CAAC;IACvC;EACJ;EAEA,MAAMgB,OAAO,CAACC,GAAG,CAACgB,KAAK,CAACC,IAAI,CAAC;IAAEvD,MAAM,EAAE2C;EAAY,CAAC,EAAE,MAAMG,MAAM,CAAC,CAAC,CAAC,CAAC;EACtE,OAAOD,OAAO;AAClB;AAGO,eAAeW,qBAAqBA,CACvCnE,kBAAkD,EAClDkC,OAAiB,EACjBoB,WAAW,GAAG,CAAC,EACuC;EACtD,IAAMc,IAAmB,GAAG,CAAC,CAAC;EAC9B,IAAMC,OAAO,GAAG,IAAIJ,KAAK,CAAU/B,OAAO,CAACvB,MAAM,CAAC;EAClD,IAAI2D,SAAS,GAAG,CAAC;EAEjB,IAAMC,KAAK,GAAIC,EAAU,IAAK,IAAIxB,OAAO,CAAEyB,CAAC,IAAKC,UAAU,CAACD,CAAC,EAAED,EAAE,CAAC,CAAC;EAEnE,eAAeG,QAAQA,CAACf,MAAc,EAAEgB,OAAO,GAAG,CAAC,EAAgC;IAC/E,IAAMzD,GAAG,GACLnB,kBAAkB,CAACoB,WAAW,yBACXqB,kBAAkB,CAACmB,MAAM,CAAC,uCAAmC;IAEpF,IAAMtC,GAAG,GAAG,MAAMC,KAAK,CAACJ,GAAG,EAAE;MACzBM,OAAO,EAAE;QAAEC,aAAa,cAAY1B,kBAAkB,CAAC2B;MAAY;IACvE,CAAC,CAAC;IAEF,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAACkD,QAAQ,CAACvD,GAAG,CAACwD,MAAM,CAAC,IAAIF,OAAO,GAAG,CAAC,EAAE;MAC/D,IAAMG,SAAS,GAAG,GAAG,GAAG3C,IAAI,CAAC4C,GAAG,CAAC,CAAC,EAAEJ,OAAO,CAAC,GAAGxC,IAAI,CAAC6C,KAAK,CAAC7C,IAAI,CAACC,MAAM,CAAC,CAAC,GAAG,GAAG,CAAC;MAC9E,MAAMkC,KAAK,CAACQ,SAAS,CAAC;MACtB,OAAOJ,QAAQ,CAACf,MAAM,EAAEgB,OAAO,GAAG,CAAC,CAAC;IACxC;IAEA,IAAI,CAACtD,GAAG,CAACM,EAAE,EAAE,MAAM,MAAM,IAAAC,wBAAe,EAACP,GAAG,EAAE;MAAE0C,IAAI,EAAE;QAAEJ;MAAO;IAAE,CAAC,CAAC;IAEnE,IAAMhB,IAAI,GAAG,MAAMtB,GAAG,CAACsB,IAAI,CAAC,CAAC;IAC7B,OAAOA,IAAI,GAAIkB,IAAI,CAACoB,KAAK,CAACtC,IAAI,CAAC,GAAeuC,SAAS;EAC3D;EAEA,eAAe1B,MAAMA,CAAA,EAAG;IACpB,OAAO,IAAI,EAAE;MACT,IAAMjB,CAAC,GAAG8B,SAAS,EAAE;MACrB,IAAI9B,CAAC,IAAIN,OAAO,CAACvB,MAAM,EAAE;MAEzB,IAAMiD,MAAM,GAAG1B,OAAO,CAACM,CAAC,CAAC;MACzB,IAAMU,GAAG,GAAG,MAAMyB,QAAQ,CAACf,MAAM,CAAC;MAElCS,OAAO,CAAC7B,CAAC,CAAC,GAAG,IAAAqB,qBAAc,EAACX,GAAG,CAAC;MAChCkB,IAAI,CAACR,MAAM,CAAC,GAAGV,GAAU,CAAC,CAAC;IAC/B;EACJ;EAEA,MAAMF,OAAO,CAACC,GAAG,CAACgB,KAAK,CAACC,IAAI,CAAC;IAAEvD,MAAM,EAAE2C;EAAY,CAAC,EAAE,MAAMG,MAAM,CAAC,CAAC,CAAC,CAAC;EACtE,OAAO;IAAEW,IAAI;IAAEC;EAAQ,CAAC;AAC5B","ignoreList":[]}

package/dist/cjs/plugins/replication-google-drive/downstream.js

@@ -0,0 +1,73 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.fetchChanges = fetchChanges;
+exports.fetchChangesFiles = fetchChangesFiles;
+var _rxError = require("../../rx-error.js");
+var _index = require("../utils/index.js");
+var _documentHandling = require("./document-handling.js");
+async function fetchChanges(googleDriveOptions, init, checkpoint, batchSize = 10) {
+  var filesResult = await fetchChangesFiles(googleDriveOptions, init, checkpoint, batchSize);
+  var contents = await (0, _documentHandling.fetchDocumentContents)(googleDriveOptions, filesResult.files.map(file => file.id));
+  return {
+    checkpoint: filesResult.checkpoint,
+    documents: contents.ordered
+  };
+}
+async function fetchChangesFiles(googleDriveOptions, init, checkpoint, batchSize = 10) {
+  var queryParts = ["'" + init.docsFolderId + "' in parents", "and trashed = false"];
+  if (checkpoint) {
+    queryParts.push("and modifiedTime >= '" + checkpoint.modifiedTime + "'");
+  }
+
+  /**
+   * Intentionally overfetch in case
+   * multiple docs have the same modifiedTime.
+   * We later have to strip the additional ones.
+   */
+  var OVERFETCH_AMOUNT = 6;
+  var params = new URLSearchParams({
+    q: queryParts.join(' '),
+    pageSize: batchSize + OVERFETCH_AMOUNT + '',
+    orderBy: "modifiedTime asc,name asc",
+    fields: "files(id,name,mimeType,parents,modifiedTime,size)",
+    supportsAllDrives: "true",
+    includeItemsFromAllDrives: "true"
+  });
+  var url = googleDriveOptions.apiEndpoint + "/drive/v3/files?" + params.toString();
+  var res = await fetch(url, {
+    headers: {
+      Authorization: "Bearer " + googleDriveOptions.authToken
+    }
+  });
+  if (!res.ok) {
+    throw await (0, _rxError.newRxFetchError)(res);
+  }
+  var data = await res.json();
+  var files = data.files;
+  if (checkpoint) {
+    files = files.filter(file => !(file.modifiedTime === checkpoint.modifiedTime && checkpoint.docIdsWithSameModifiedTime.includes(file.name)));
+  }
+  files = files.slice(0, batchSize);
+  var first = files[0];
+  var newCheckpoint = checkpoint;
+  var last = (0, _index.lastOfArray)(files);
+  if (last) {
+    var lastModified = (0, _index.ensureNotFalsy)(last.modifiedTime);
+    var docIdsWithSameModifiedTime = files.filter(file => file.modifiedTime === lastModified).map(file => file.name);
+    if (checkpoint && first.modifiedTime === checkpoint.modifiedTime) {
+      docIdsWithSameModifiedTime = docIdsWithSameModifiedTime.concat(checkpoint.docIdsWithSameModifiedTime);
+    }
+    newCheckpoint = {
+      docIdsWithSameModifiedTime,
+      modifiedTime: lastModified
+    };
+  }
+  return {
+    checkpoint: newCheckpoint,
+    files
+  };
+}
+//# sourceMappingURL=downstream.js.map

package/dist/cjs/plugins/replication-google-drive/downstream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"downstream.js","names":["_rxError","require","_index","_documentHandling","fetchChanges","googleDriveOptions","init","checkpoint","batchSize","filesResult","fetchChangesFiles","contents","fetchDocumentContents","files","map","file","id","documents","ordered","queryParts","docsFolderId","push","modifiedTime","OVERFETCH_AMOUNT","params","URLSearchParams","q","join","pageSize","orderBy","fields","supportsAllDrives","includeItemsFromAllDrives","url","apiEndpoint","toString","res","fetch","headers","Authorization","authToken","ok","newRxFetchError","data","json","filter","docIdsWithSameModifiedTime","includes","name","slice","first","newCheckpoint","last","lastOfArray","lastModified","ensureNotFalsy","concat"],"sources":["../../../../src/plugins/replication-google-drive/downstream.ts"],"sourcesContent":["import { newRxFetchError } from '../../rx-error.ts';\r\nimport { ensureNotFalsy, lastOfArray } from '../utils/index.ts';\r\nimport { fetchDocumentContents } from './document-handling.ts';\r\nimport type {\r\n    DriveFileMetadata,\r\n    GoogleDriveCheckpointType,\r\n    GoogleDriveOptionsWithDefaults\r\n} from './google-drive-types';\r\nimport { DriveStructure } from './init.ts';\r\n\r\nexport async function fetchChanges<DocType>(\r\n    googleDriveOptions: GoogleDriveOptionsWithDefaults,\r\n    init: DriveStructure,\r\n    checkpoint?: GoogleDriveCheckpointType,\r\n    batchSize: number = 10\r\n): Promise<{\r\n    checkpoint: GoogleDriveCheckpointType | undefined,\r\n    documents: DocType[]\r\n}> {\r\n    const filesResult = await fetchChangesFiles(\r\n        googleDriveOptions,\r\n        init,\r\n        checkpoint,\r\n        batchSize\r\n    );\r\n\r\n    const contents = await fetchDocumentContents<DocType>(\r\n        googleDriveOptions,\r\n        filesResult.files.map(file => file.id)\r\n    );\r\n\r\n    return {\r\n        checkpoint: filesResult.checkpoint,\r\n        documents: contents.ordered\r\n    };\r\n}\r\nexport async function fetchChangesFiles(\r\n    googleDriveOptions: GoogleDriveOptionsWithDefaults,\r\n    init: DriveStructure,\r\n    checkpoint?: GoogleDriveCheckpointType,\r\n    batchSize: number = 10\r\n): Promise<{\r\n    checkpoint?: GoogleDriveCheckpointType,\r\n    files: DriveFileMetadata[]\r\n}> {\r\n\r\n    const queryParts = [\r\n        `'${init.docsFolderId}' in parents`,\r\n        `and trashed = false`\r\n    ];\r\n\r\n    if (checkpoint) {\r\n        queryParts.push(`and modifiedTime >= '${checkpoint.modifiedTime}'`);\r\n    }\r\n\r\n    /**\r\n     * Intentionally overfetch in case\r\n     * multiple docs have the same modifiedTime.\r\n     * We later have to strip the additional ones.\r\n     */\r\n    const OVERFETCH_AMOUNT = 6;\r\n    const params = new URLSearchParams({\r\n        q: queryParts.join(' '),\r\n        pageSize: (batchSize + OVERFETCH_AMOUNT) + '',\r\n        orderBy: \"modifiedTime asc,name asc\",\r\n        fields: \"files(id,name,mimeType,parents,modifiedTime,size)\",\r\n        supportsAllDrives: \"true\",\r\n        includeItemsFromAllDrives: \"true\",\r\n    });\r\n\r\n    const url =\r\n        googleDriveOptions.apiEndpoint +\r\n        \"/drive/v3/files?\" +\r\n        params.toString();\r\n\r\n    const res = await fetch(url, {\r\n        headers: {\r\n            Authorization: `Bearer ${googleDriveOptions.authToken}`,\r\n        },\r\n    });\r\n    if (!res.ok) {\r\n        throw await newRxFetchError(res);\r\n    }\r\n\r\n    const data: { files: DriveFileMetadata[] } = await res.json();\r\n\r\n    let files = data.files;\r\n    if (checkpoint) {\r\n        files = files\r\n            .filter(file => !(file.modifiedTime === checkpoint.modifiedTime && checkpoint.docIdsWithSameModifiedTime.includes(file.name)));\r\n    }\r\n    files = files.slice(0, batchSize);\r\n    const first = files[0];\r\n\r\n\r\n    let newCheckpoint = checkpoint;\r\n    const last = lastOfArray(files);\r\n    if (last) {\r\n        const lastModified = ensureNotFalsy(last.modifiedTime);\r\n        let docIdsWithSameModifiedTime = files\r\n            .filter(file => file.modifiedTime === lastModified)\r\n            .map(file => file.name);\r\n\r\n        if (checkpoint && first.modifiedTime === checkpoint.modifiedTime) {\r\n            docIdsWithSameModifiedTime = docIdsWithSameModifiedTime.concat(checkpoint.docIdsWithSameModifiedTime);\r\n        }\r\n\r\n        newCheckpoint = {\r\n            docIdsWithSameModifiedTime,\r\n            modifiedTime: lastModified\r\n        }\r\n    }\r\n\r\n    return {\r\n        checkpoint: newCheckpoint,\r\n        files\r\n    };\r\n}\r\n"],"mappings":";;;;;;;AAAA,IAAAA,QAAA,GAAAC,OAAA;AACA,IAAAC,MAAA,GAAAD,OAAA;AACA,IAAAE,iBAAA,GAAAF,OAAA;AAQO,eAAeG,YAAYA,CAC9BC,kBAAkD,EAClDC,IAAoB,EACpBC,UAAsC,EACtCC,SAAiB,GAAG,EAAE,EAIvB;EACC,IAAMC,WAAW,GAAG,MAAMC,iBAAiB,CACvCL,kBAAkB,EAClBC,IAAI,EACJC,UAAU,EACVC,SACJ,CAAC;EAED,IAAMG,QAAQ,GAAG,MAAM,IAAAC,uCAAqB,EACxCP,kBAAkB,EAClBI,WAAW,CAACI,KAAK,CAACC,GAAG,CAACC,IAAI,IAAIA,IAAI,CAACC,EAAE,CACzC,CAAC;EAED,OAAO;IACHT,UAAU,EAAEE,WAAW,CAACF,UAAU;IAClCU,SAAS,EAAEN,QAAQ,CAACO;EACxB,CAAC;AACL;AACO,eAAeR,iBAAiBA,CACnCL,kBAAkD,EAClDC,IAAoB,EACpBC,UAAsC,EACtCC,SAAiB,GAAG,EAAE,EAIvB;EAEC,IAAMW,UAAU,GAAG,OACXb,IAAI,CAACc,YAAY,yCAExB;EAED,IAAIb,UAAU,EAAE;IACZY,UAAU,CAACE,IAAI,2BAAyBd,UAAU,CAACe,YAAY,MAAG,CAAC;EACvE;;EAEA;AACJ;AACA;AACA;AACA;EACI,IAAMC,gBAAgB,GAAG,CAAC;EAC1B,IAAMC,MAAM,GAAG,IAAIC,eAAe,CAAC;IAC/BC,CAAC,EAAEP,UAAU,CAACQ,IAAI,CAAC,GAAG,CAAC;IACvBC,QAAQ,EAAGpB,SAAS,GAAGe,gBAAgB,GAAI,EAAE;IAC7CM,OAAO,EAAE,2BAA2B;IACpCC,MAAM,EAAE,mDAAmD;IAC3DC,iBAAiB,EAAE,MAAM;IACzBC,yBAAyB,EAAE;EAC/B,CAAC,CAAC;EAEF,IAAMC,GAAG,GACL5B,kBAAkB,CAAC6B,WAAW,GAC9B,kBAAkB,GAClBV,MAAM,CAACW,QAAQ,CAAC,CAAC;EAErB,IAAMC,GAAG,GAAG,MAAMC,KAAK,CAACJ,GAAG,EAAE;IACzBK,OAAO,EAAE;MACLC,aAAa,cAAYlC,kBAAkB,CAACmC;IAChD;EACJ,CAAC,CAAC;EACF,IAAI,CAACJ,GAAG,CAACK,EAAE,EAAE;IACT,MAAM,MAAM,IAAAC,wBAAe,EAACN,GAAG,CAAC;EACpC;EAEA,IAAMO,IAAoC,GAAG,MAAMP,GAAG,CAACQ,IAAI,CAAC,CAAC;EAE7D,IAAI/B,KAAK,GAAG8B,IAAI,CAAC9B,KAAK;EACtB,IAAIN,UAAU,EAAE;IACZM,KAAK,GAAGA,KAAK,CACRgC,MAAM,CAAC9B,IAAI,IAAI,EAAEA,IAAI,CAACO,YAAY,KAAKf,UAAU,CAACe,YAAY,IAAIf,UAAU,CAACuC,0BAA0B,CAACC,QAAQ,CAAChC,IAAI,CAACiC,IAAI,CAAC,CAAC,CAAC;EACtI;EACAnC,KAAK,GAAGA,KAAK,CAACoC,KAAK,CAAC,CAAC,EAAEzC,SAAS,CAAC;EACjC,IAAM0C,KAAK,GAAGrC,KAAK,CAAC,CAAC,CAAC;EAGtB,IAAIsC,aAAa,GAAG5C,UAAU;EAC9B,IAAM6C,IAAI,GAAG,IAAAC,kBAAW,EAACxC,KAAK,CAAC;EAC/B,IAAIuC,IAAI,EAAE;IACN,IAAME,YAAY,GAAG,IAAAC,qBAAc,EAACH,IAAI,CAAC9B,YAAY,CAAC;IACtD,IAAIwB,0BAA0B,GAAGjC,KAAK,CACjCgC,MAAM,CAAC9B,IAAI,IAAIA,IAAI,CAACO,YAAY,KAAKgC,YAAY,CAAC,CAClDxC,GAAG,CAACC,IAAI,IAAIA,IAAI,CAACiC,IAAI,CAAC;IAE3B,IAAIzC,UAAU,IAAI2C,KAAK,CAAC5B,YAAY,KAAKf,UAAU,CAACe,YAAY,EAAE;MAC9DwB,0BAA0B,GAAGA,0BAA0B,CAACU,MAAM,CAACjD,UAAU,CAACuC,0BAA0B,CAAC;IACzG;IAEAK,aAAa,GAAG;MACZL,0BAA0B;MAC1BxB,YAAY,EAAEgC;IAClB,CAAC;EACL;EAEA,OAAO;IACH/C,UAAU,EAAE4C,aAAa;IACzBtC;EACJ,CAAC;AACL","ignoreList":[]}