@omegup/msync 0.1.28 → 0.1.30

package/index.esm.js

@@ -1,4 +1,4 @@
-import { Timestamp, UUID, MongoClient } from 'mongodb';
+import { UUID, Timestamp, MongoClient } from 'mongodb';
 import crypto$1 from 'crypto';
 import { canonicalize } from 'json-canonicalize';
 import { SynchronousPromise } from 'synchronous-promise';
@@ -82,7 +82,7 @@ const val = (val) => asExpr({
         : val),
 });
 const current = asExpr({
-    raw: () => asExprRaw(new Timestamp(0xffffffffffffffffn)),
+    raw: () => asExprRaw('$$CLUSTER_TIME'),
 });
 const $let = (vars, inExpr) => asExpr({
     raw: f => asExprRaw({
@@ -918,6 +918,7 @@ const subMerge = (args, out, gid, extra, idPrefix, first) => {
     };
     const addTSAndExtra = {
         ...mapExact0(e, to),
+        ...(out.whenNotMatched === 'insert' ? { deletedAt: ['deletedAt', to(nil)] } : {}),
         touchedAt: ['touchedAt', to(current)],
     };
     const updater = set()(addTSAndExtra);
@@ -1280,7 +1281,8 @@ const createIndexWithRetry = async (collection, indexSpec, options) => {
         }
         catch (e) {
             if ([85, 276].includes(e.code)) {
-                log('Index created with different name', e.code, { collection: collection.collectionName});
+                log('Index created with different name', e.code, {
+                    collection: collection.collectionName});
                 break;
             }
             if (e.code == 12587) {
@@ -1295,6 +1297,11 @@ const createIndexWithRetry = async (collection, indexSpec, options) => {
         break;
     }
 };
+const ensureCollection = async (db, collectionName) => {
+    if (!(await db.listCollections({ name: collectionName }, { nameOnly: true }).next())) {
+        await db.createCollection(collectionName);
+    }
+};
 
 const patch = ({ ...x }, k, v) => {
     delete x[k];
@@ -1600,7 +1607,6 @@ const addTeardown = (it, tr) => {
 
 const sleep = (ms) => new Promise(r => setTimeout(r, ms));
 const maxTimestamp = new Timestamp(0xffffffffffffffffn);
-const isMax = (x) => x instanceof Timestamp && x.equals(maxTimestamp);
 const getCurrentTimestamp = async (db) => {
     const adminDb = db.admin();
     const serverStatus = await adminDb.command({ serverStatus: 1 });
@@ -1623,9 +1629,9 @@ async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000
         await sleep(pollMs);
     }
 }
-async function* tailOplog(db, opts) {
-    let lastTs = opts.since ?? (await getCurrentTimestamp(db));
-    const reopenDelayMs = opts.reopenDelayMs ?? 250;
+async function* tailOplog(db) {
+    let lastTs = await getCurrentTimestamp(db);
+    const reopenDelayMs = 250;
     const coll = db.client.db('local').collection('oplog.rs');
     while (true) {
         const cursor = coll.find({
@@ -1639,14 +1645,12 @@ async function* tailOplog(db, opts) {
         });
         try {
             for await (const doc of cursor) {
-                lastTs = doc.ts;
                 if (doc.op === 'i' || '_id' in doc.o) {
                     const fields = new Set(Object.keys(doc.o));
                     fields.delete('_id');
-                    yield { fields, doc, changeTouched: isMax(doc.o['touchedAt']) };
+                    yield { fields, doc };
                 }
                 else {
-                    let changeTouched = false;
                     if (doc.o['$v'] !== 2) {
                         throw new Error(`Expected update with $v: 2, got ${JSON.stringify(doc)}`);
                     }
@@ -1655,21 +1659,19 @@ async function* tailOplog(db, opts) {
                     for (const updateOp in diff) {
                         if (['u', 'i', 'd'].includes(updateOp)) {
                             updatedFields.push(...Object.keys(diff[updateOp]));
-                            if (isMax(diff[updateOp]['touchedAt'])) {
-                                changeTouched = true;
-                            }
                         }
                         else if (updateOp.startsWith('s')) {
                             updatedFields.push(updateOp.slice(1));
                         }
                     }
-                    yield { fields: new Set(updatedFields), doc, changeTouched };
+                    yield { fields: new Set(updatedFields), doc };
                 }
             }
         }
         catch (e) {
             log('oplog loop error, notifying watchers and reopening');
             console.error(e);
+            lastTs = await getCurrentTimestamp(db);
             yield null;
         }
         finally {
@@ -1691,7 +1693,7 @@ const loop = async (db) => {
     let notify = makePromise();
     let batch = [];
     const run = async () => {
-        for await (const event of tailOplog(db, {})) {
+        for await (const event of tailOplog(db)) {
             if (event?.fields.size === 0)
                 continue;
             batch = event && batch ? [...batch, event] : null;
@@ -1718,23 +1720,6 @@ const loop = async (db) => {
             }
             continue;
         }
-        const groups = Object.groupBy(events.filter(e => e.changeTouched), ev => ev.doc.ns);
-        for (const [ns, evs] of Object.entries(groups)) {
-            if (!evs)
-                continue;
-            const [dbName, collName] = ns.split('.');
-            if (dbName !== db.databaseName)
-                continue;
-            const coll = db.collection(collName);
-            coll
-                .bulkWrite(evs.map((e) => ({
-                updateOne: {
-                    filter: { _id: e.doc.o['_id'] ?? e.doc.o2?._id },
-                    update: { $set: { touchedAt: e.doc.ts } },
-                },
-            })))
-                .catch(() => { });
-        }
         for (const { fields, doc } of events) {
             const m = watchers.get(doc.ns);
             if (!m)
@@ -1802,6 +1787,7 @@ const actions = {
     ],
 };
 
+const previous = (ts) => new Timestamp({ t: ts.high - 60, i: 0 });
 const getFirstStages = (view, needs) => {
     const { projection, hardMatch: pre, match } = view;
     const projectInput = projection && $project_(spread(projection, {
@@ -1812,7 +1798,7 @@ const getFirstStages = (view, needs) => {
     const hardMatch = removeNotYetSynchronizedFields ? $and(pre, ...removeNotYetSynchronizedFields) : pre;
     const firstStages = (lastTS, keepNulls = false) => {
         const hardQuery = $and(lastTS
-            ? root().of('touchedAt').has($gtTs(lastTS.ts))
+            ? root().of('touchedAt').has($gtTs(previous(lastTS.ts)))
             : root().of('deletedAt').has($eq(null)), lastTS ? null : match && $expr(match), keepNulls ? pre : hardMatch);
         const ln = link()
             .with($match_(hardQuery));
@@ -1967,6 +1953,7 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
                 partialFilterExpression: { updated: true, after: null, before: null },
                 name: 'updated_nulls_' + new UUID().toString('base64'),
             });
+            await ensureCollection(db, coll);
             await db.command({
                 collMod: coll,
                 changeStreamPreAndPostImages: { enabled: true },
@@ -2118,7 +2105,7 @@ const executes$1 = (view, input, streamName, needs) => {
         }));
     const notDeleted = root().of('deletedAt').has($eq(null));
     const stages = (lastTS) => {
-        const hardQuery = $and(lastTS && root().of('touchedAt').has($gteTs(lastTS.ts)), hardMatch, notDeleted, match && $expr(match));
+        const hardQuery = $and(lastTS && root().of('touchedAt').has($gteTs(previous(lastTS.ts))), hardMatch, notDeleted, match && $expr(match));
         const ln = link().with($match_(hardQuery));
         return (projectInput ? ln.with(projectInput) : ln).with(input);
     };
@@ -2148,6 +2135,7 @@ const executes$1 = (view, input, streamName, needs) => {
         const stop = withStop(step0);
         const step1 = async () => {
             log('creating indexes');
+            await ensureCollection(db, coll);
             await db.command({
                 collMod: coll,
                 changeStreamPreAndPostImages: { enabled: true },

package/index.js

@@ -84,7 +84,7 @@ const val = (val) => asExpr({
         : val),
 });
 const current = asExpr({
-    raw: () => asExprRaw(new mongodb.Timestamp(0xffffffffffffffffn)),
+    raw: () => asExprRaw('$$CLUSTER_TIME'),
 });
 const $let = (vars, inExpr) => asExpr({
     raw: f => asExprRaw({
@@ -920,6 +920,7 @@ const subMerge = (args, out, gid, extra, idPrefix, first) => {
     };
     const addTSAndExtra = {
         ...mapExact0(e, to),
+        ...(out.whenNotMatched === 'insert' ? { deletedAt: ['deletedAt', to(nil)] } : {}),
         touchedAt: ['touchedAt', to(current)],
     };
     const updater = set()(addTSAndExtra);
@@ -1282,7 +1283,8 @@ const createIndexWithRetry = async (collection, indexSpec, options) => {
         }
         catch (e) {
             if ([85, 276].includes(e.code)) {
-                log('Index created with different name', e.code, { collection: collection.collectionName});
+                log('Index created with different name', e.code, {
+                    collection: collection.collectionName});
                 break;
             }
             if (e.code == 12587) {
@@ -1297,6 +1299,11 @@ const createIndexWithRetry = async (collection, indexSpec, options) => {
         break;
     }
 };
+const ensureCollection = async (db, collectionName) => {
+    if (!(await db.listCollections({ name: collectionName }, { nameOnly: true }).next())) {
+        await db.createCollection(collectionName);
+    }
+};
 
 const patch = ({ ...x }, k, v) => {
     delete x[k];
@@ -1602,7 +1609,6 @@ const addTeardown = (it, tr) => {
 
 const sleep = (ms) => new Promise(r => setTimeout(r, ms));
 const maxTimestamp = new mongodb.Timestamp(0xffffffffffffffffn);
-const isMax = (x) => x instanceof mongodb.Timestamp && x.equals(maxTimestamp);
 const getCurrentTimestamp = async (db) => {
     const adminDb = db.admin();
     const serverStatus = await adminDb.command({ serverStatus: 1 });
@@ -1625,9 +1631,9 @@ async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000
         await sleep(pollMs);
     }
 }
-async function* tailOplog(db, opts) {
-    let lastTs = opts.since ?? (await getCurrentTimestamp(db));
-    const reopenDelayMs = opts.reopenDelayMs ?? 250;
+async function* tailOplog(db) {
+    let lastTs = await getCurrentTimestamp(db);
+    const reopenDelayMs = 250;
     const coll = db.client.db('local').collection('oplog.rs');
     while (true) {
         const cursor = coll.find({
@@ -1641,14 +1647,12 @@ async function* tailOplog(db, opts) {
         });
         try {
             for await (const doc of cursor) {
-                lastTs = doc.ts;
                 if (doc.op === 'i' || '_id' in doc.o) {
                     const fields = new Set(Object.keys(doc.o));
                     fields.delete('_id');
-                    yield { fields, doc, changeTouched: isMax(doc.o['touchedAt']) };
+                    yield { fields, doc };
                 }
                 else {
-                    let changeTouched = false;
                     if (doc.o['$v'] !== 2) {
                         throw new Error(`Expected update with $v: 2, got ${JSON.stringify(doc)}`);
                     }
@@ -1657,21 +1661,19 @@ async function* tailOplog(db, opts) {
                     for (const updateOp in diff) {
                         if (['u', 'i', 'd'].includes(updateOp)) {
                             updatedFields.push(...Object.keys(diff[updateOp]));
-                            if (isMax(diff[updateOp]['touchedAt'])) {
-                                changeTouched = true;
-                            }
                         }
                         else if (updateOp.startsWith('s')) {
                             updatedFields.push(updateOp.slice(1));
                         }
                     }
-                    yield { fields: new Set(updatedFields), doc, changeTouched };
+                    yield { fields: new Set(updatedFields), doc };
                 }
             }
         }
         catch (e) {
             log('oplog loop error, notifying watchers and reopening');
             console.error(e);
+            lastTs = await getCurrentTimestamp(db);
             yield null;
         }
         finally {
@@ -1693,7 +1695,7 @@ const loop = async (db) => {
     let notify = makePromise();
     let batch = [];
     const run = async () => {
-        for await (const event of tailOplog(db, {})) {
+        for await (const event of tailOplog(db)) {
             if (event?.fields.size === 0)
                 continue;
             batch = event && batch ? [...batch, event] : null;
@@ -1720,23 +1722,6 @@ const loop = async (db) => {
             }
             continue;
         }
-        const groups = Object.groupBy(events.filter(e => e.changeTouched), ev => ev.doc.ns);
-        for (const [ns, evs] of Object.entries(groups)) {
-            if (!evs)
-                continue;
-            const [dbName, collName] = ns.split('.');
-            if (dbName !== db.databaseName)
-                continue;
-            const coll = db.collection(collName);
-            coll
-                .bulkWrite(evs.map((e) => ({
-                updateOne: {
-                    filter: { _id: e.doc.o['_id'] ?? e.doc.o2?._id },
-                    update: { $set: { touchedAt: e.doc.ts } },
-                },
-            })))
-                .catch(() => { });
-        }
         for (const { fields, doc } of events) {
             const m = watchers.get(doc.ns);
             if (!m)
@@ -1804,6 +1789,7 @@ const actions = {
     ],
 };
 
+const previous = (ts) => new mongodb.Timestamp({ t: ts.high - 60, i: 0 });
 const getFirstStages = (view, needs) => {
     const { projection, hardMatch: pre, match } = view;
     const projectInput = projection && $project_(spread(projection, {
@@ -1814,7 +1800,7 @@ const getFirstStages = (view, needs) => {
     const hardMatch = removeNotYetSynchronizedFields ? $and(pre, ...removeNotYetSynchronizedFields) : pre;
     const firstStages = (lastTS, keepNulls = false) => {
         const hardQuery = $and(lastTS
-            ? root().of('touchedAt').has($gtTs(lastTS.ts))
+            ? root().of('touchedAt').has($gtTs(previous(lastTS.ts)))
             : root().of('deletedAt').has($eq(null)), lastTS ? null : match && $expr(match), keepNulls ? pre : hardMatch);
         const ln = link()
             .with($match_(hardQuery));
@@ -1969,6 +1955,7 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
                 partialFilterExpression: { updated: true, after: null, before: null },
                 name: 'updated_nulls_' + new mongodb.UUID().toString('base64'),
             });
+            await ensureCollection(db, coll);
             await db.command({
                 collMod: coll,
                 changeStreamPreAndPostImages: { enabled: true },
@@ -2120,7 +2107,7 @@ const executes$1 = (view, input, streamName, needs) => {
         }));
     const notDeleted = root().of('deletedAt').has($eq(null));
     const stages = (lastTS) => {
-        const hardQuery = $and(lastTS && root().of('touchedAt').has($gteTs(lastTS.ts)), hardMatch, notDeleted, match && $expr(match));
+        const hardQuery = $and(lastTS && root().of('touchedAt').has($gteTs(previous(lastTS.ts))), hardMatch, notDeleted, match && $expr(match));
         const ln = link().with($match_(hardQuery));
         return (projectInput ? ln.with(projectInput) : ln).with(input);
     };
@@ -2150,6 +2137,7 @@ const executes$1 = (view, input, streamName, needs) => {
         const stop = withStop(step0);
         const step1 = async () => {
             log('creating indexes');
+            await ensureCollection(db, coll);
             await db.command({
                 collMod: coll,
                 changeStreamPreAndPostImages: { enabled: true },

package/package.json

@@ -3,7 +3,7 @@
     "module": "index.esm.js",
     "typings": "index.d.ts",
     "name": "@omegup/msync",
-    "version": "0.1.28",
+    "version": "0.1.30",
     "dependencies": {
         "dayjs": "^1.11.9",
         "dotenv": "^16.3.1",