@omegup/msync 0.1.13 → 0.1.15

package/index.esm.js

@@ -829,12 +829,10 @@ const getWhenMatchedForMerge = (whenNotMatched) => {
 };
 const getWhenMatched = (whenNotMatched) => {
     const orNull = (e) => whenNotMatched === 'discard' ? $ifNull(e, nil) : e;
-    const preMergeOld = whenNotMatched !== 'fail'
-        ? mergeObjects(root().of('old').expr(), field({
-            deletedAt: ['deletedAt', orNull(root().of('old').of('deletedAt').expr())],
-            touchedAt: ['touchedAt', root().of('merged').of('touchedAt').expr()],
-        }))
-        : root().of('old').expr();
+    const preMergeOld = mergeObjects(root().of('old').expr(), field({
+        deletedAt: ['deletedAt', orNull(root().of('old').of('deletedAt').expr())],
+        touchedAt: ['touchedAt', root().of('merged').of('touchedAt').expr()],
+    }));
     const same = eq(preMergeOld)(root().of('merged').expr());
     return link().with($replaceWith_(ite(same, root().of('old').expr(), root().of('merged').expr()))).stages;
 };
@@ -918,8 +916,11 @@ const subMerge = (args, out, gid, extra, idPrefix, first) => {
         ...F1,
         ...F2,
     };
-    const addExtra = mapExact0(e, to);
-    const updater = set()(addExtra);
+    const addTSAndExtra = {
+        ...mapExact0(e, to),
+        touchedAt: ['touchedAt', to(current)],
+    };
+    const updater = set()(addTSAndExtra);
     const whenMatched = getWhenMatched(out.whenNotMatched);
     return link()
         .with($set_(set()(addExtraAndMerge)))
@@ -1760,16 +1761,23 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
         const step4 = ({ result, ts }) => async () => {
             const start = Date.now();
             await snapshotCollection.updateMany({ before: null }, { $set: { before: null } });
+            const stages = finalInput.raw(ts === undefined);
             const aggResult = await aggregate(streamName, c => c({
                 coll: snapshotCollection,
                 input: link()
                     .with($match_(root().of('updated').has($eq(true))))
-                    .with($match_($expr(ne(root().of('after').expr())(root().of('before').expr()))))
                     .with(input.delta)
-                    .with(finalInput.raw(ts === undefined)).stages,
+                    .with(stages).stages,
             }), false, start);
             const stream = makeStream(result.cursor.atClusterTime);
-            return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream }), 'remove handled deleted updated', () => stream.close());
+            const nextRes = stream.tryNext();
+            const intoColl = stages.at(-1).$merge.into.coll;
+            const startx = Date.now();
+            await db
+                .collection(intoColl)
+                .countDocuments({ touchedAt: { $gte: result.cursor.atClusterTime } })
+                .then(count => log(`documents updated ${intoColl}`, count, 'took', Date.now() - startx));
+            return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream, nextRes }), 'remove handled deleted updated', () => stream.close());
         };
         const step5 = (l) => async () => {
             log(`remove handled deleted updated db['${snapshotCollection.collectionName}'].deleteMany({ updated: true, after: null })`);
@@ -1801,13 +1809,14 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
             return step8(l);
         };
         const step8 = (l) => {
-            return nextData(l.aggResult.cursor.firstBatch)(() => l.stream
-                .tryNext()
+            return nextData(l.aggResult.cursor.firstBatch)(() => l.nextRes
                 .catch((err) => {
                 log('restarting', err);
                 return { ts: null };
             })
-                .then(doc => (doc ? next(step3({ _id: streamName, ts: l.ts }), 'restart') : step8(l))), 'wait for change');
+                .then(doc => doc
+                ? next(step3({ _id: streamName, ts: l.ts }), 'restart')
+                : step8({ ...l, nextRes: l.stream.tryNext() })), 'wait for change');
         };
         return skip
             ? withStop(() => SynchronousPromise.resolve(next(step3(null), 'clone into new collection')))
@@ -1835,8 +1844,11 @@ const executes$1 = (view, input, streamName, needs) => {
     else if (streamNames[streamName] != hash)
         throw new Error('streamName already used');
     const { collection, projection, hardMatch: pre, match } = view;
-    const removeNotYetSynchronizedFields = projection && Object.values(mapExactToObject(projection, (_, k) => (needs[k] ?? k.startsWith('_')) ? root().of(k).has($exists(true)) : null));
-    const hardMatch = removeNotYetSynchronizedFields ? $and(pre, ...removeNotYetSynchronizedFields) : pre;
+    const removeNotYetSynchronizedFields = projection &&
+        Object.values(mapExactToObject(projection, (_, k) => (needs[k] ?? k.startsWith('_')) ? root().of(k).has($exists(true)) : null));
+    const hardMatch = removeNotYetSynchronizedFields
+        ? $and(pre, ...removeNotYetSynchronizedFields)
+        : pre;
     const job = {};
     const db = collection.s.db, coll = collection.collectionName;
     db.command({
@@ -1848,15 +1860,15 @@ const executes$1 = (view, input, streamName, needs) => {
         name: 'touchedAt_' + new UUID().toString('base64'),
     });
     const last = db.collection('__last');
-    const projectInput = projection && $project_(spread(projection, {
-        deletedAt: ['deletedAt', 1],
-        _id: ['_id', 1],
-    }));
+    const projectInput = projection &&
+        $project_(spread(projection, {
+            deletedAt: ['deletedAt', 1],
+            _id: ['_id', 1],
+        }));
     const notDeleted = root().of('deletedAt').has($eq(null));
     const stages = (lastTS) => {
         const hardQuery = $and(lastTS && root().of('touchedAt').has($gteTs(lastTS.ts)), hardMatch, notDeleted, match && $expr(match));
-        const ln = link()
-            .with($match_(hardQuery));
+        const ln = link().with($match_(hardQuery));
         return (projectInput ? ln.with(projectInput) : ln).with(input);
     };
     const run = (finalInput) => {
@@ -1907,20 +1919,22 @@ const executes$1 = (view, input, streamName, needs) => {
             return {
                 cont: withStop(async () => {
                     await new Promise(resolve => setTimeout(resolve, 1000));
-                    return step4(same)();
+                    return next(step4(same), 'clone into new collection');
                 }),
                 data: [],
-                info: { debug: 'clone into new collection', job: undefined },
+                info: { debug: 'wait for clone into new collection', job: undefined },
             };
         };
         const makeStream = (startAt) => makeWatchStream(db, view, startAt, streamName);
         const step4 = (lastTS) => async () => {
+            const raw = stages(lastTS).with(finalInput.raw(lastTS === null)).stages;
             const aggResult = await aggregate(streamName, c => c({
                 coll: collection,
-                input: stages(lastTS).with(finalInput.raw(lastTS === null)).stages,
+                input: raw,
             }));
             const stream = makeStream(aggResult.cursor.atClusterTime);
-            return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream }), 'update __last', () => stream.close());
+            const nextRes = stream.tryNext();
+            return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream, nextRes }), 'update __last', () => stream.close());
         };
         const step7 = (l) => async () => {
             await last.updateOne({ _id: streamName }, { $set: { ts: l.ts, data } }, { upsert: true });
@@ -1930,13 +1944,14 @@ const executes$1 = (view, input, streamName, needs) => {
             return {
                 data: l.aggResult.cursor.firstBatch,
                 info: { job: undefined, debug: 'wait for change' },
-                cont: withStop(() => l.stream
-                    .tryNext()
+                cont: withStop(() => l.nextRes
                     .catch((err) => {
                     log('restarting', err);
                     return { ts: null };
                 })
-                    .then(doc => (doc ? next(step4({ _id: streamName, ts: l.ts }), 'restart') : step8(l)))),
+                    .then(doc => doc
+                    ? next(step4({ _id: streamName, ts: l.ts }), 'restart')
+                    : step8({ ...l, nextRes: l.stream.tryNext() }))),
             };
         };
         return stop;

package/index.js

@@ -831,12 +831,10 @@ const getWhenMatchedForMerge = (whenNotMatched) => {
 };
 const getWhenMatched = (whenNotMatched) => {
     const orNull = (e) => whenNotMatched === 'discard' ? $ifNull(e, nil) : e;
-    const preMergeOld = whenNotMatched !== 'fail'
-        ? mergeObjects(root().of('old').expr(), field({
-            deletedAt: ['deletedAt', orNull(root().of('old').of('deletedAt').expr())],
-            touchedAt: ['touchedAt', root().of('merged').of('touchedAt').expr()],
-        }))
-        : root().of('old').expr();
+    const preMergeOld = mergeObjects(root().of('old').expr(), field({
+        deletedAt: ['deletedAt', orNull(root().of('old').of('deletedAt').expr())],
+        touchedAt: ['touchedAt', root().of('merged').of('touchedAt').expr()],
+    }));
     const same = eq(preMergeOld)(root().of('merged').expr());
     return link().with($replaceWith_(ite(same, root().of('old').expr(), root().of('merged').expr()))).stages;
 };
@@ -920,8 +918,11 @@ const subMerge = (args, out, gid, extra, idPrefix, first) => {
         ...F1,
         ...F2,
     };
-    const addExtra = mapExact0(e, to);
-    const updater = set()(addExtra);
+    const addTSAndExtra = {
+        ...mapExact0(e, to),
+        touchedAt: ['touchedAt', to(current)],
+    };
+    const updater = set()(addTSAndExtra);
     const whenMatched = getWhenMatched(out.whenNotMatched);
     return link()
         .with($set_(set()(addExtraAndMerge)))
@@ -1762,16 +1763,23 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
         const step4 = ({ result, ts }) => async () => {
             const start = Date.now();
             await snapshotCollection.updateMany({ before: null }, { $set: { before: null } });
+            const stages = finalInput.raw(ts === undefined);
             const aggResult = await aggregate(streamName, c => c({
                 coll: snapshotCollection,
                 input: link()
                     .with($match_(root().of('updated').has($eq(true))))
-                    .with($match_($expr(ne(root().of('after').expr())(root().of('before').expr()))))
                     .with(input.delta)
-                    .with(finalInput.raw(ts === undefined)).stages,
+                    .with(stages).stages,
             }), false, start);
             const stream = makeStream(result.cursor.atClusterTime);
-            return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream }), 'remove handled deleted updated', () => stream.close());
+            const nextRes = stream.tryNext();
+            const intoColl = stages.at(-1).$merge.into.coll;
+            const startx = Date.now();
+            await db
+                .collection(intoColl)
+                .countDocuments({ touchedAt: { $gte: result.cursor.atClusterTime } })
+                .then(count => log(`documents updated ${intoColl}`, count, 'took', Date.now() - startx));
+            return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream, nextRes }), 'remove handled deleted updated', () => stream.close());
         };
         const step5 = (l) => async () => {
             log(`remove handled deleted updated db['${snapshotCollection.collectionName}'].deleteMany({ updated: true, after: null })`);
@@ -1803,13 +1811,14 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
             return step8(l);
         };
         const step8 = (l) => {
-            return nextData(l.aggResult.cursor.firstBatch)(() => l.stream
-                .tryNext()
+            return nextData(l.aggResult.cursor.firstBatch)(() => l.nextRes
                 .catch((err) => {
                 log('restarting', err);
                 return { ts: null };
             })
-                .then(doc => (doc ? next(step3({ _id: streamName, ts: l.ts }), 'restart') : step8(l))), 'wait for change');
+                .then(doc => doc
+                ? next(step3({ _id: streamName, ts: l.ts }), 'restart')
+                : step8({ ...l, nextRes: l.stream.tryNext() })), 'wait for change');
         };
         return skip
             ? withStop(() => synchronousPromise.SynchronousPromise.resolve(next(step3(null), 'clone into new collection')))
@@ -1837,8 +1846,11 @@ const executes$1 = (view, input, streamName, needs) => {
     else if (streamNames[streamName] != hash)
         throw new Error('streamName already used');
     const { collection, projection, hardMatch: pre, match } = view;
-    const removeNotYetSynchronizedFields = projection && Object.values(mapExactToObject(projection, (_, k) => (needs[k] ?? k.startsWith('_')) ? root().of(k).has($exists(true)) : null));
-    const hardMatch = removeNotYetSynchronizedFields ? $and(pre, ...removeNotYetSynchronizedFields) : pre;
+    const removeNotYetSynchronizedFields = projection &&
+        Object.values(mapExactToObject(projection, (_, k) => (needs[k] ?? k.startsWith('_')) ? root().of(k).has($exists(true)) : null));
+    const hardMatch = removeNotYetSynchronizedFields
+        ? $and(pre, ...removeNotYetSynchronizedFields)
+        : pre;
     const job = {};
     const db = collection.s.db, coll = collection.collectionName;
     db.command({
@@ -1850,15 +1862,15 @@ const executes$1 = (view, input, streamName, needs) => {
         name: 'touchedAt_' + new mongodb.UUID().toString('base64'),
     });
     const last = db.collection('__last');
-    const projectInput = projection && $project_(spread(projection, {
-        deletedAt: ['deletedAt', 1],
-        _id: ['_id', 1],
-    }));
+    const projectInput = projection &&
+        $project_(spread(projection, {
+            deletedAt: ['deletedAt', 1],
+            _id: ['_id', 1],
+        }));
     const notDeleted = root().of('deletedAt').has($eq(null));
     const stages = (lastTS) => {
         const hardQuery = $and(lastTS && root().of('touchedAt').has($gteTs(lastTS.ts)), hardMatch, notDeleted, match && $expr(match));
-        const ln = link()
-            .with($match_(hardQuery));
+        const ln = link().with($match_(hardQuery));
         return (projectInput ? ln.with(projectInput) : ln).with(input);
     };
     const run = (finalInput) => {
@@ -1909,20 +1921,22 @@ const executes$1 = (view, input, streamName, needs) => {
             return {
                 cont: withStop(async () => {
                     await new Promise(resolve => setTimeout(resolve, 1000));
-                    return step4(same)();
+                    return next(step4(same), 'clone into new collection');
                 }),
                 data: [],
-                info: { debug: 'clone into new collection', job: undefined },
+                info: { debug: 'wait for clone into new collection', job: undefined },
             };
         };
         const makeStream = (startAt) => makeWatchStream(db, view, startAt, streamName);
         const step4 = (lastTS) => async () => {
+            const raw = stages(lastTS).with(finalInput.raw(lastTS === null)).stages;
             const aggResult = await aggregate(streamName, c => c({
                 coll: collection,
-                input: stages(lastTS).with(finalInput.raw(lastTS === null)).stages,
+                input: raw,
             }));
             const stream = makeStream(aggResult.cursor.atClusterTime);
-            return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream }), 'update __last', () => stream.close());
+            const nextRes = stream.tryNext();
+            return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream, nextRes }), 'update __last', () => stream.close());
         };
         const step7 = (l) => async () => {
             await last.updateOne({ _id: streamName }, { $set: { ts: l.ts, data } }, { upsert: true });
@@ -1932,13 +1946,14 @@ const executes$1 = (view, input, streamName, needs) => {
             return {
                 data: l.aggResult.cursor.firstBatch,
                 info: { job: undefined, debug: 'wait for change' },
-                cont: withStop(() => l.stream
-                    .tryNext()
+                cont: withStop(() => l.nextRes
                     .catch((err) => {
                     log('restarting', err);
                     return { ts: null };
                 })
-                    .then(doc => (doc ? next(step4({ _id: streamName, ts: l.ts }), 'restart') : step8(l)))),
+                    .then(doc => doc
+                    ? next(step4({ _id: streamName, ts: l.ts }), 'restart')
+                    : step8({ ...l, nextRes: l.stream.tryNext() }))),
             };
         };
         return stop;

package/package.json

@@ -3,7 +3,7 @@
     "module": "index.esm.js",
     "typings": "index.d.ts",
     "name": "@omegup/msync",
-    "version": "0.1.13",
+    "version": "0.1.15",
     "dependencies": {
         "dayjs": "^1.11.9",
         "dotenv": "^16.3.1",