@omegup/msync 0.1.14 → 0.1.16

package/index.esm.js

@@ -1331,13 +1331,50 @@ const mergeIterators = (params) => {
     };
 };
 
+const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
+const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
+const json = (a) => replace(JSON.stringify(a));
+const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
+
+const state = { steady: false };
+let timeout = null;
+const aggregate = (streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
+    const req = {
+        aggregate: coll.collectionName,
+        pipeline: input,
+        cursor: {},
+        ...(snapshot && { readConcern: { level: 'snapshot' } }),
+    };
+    if (timeout !== null) {
+        clearTimeout(timeout);
+        timeout = null;
+    }
+    log('exec', streamName, req);
+    return coll.s.db.command(req).then(result => {
+        const r = result;
+        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('$$CLUSTER_TIME', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
+        if (!state.steady) {
+            if (timeout !== null)
+                throw new Error('timeout should be null');
+            timeout = setTimeout(() => {
+                state.steady = true;
+                console.log('steady');
+            }, 1000);
+        }
+        return r;
+    }, err => {
+        log('err', req, err);
+        throw new Error(err);
+    });
+});
+
 const firstWorksMerge = (iters) => {
     const iterator = () => {
         const results = iters.map(iter => iter());
         const sources = { ...results };
         return mergeIterators({
             sources,
-            interrupt: key => false,
+            interrupt: key => state.steady
         });
     };
     return iterator;
@@ -1501,29 +1538,6 @@ const $insertPart = (out, ext) => {
 const $insert = (out) => $insertPart(out, {});
 const assertNotNull = (expr) => expr;
 
-const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
-const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
-const json = (a) => replace(JSON.stringify(a));
-const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
-
-const aggregate = (streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
-    const req = {
-        aggregate: coll.collectionName,
-        pipeline: input,
-        cursor: {},
-        ...(snapshot && { readConcern: { level: 'snapshot' } }),
-    };
-    log('exec', streamName, req);
-    return coll.s.db.command(req).then(result => {
-        const r = result;
-        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('$$CLUSTER_TIME', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
-        return r;
-    }, err => {
-        log('err', req, err);
-        throw new Error(err);
-    });
-});
-
 const addTeardown = (it, tr) => {
     if (!tr)
         return it;
@@ -1755,22 +1769,32 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
             })).stages;
             const r = await aggregate(streamName, c => c({ coll: collection, input: cloneIntoNew }));
             await snapshotCollection.deleteMany({ updated: true, after: null, before: null });
+            const start = Date.now();
+            const count = await snapshotCollection.countDocuments({ updated: true });
+            log(streamName, `documents to update ${snapshotCollection.collectionName}`, count, 'took', Date.now() - start);
             return next(step4({ result: r, ts: lastTS?.ts }), 'run the aggregation');
         };
         const makeStream = (startAt) => makeWatchStream(db, view, startAt, streamName);
         const step4 = ({ result, ts }) => async () => {
             const start = Date.now();
             await snapshotCollection.updateMany({ before: null }, { $set: { before: null } });
+            const stages = finalInput.raw(ts === undefined);
             const aggResult = await aggregate(streamName, c => c({
                 coll: snapshotCollection,
                 input: link()
                     .with($match_(root().of('updated').has($eq(true))))
-                    .with($match_($expr(ne(root().of('after').expr())(root().of('before').expr()))))
                     .with(input.delta)
-                    .with(finalInput.raw(ts === undefined)).stages,
+                    .with(stages).stages,
             }), false, start);
             const stream = makeStream(result.cursor.atClusterTime);
-            return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream }), 'remove handled deleted updated', () => stream.close());
+            const nextRes = stream.tryNext();
+            const intoColl = stages.at(-1).$merge.into.coll;
+            const startx = Date.now();
+            await db
+                .collection(intoColl)
+                .countDocuments({ touchedAt: { $gte: result.cursor.atClusterTime } })
+                .then(count => log(`documents updated ${intoColl}`, count, 'took', Date.now() - startx));
+            return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream, nextRes }), 'remove handled deleted updated', () => stream.close());
         };
         const step5 = (l) => async () => {
             log(`remove handled deleted updated db['${snapshotCollection.collectionName}'].deleteMany({ updated: true, after: null })`);
@@ -1802,13 +1826,14 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
             return step8(l);
         };
         const step8 = (l) => {
-            return nextData(l.aggResult.cursor.firstBatch)(() => l.stream
-                .tryNext()
+            return nextData(l.aggResult.cursor.firstBatch)(() => l.nextRes
                 .catch((err) => {
                 log('restarting', err);
                 return { ts: null };
             })
-                .then(doc => (doc ? next(step3({ _id: streamName, ts: l.ts }), 'restart') : step8(l))), 'wait for change');
+                .then(doc => doc
+                ? next(step3({ _id: streamName, ts: l.ts }), 'restart')
+                : step8({ ...l, nextRes: l.stream.tryNext() })), 'wait for change');
         };
         return skip
             ? withStop(() => SynchronousPromise.resolve(next(step3(null), 'clone into new collection')))
@@ -1836,8 +1861,11 @@ const executes$1 = (view, input, streamName, needs) => {
     else if (streamNames[streamName] != hash)
         throw new Error('streamName already used');
     const { collection, projection, hardMatch: pre, match } = view;
-    const removeNotYetSynchronizedFields = projection && Object.values(mapExactToObject(projection, (_, k) => (needs[k] ?? k.startsWith('_')) ? root().of(k).has($exists(true)) : null));
-    const hardMatch = removeNotYetSynchronizedFields ? $and(pre, ...removeNotYetSynchronizedFields) : pre;
+    const removeNotYetSynchronizedFields = projection &&
+        Object.values(mapExactToObject(projection, (_, k) => (needs[k] ?? k.startsWith('_')) ? root().of(k).has($exists(true)) : null));
+    const hardMatch = removeNotYetSynchronizedFields
+        ? $and(pre, ...removeNotYetSynchronizedFields)
+        : pre;
     const job = {};
     const db = collection.s.db, coll = collection.collectionName;
     db.command({
@@ -1849,15 +1877,15 @@ const executes$1 = (view, input, streamName, needs) => {
         name: 'touchedAt_' + new UUID().toString('base64'),
     });
     const last = db.collection('__last');
-    const projectInput = projection && $project_(spread(projection, {
-        deletedAt: ['deletedAt', 1],
-        _id: ['_id', 1],
-    }));
+    const projectInput = projection &&
+        $project_(spread(projection, {
+            deletedAt: ['deletedAt', 1],
+            _id: ['_id', 1],
+        }));
     const notDeleted = root().of('deletedAt').has($eq(null));
     const stages = (lastTS) => {
         const hardQuery = $and(lastTS && root().of('touchedAt').has($gteTs(lastTS.ts)), hardMatch, notDeleted, match && $expr(match));
-        const ln = link()
-            .with($match_(hardQuery));
+        const ln = link().with($match_(hardQuery));
         return (projectInput ? ln.with(projectInput) : ln).with(input);
     };
     const run = (finalInput) => {
@@ -1908,20 +1936,26 @@ const executes$1 = (view, input, streamName, needs) => {
             return {
                 cont: withStop(async () => {
                     await new Promise(resolve => setTimeout(resolve, 1000));
-                    return step4(same)();
+                    return next(step4(same), 'clone into new collection');
                 }),
                 data: [],
-                info: { debug: 'clone into new collection', job: undefined },
+                info: { debug: 'wait for clone into new collection', job: undefined },
             };
         };
         const makeStream = (startAt) => makeWatchStream(db, view, startAt, streamName);
         const step4 = (lastTS) => async () => {
+            const raw = stages(lastTS).with(finalInput.raw(lastTS === null)).stages;
             const aggResult = await aggregate(streamName, c => c({
                 coll: collection,
-                input: stages(lastTS).with(finalInput.raw(lastTS === null)).stages,
+                input: raw,
             }));
             const stream = makeStream(aggResult.cursor.atClusterTime);
-            return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream }), 'update __last', () => stream.close());
+            const nextRes = stream.tryNext();
+            const intoColl = raw.at(-1).$merge.into.coll;
+            db.collection(intoColl)
+                .countDocuments({ touchedAt: { $gte: aggResult.cursor.atClusterTime } })
+                .then(count => log(`documents updated ${intoColl}`, count));
+            return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream, nextRes }), 'update __last', () => stream.close());
         };
         const step7 = (l) => async () => {
             await last.updateOne({ _id: streamName }, { $set: { ts: l.ts, data } }, { upsert: true });
@@ -1931,13 +1965,14 @@ const executes$1 = (view, input, streamName, needs) => {
             return {
                 data: l.aggResult.cursor.firstBatch,
                 info: { job: undefined, debug: 'wait for change' },
-                cont: withStop(() => l.stream
-                    .tryNext()
+                cont: withStop(() => l.nextRes
                     .catch((err) => {
                     log('restarting', err);
                     return { ts: null };
                 })
-                    .then(doc => (doc ? next(step4({ _id: streamName, ts: l.ts }), 'restart') : step8(l)))),
+                    .then(doc => doc
+                    ? next(step4({ _id: streamName, ts: l.ts }), 'restart')
+                    : step8({ ...l, nextRes: l.stream.tryNext() }))),
             };
         };
         return stop;

package/index.js

@@ -1333,13 +1333,50 @@ const mergeIterators = (params) => {
     };
 };
 
+const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
+const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
+const json = (a) => replace(JSON.stringify(a));
+const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
+
+const state = { steady: false };
+let timeout = null;
+const aggregate = (streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
+    const req = {
+        aggregate: coll.collectionName,
+        pipeline: input,
+        cursor: {},
+        ...(snapshot && { readConcern: { level: 'snapshot' } }),
+    };
+    if (timeout !== null) {
+        clearTimeout(timeout);
+        timeout = null;
+    }
+    log('exec', streamName, req);
+    return coll.s.db.command(req).then(result => {
+        const r = result;
+        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('$$CLUSTER_TIME', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
+        if (!state.steady) {
+            if (timeout !== null)
+                throw new Error('timeout should be null');
+            timeout = setTimeout(() => {
+                state.steady = true;
+                console.log('steady');
+            }, 1000);
+        }
+        return r;
+    }, err => {
+        log('err', req, err);
+        throw new Error(err);
+    });
+});
+
 const firstWorksMerge = (iters) => {
     const iterator = () => {
         const results = iters.map(iter => iter());
         const sources = { ...results };
         return mergeIterators({
             sources,
-            interrupt: key => false,
+            interrupt: key => state.steady
         });
     };
     return iterator;
@@ -1503,29 +1540,6 @@ const $insertPart = (out, ext) => {
 const $insert = (out) => $insertPart(out, {});
 const assertNotNull = (expr) => expr;
 
-const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
-const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
-const json = (a) => replace(JSON.stringify(a));
-const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
-
-const aggregate = (streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
-    const req = {
-        aggregate: coll.collectionName,
-        pipeline: input,
-        cursor: {},
-        ...(snapshot && { readConcern: { level: 'snapshot' } }),
-    };
-    log('exec', streamName, req);
-    return coll.s.db.command(req).then(result => {
-        const r = result;
-        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('$$CLUSTER_TIME', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
-        return r;
-    }, err => {
-        log('err', req, err);
-        throw new Error(err);
-    });
-});
-
 const addTeardown = (it, tr) => {
     if (!tr)
         return it;
@@ -1757,22 +1771,32 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
             })).stages;
             const r = await aggregate(streamName, c => c({ coll: collection, input: cloneIntoNew }));
             await snapshotCollection.deleteMany({ updated: true, after: null, before: null });
+            const start = Date.now();
+            const count = await snapshotCollection.countDocuments({ updated: true });
+            log(streamName, `documents to update ${snapshotCollection.collectionName}`, count, 'took', Date.now() - start);
             return next(step4({ result: r, ts: lastTS?.ts }), 'run the aggregation');
         };
         const makeStream = (startAt) => makeWatchStream(db, view, startAt, streamName);
         const step4 = ({ result, ts }) => async () => {
             const start = Date.now();
             await snapshotCollection.updateMany({ before: null }, { $set: { before: null } });
+            const stages = finalInput.raw(ts === undefined);
             const aggResult = await aggregate(streamName, c => c({
                 coll: snapshotCollection,
                 input: link()
                     .with($match_(root().of('updated').has($eq(true))))
-                    .with($match_($expr(ne(root().of('after').expr())(root().of('before').expr()))))
                     .with(input.delta)
-                    .with(finalInput.raw(ts === undefined)).stages,
+                    .with(stages).stages,
             }), false, start);
             const stream = makeStream(result.cursor.atClusterTime);
-            return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream }), 'remove handled deleted updated', () => stream.close());
+            const nextRes = stream.tryNext();
+            const intoColl = stages.at(-1).$merge.into.coll;
+            const startx = Date.now();
+            await db
+                .collection(intoColl)
+                .countDocuments({ touchedAt: { $gte: result.cursor.atClusterTime } })
+                .then(count => log(`documents updated ${intoColl}`, count, 'took', Date.now() - startx));
+            return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream, nextRes }), 'remove handled deleted updated', () => stream.close());
         };
         const step5 = (l) => async () => {
             log(`remove handled deleted updated db['${snapshotCollection.collectionName}'].deleteMany({ updated: true, after: null })`);
@@ -1804,13 +1828,14 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
             return step8(l);
         };
         const step8 = (l) => {
-            return nextData(l.aggResult.cursor.firstBatch)(() => l.stream
-                .tryNext()
+            return nextData(l.aggResult.cursor.firstBatch)(() => l.nextRes
                 .catch((err) => {
                 log('restarting', err);
                 return { ts: null };
             })
-                .then(doc => (doc ? next(step3({ _id: streamName, ts: l.ts }), 'restart') : step8(l))), 'wait for change');
+                .then(doc => doc
+                ? next(step3({ _id: streamName, ts: l.ts }), 'restart')
+                : step8({ ...l, nextRes: l.stream.tryNext() })), 'wait for change');
         };
         return skip
             ? withStop(() => synchronousPromise.SynchronousPromise.resolve(next(step3(null), 'clone into new collection')))
@@ -1838,8 +1863,11 @@ const executes$1 = (view, input, streamName, needs) => {
     else if (streamNames[streamName] != hash)
         throw new Error('streamName already used');
     const { collection, projection, hardMatch: pre, match } = view;
-    const removeNotYetSynchronizedFields = projection && Object.values(mapExactToObject(projection, (_, k) => (needs[k] ?? k.startsWith('_')) ? root().of(k).has($exists(true)) : null));
-    const hardMatch = removeNotYetSynchronizedFields ? $and(pre, ...removeNotYetSynchronizedFields) : pre;
+    const removeNotYetSynchronizedFields = projection &&
+        Object.values(mapExactToObject(projection, (_, k) => (needs[k] ?? k.startsWith('_')) ? root().of(k).has($exists(true)) : null));
+    const hardMatch = removeNotYetSynchronizedFields
+        ? $and(pre, ...removeNotYetSynchronizedFields)
+        : pre;
     const job = {};
     const db = collection.s.db, coll = collection.collectionName;
     db.command({
@@ -1851,15 +1879,15 @@ const executes$1 = (view, input, streamName, needs) => {
         name: 'touchedAt_' + new mongodb.UUID().toString('base64'),
     });
     const last = db.collection('__last');
-    const projectInput = projection && $project_(spread(projection, {
-        deletedAt: ['deletedAt', 1],
-        _id: ['_id', 1],
-    }));
+    const projectInput = projection &&
+        $project_(spread(projection, {
+            deletedAt: ['deletedAt', 1],
+            _id: ['_id', 1],
+        }));
     const notDeleted = root().of('deletedAt').has($eq(null));
     const stages = (lastTS) => {
         const hardQuery = $and(lastTS && root().of('touchedAt').has($gteTs(lastTS.ts)), hardMatch, notDeleted, match && $expr(match));
-        const ln = link()
-            .with($match_(hardQuery));
+        const ln = link().with($match_(hardQuery));
         return (projectInput ? ln.with(projectInput) : ln).with(input);
     };
     const run = (finalInput) => {
@@ -1910,20 +1938,26 @@ const executes$1 = (view, input, streamName, needs) => {
             return {
                 cont: withStop(async () => {
                     await new Promise(resolve => setTimeout(resolve, 1000));
-                    return step4(same)();
+                    return next(step4(same), 'clone into new collection');
                 }),
                 data: [],
-                info: { debug: 'clone into new collection', job: undefined },
+                info: { debug: 'wait for clone into new collection', job: undefined },
             };
         };
         const makeStream = (startAt) => makeWatchStream(db, view, startAt, streamName);
         const step4 = (lastTS) => async () => {
+            const raw = stages(lastTS).with(finalInput.raw(lastTS === null)).stages;
             const aggResult = await aggregate(streamName, c => c({
                 coll: collection,
-                input: stages(lastTS).with(finalInput.raw(lastTS === null)).stages,
+                input: raw,
             }));
             const stream = makeStream(aggResult.cursor.atClusterTime);
-            return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream }), 'update __last', () => stream.close());
+            const nextRes = stream.tryNext();
+            const intoColl = raw.at(-1).$merge.into.coll;
+            db.collection(intoColl)
+                .countDocuments({ touchedAt: { $gte: aggResult.cursor.atClusterTime } })
+                .then(count => log(`documents updated ${intoColl}`, count));
+            return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream, nextRes }), 'update __last', () => stream.close());
         };
         const step7 = (l) => async () => {
             await last.updateOne({ _id: streamName }, { $set: { ts: l.ts, data } }, { upsert: true });
@@ -1933,13 +1967,14 @@ const executes$1 = (view, input, streamName, needs) => {
             return {
                 data: l.aggResult.cursor.firstBatch,
                 info: { job: undefined, debug: 'wait for change' },
-                cont: withStop(() => l.stream
-                    .tryNext()
+                cont: withStop(() => l.nextRes
                     .catch((err) => {
                     log('restarting', err);
                     return { ts: null };
                 })
-                    .then(doc => (doc ? next(step4({ _id: streamName, ts: l.ts }), 'restart') : step8(l)))),
+                    .then(doc => doc
+                    ? next(step4({ _id: streamName, ts: l.ts }), 'restart')
+                    : step8({ ...l, nextRes: l.stream.tryNext() }))),
             };
         };
         return stop;

package/package.json

@@ -3,7 +3,7 @@
     "module": "index.esm.js",
     "typings": "index.d.ts",
     "name": "@omegup/msync",
-    "version": "0.1.14",
+    "version": "0.1.16",
     "dependencies": {
         "dayjs": "^1.11.9",
         "dotenv": "^16.3.1",