@omegup/msync 0.1.15 → 0.1.16

package/index.esm.js

@@ -1331,13 +1331,50 @@ const mergeIterators = (params) => {
     };
 };
 
+const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
+const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
+const json = (a) => replace(JSON.stringify(a));
+const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
+
+const state = { steady: false };
+let timeout = null;
+const aggregate = (streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
+    const req = {
+        aggregate: coll.collectionName,
+        pipeline: input,
+        cursor: {},
+        ...(snapshot && { readConcern: { level: 'snapshot' } }),
+    };
+    if (timeout !== null) {
+        clearTimeout(timeout);
+        timeout = null;
+    }
+    log('exec', streamName, req);
+    return coll.s.db.command(req).then(result => {
+        const r = result;
+        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('$$CLUSTER_TIME', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
+        if (!state.steady) {
+            if (timeout !== null)
+                throw new Error('timeout should be null');
+            timeout = setTimeout(() => {
+                state.steady = true;
+                console.log('steady');
+            }, 1000);
+        }
+        return r;
+    }, err => {
+        log('err', req, err);
+        throw new Error(err);
+    });
+});
+
 const firstWorksMerge = (iters) => {
     const iterator = () => {
         const results = iters.map(iter => iter());
         const sources = { ...results };
         return mergeIterators({
             sources,
-            interrupt: key => false,
+            interrupt: key => state.steady
         });
     };
     return iterator;
@@ -1501,29 +1538,6 @@ const $insertPart = (out, ext) => {
 const $insert = (out) => $insertPart(out, {});
 const assertNotNull = (expr) => expr;
 
-const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
-const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
-const json = (a) => replace(JSON.stringify(a));
-const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
-
-const aggregate = (streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
-    const req = {
-        aggregate: coll.collectionName,
-        pipeline: input,
-        cursor: {},
-        ...(snapshot && { readConcern: { level: 'snapshot' } }),
-    };
-    log('exec', streamName, req);
-    return coll.s.db.command(req).then(result => {
-        const r = result;
-        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('$$CLUSTER_TIME', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
-        return r;
-    }, err => {
-        log('err', req, err);
-        throw new Error(err);
-    });
-});
-
 const addTeardown = (it, tr) => {
     if (!tr)
         return it;
@@ -1755,6 +1769,9 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
             })).stages;
             const r = await aggregate(streamName, c => c({ coll: collection, input: cloneIntoNew }));
             await snapshotCollection.deleteMany({ updated: true, after: null, before: null });
+            const start = Date.now();
+            const count = await snapshotCollection.countDocuments({ updated: true });
+            log(streamName, `documents to update ${snapshotCollection.collectionName}`, count, 'took', Date.now() - start);
             return next(step4({ result: r, ts: lastTS?.ts }), 'run the aggregation');
         };
         const makeStream = (startAt) => makeWatchStream(db, view, startAt, streamName);
@@ -1934,6 +1951,10 @@ const executes$1 = (view, input, streamName, needs) => {
             }));
             const stream = makeStream(aggResult.cursor.atClusterTime);
             const nextRes = stream.tryNext();
+            const intoColl = raw.at(-1).$merge.into.coll;
+            db.collection(intoColl)
+                .countDocuments({ touchedAt: { $gte: aggResult.cursor.atClusterTime } })
+                .then(count => log(`documents updated ${intoColl}`, count));
             return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream, nextRes }), 'update __last', () => stream.close());
         };
         const step7 = (l) => async () => {

package/index.js

@@ -1333,13 +1333,50 @@ const mergeIterators = (params) => {
     };
 };
 
+const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
+const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
+const json = (a) => replace(JSON.stringify(a));
+const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
+
+const state = { steady: false };
+let timeout = null;
+const aggregate = (streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
+    const req = {
+        aggregate: coll.collectionName,
+        pipeline: input,
+        cursor: {},
+        ...(snapshot && { readConcern: { level: 'snapshot' } }),
+    };
+    if (timeout !== null) {
+        clearTimeout(timeout);
+        timeout = null;
+    }
+    log('exec', streamName, req);
+    return coll.s.db.command(req).then(result => {
+        const r = result;
+        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('$$CLUSTER_TIME', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
+        if (!state.steady) {
+            if (timeout !== null)
+                throw new Error('timeout should be null');
+            timeout = setTimeout(() => {
+                state.steady = true;
+                console.log('steady');
+            }, 1000);
+        }
+        return r;
+    }, err => {
+        log('err', req, err);
+        throw new Error(err);
+    });
+});
+
 const firstWorksMerge = (iters) => {
     const iterator = () => {
         const results = iters.map(iter => iter());
         const sources = { ...results };
         return mergeIterators({
             sources,
-            interrupt: key => false,
+            interrupt: key => state.steady
         });
     };
     return iterator;
@@ -1503,29 +1540,6 @@ const $insertPart = (out, ext) => {
 const $insert = (out) => $insertPart(out, {});
 const assertNotNull = (expr) => expr;
 
-const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
-const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
-const json = (a) => replace(JSON.stringify(a));
-const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
-
-const aggregate = (streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
-    const req = {
-        aggregate: coll.collectionName,
-        pipeline: input,
-        cursor: {},
-        ...(snapshot && { readConcern: { level: 'snapshot' } }),
-    };
-    log('exec', streamName, req);
-    return coll.s.db.command(req).then(result => {
-        const r = result;
-        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('$$CLUSTER_TIME', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
-        return r;
-    }, err => {
-        log('err', req, err);
-        throw new Error(err);
-    });
-});
-
 const addTeardown = (it, tr) => {
     if (!tr)
         return it;
@@ -1757,6 +1771,9 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
             })).stages;
             const r = await aggregate(streamName, c => c({ coll: collection, input: cloneIntoNew }));
             await snapshotCollection.deleteMany({ updated: true, after: null, before: null });
+            const start = Date.now();
+            const count = await snapshotCollection.countDocuments({ updated: true });
+            log(streamName, `documents to update ${snapshotCollection.collectionName}`, count, 'took', Date.now() - start);
             return next(step4({ result: r, ts: lastTS?.ts }), 'run the aggregation');
         };
         const makeStream = (startAt) => makeWatchStream(db, view, startAt, streamName);
@@ -1936,6 +1953,10 @@ const executes$1 = (view, input, streamName, needs) => {
             }));
             const stream = makeStream(aggResult.cursor.atClusterTime);
             const nextRes = stream.tryNext();
+            const intoColl = raw.at(-1).$merge.into.coll;
+            db.collection(intoColl)
+                .countDocuments({ touchedAt: { $gte: aggResult.cursor.atClusterTime } })
+                .then(count => log(`documents updated ${intoColl}`, count));
             return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream, nextRes }), 'update __last', () => stream.close());
         };
         const step7 = (l) => async () => {

package/package.json

@@ -3,7 +3,7 @@
     "module": "index.esm.js",
     "typings": "index.d.ts",
     "name": "@omegup/msync",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "dependencies": {
         "dayjs": "^1.11.9",
         "dotenv": "^16.3.1",