@omegup/msync 0.0.48 → 0.0.49

package/index.d.ts

@@ -645,6 +645,7 @@ type MongoTypes = {
 };
 type MongoTypeNames = keyof MongoTypes;
 declare const $type: <T extends keyof MongoTypes>(operand: rawItem & readonly T[]) => Predicate<N | MongoTypes[T]>;
+declare const $exists: <T extends keyof MongoTypes>(operand: rawItem & boolean) => Predicate<unknown>;
 
 declare const $expr: <D extends O, C>(expr: Expr<boolean, D, C>) => Query<D, C>;
 
@@ -675,4 +676,4 @@ declare const enablePreAndPostImages: <T extends doc>(coll: Collection<T>) => Pr
 declare const prepare: (testName?: string) => Promise<MongoClient$1>;
 declare const makeCol: <T extends ID>(docs: readonly OptionalUnlessRequiredId<T>[], database: Db, name?: string) => Promise<Collection<T>>;
 
-export { $accumulator, $and, $countDict, $entries, $eq, $expr, $getField, $group, $groupId, $groupMerge, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map1, $match, $matchDelta, $merge, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $replaceWith, $set, $simpleInsert, $simpleMerge, $sum, $type, $unwind, $unwindDelta, type Accumulators, type Arr, type AsLiteral, type Delta, type DeltaAccumulator, type DeltaAccumulators, Expr, type ExprHKT, type Exprs, type ExprsExact, type ExprsExactHKT, type ExprsPart, Field, type ID, type Loose, Machine, type Merge, type MergeArgs, type MergeInto, type MergeMapOArgs, type Model, type MongoTypeNames, type N, type NoRaw, type NullToOBJ, type O, type OPick, type OPickD, type RONoRaw, type RORec, type RawStages, type Rec, type Replace, type SnapshotStreamExecutionResult, type StrKey, type Strict, type TS, Type, type WriteonlyCollection, add, and, array, ceil, comp, concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, type doc, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first, firstSure, floor, from, func, gt, gte, inArray, isArray, ite, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, type notArr, notNull, now, or, pair, prepare, rand, range, root, set, setField, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };
+export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map1, $match, $matchDelta, $merge, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $replaceWith, $set, $simpleInsert, $simpleMerge, $sum, $type, $unwind, $unwindDelta, type Accumulators, type Arr, type AsLiteral, type Delta, type DeltaAccumulator, type DeltaAccumulators, Expr, type ExprHKT, type Exprs, type ExprsExact, type ExprsExactHKT, type ExprsPart, Field, type ID, type Loose, Machine, type Merge, type MergeArgs, type MergeInto, type MergeMapOArgs, type Model, type MongoTypeNames, type N, type NoRaw, type NullToOBJ, type O, type OPick, type OPickD, type RONoRaw, type RORec, type RawStages, type Rec, type Replace, type SnapshotStreamExecutionResult, type StrKey, type Strict, type TS, Type, type WriteonlyCollection, add, and, array, ceil, comp, concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, type doc, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first, firstSure, floor, from, func, gt, gte, inArray, isArray, ite, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, type notArr, notNull, now, or, pair, prepare, rand, range, root, set, setField, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };

package/index.esm.js

@@ -1,7 +1,6 @@
 import { UUID, MongoClient } from 'mongodb';
 import { SynchronousPromise } from 'synchronous-promise';
 import crypto$1 from 'crypto';
-import { log as log$1 } from 'console';
 import { writeFile } from 'fs/promises';
 
 const asExprRaw = (raw) => ({ get: () => raw });
@@ -900,6 +899,7 @@ const dateLt = comp('$lt');
 const $gte = comp('$gte');
 const $lte = comp('$lte');
 const $type = operator()('$type');
+const $exists = operator()('$exists');
 
 const $expr = (expr) => ({
     raw: f => ({ $expr: expr.raw(f).get() }),
@@ -1382,16 +1382,16 @@ const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
 const json = (a) => replace(JSON.stringify(a));
 const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
 
-const aggregate = (input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
+const aggregate = (streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
     const req = {
         aggregate: coll.collectionName,
         pipeline: input,
         cursor: {},
         ...(snapshot && { readConcern: { level: 'snapshot' } }),
     };
-    log('exec', req);
+    log('exec', streamName, req);
     return coll.s.db.command(req).then(result => {
-        log('execed', req, result, 'took', Date.now() - start);
+        log('execed', streamName, req, result, 'took', Date.now() - start);
         return result;
     }, err => {
         log('err', req, err);
@@ -1443,11 +1443,19 @@ const makeWatchStream = (db, { collection, projection: p, hardMatch: m }, startA
         },
     });
     pipeline.push({
-        $match: {
-            $or: [
-                { $expr: { $ne: ['$fullDocument', '$fullDocumentBeforeChange'] } },
-                Object.fromEntries(changeKeys.map(k => [k, null])),
-            ],
+        $replaceWith: {
+            ts: {
+                $cond: {
+                    if: {
+                        $or: [
+                            { $ne: ['$fullDocument', '$fullDocumentBeforeChange'] },
+                            { $and: changeKeys.map(k => ({ $eq: [k, null] })) },
+                        ],
+                    },
+                    then: '$clusterTime',
+                    else: null,
+                },
+            },
         },
     });
     const stream = db.collection(collection.collectionName).watch(pipeline, {
@@ -1460,7 +1468,7 @@ const makeWatchStream = (db, { collection, projection: p, hardMatch: m }, startA
         if (doc)
             await new Promise(resolve => setTimeout(resolve, 100));
         if (doc)
-            log$1('detected', streamName, collection.collectionName, doc);
+            log('detected', streamName, collection.collectionName, doc);
         return doc;
     };
     return { tryNext, close: () => stream.close() };
@@ -1483,12 +1491,7 @@ const executes$1 = (view, input, streamName) => {
     else if (streamNames[streamName] != hash)
         throw new Error(`streamName ${streamName} already used`);
     const { collection, projection, hardMatch: pre, match } = view;
-    const types = Array.from({ length: 19 }).map((_, i) => i + 1);
-    const removeNotYetSynchronizedFields = Object.values(mapExactToObject(projection, (_, k) => k.startsWith('_')
-        ? root()
-            .of(k)
-            .has($type(types.filter(x => x != 10)))
-        : null));
+    const removeNotYetSynchronizedFields = Object.values(mapExactToObject(projection, (_, k) => k.startsWith('_') ? root().of(k).has($exists(true)) : null));
     const hardMatch = $and(pre, ...removeNotYetSynchronizedFields);
     const job = {};
     const db = collection.s.db, coll = collection.collectionName;
@@ -1591,7 +1594,7 @@ const executes$1 = (view, input, streamName) => {
                 whenMatched: 'merge',
                 whenNotMatched: 'insert',
             })).stages;
-            const r = await aggregate(c => c({ coll: collection, input: cloneIntoNew }));
+            const r = await aggregate(streamName, c => c({ coll: collection, input: cloneIntoNew }));
             await snapshotCollection.deleteMany({ updated: true, after: null, before: null });
             return next(step4({ result: r, ts: lastTS?.ts }), 'run the aggregation');
         };
@@ -1599,7 +1602,7 @@ const executes$1 = (view, input, streamName) => {
         const step4 = ({ result, ts }) => async () => {
             const start = Date.now();
             await snapshotCollection.updateMany({ before: null }, { $set: { before: null } });
-            const aggResult = await aggregate(c => c({
+            const aggResult = await aggregate(streamName, c => c({
                 coll: snapshotCollection,
                 input: link()
                     .with($match_(root().of('updated').has($eq(true))))
@@ -1608,7 +1611,7 @@ const executes$1 = (view, input, streamName) => {
                     .with(finalInput.raw(ts === undefined)).stages,
             }), false, start);
             const stream = makeStream(result.cursor.atClusterTime);
-            return next(step5({ result, aggResult, stream }), 'remove handled deleted updated', () => stream.close());
+            return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream }), 'remove handled deleted updated', () => stream.close());
         };
         const step5 = (l) => async () => {
             log(`remove handled deleted updated db['${snapshotCollection.collectionName}'].deleteMany({ updated: true, after: null })`);
@@ -1633,7 +1636,7 @@ const executes$1 = (view, input, streamName) => {
         const step7 = (l) => async () => {
             await last.updateOne({ _id: streamName }, {
                 $set: {
-                    ts: l.result.cursor.atClusterTime,
+                    ts: l.ts,
                     data,
                 },
             }, { upsert: true });
@@ -1642,11 +1645,15 @@ const executes$1 = (view, input, streamName) => {
         const step8 = (l) => {
             return nextData(l.aggResult.cursor.firstBatch)(() => l.stream
                 .tryNext()
-                .catch(err => {
+                .catch((err) => {
                 log('restarting', err);
-                return 1;
+                return { ts: null };
             })
-                .then(doc => (doc ? next(step2, 'restart') : step8(l))), 'wait for change');
+                .then(doc => doc
+                ? doc.ts
+                    ? next(step7({ ...l, ts: doc.ts }), 'nothing changed')
+                    : next(step2, 'restart')
+                : step8(l)), 'wait for change');
         };
         return stop;
     };
@@ -1736,7 +1743,7 @@ const executes = (view, input, streamName) => {
         const makeStream = (startAt) => makeWatchStream(db, view, startAt, streamName);
         const step4 = (lastTS) => async () => {
             const hardQuery = $and(lastTS && root().of('touchedAt').has($gteTs(lastTS.ts)), hardMatch, notDeleted, match && $expr(match));
-            const aggResult = await aggregate(c => c({
+            const aggResult = await aggregate(streamName, c => c({
                 coll: collection,
                 input: link()
                     .with($match_(hardQuery))
@@ -1745,17 +1752,23 @@ const executes = (view, input, streamName) => {
                     .with(finalInput.raw(lastTS === null)).stages,
             }));
             const stream = makeStream(aggResult.cursor.atClusterTime);
-            return next(step7({ aggResult, result: aggResult, stream }), 'update __last', () => stream.close());
+            return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream }), 'update __last', () => stream.close());
         };
         const step7 = (l) => async () => {
-            await last.updateOne({ _id: streamName }, { $set: { ts: l.result.cursor.atClusterTime, data } }, { upsert: true });
+            await last.updateOne({ _id: streamName }, { $set: { ts: l.ts, data } }, { upsert: true });
             return step8(l);
         };
         const step8 = (l) => {
             return {
                 data: l.aggResult.cursor.firstBatch,
                 info: { job: undefined, debug: 'wait for change' },
-                cont: withStop(() => l.stream.tryNext().then(doc => (doc ? next(step1, 'restart') : step8(l)))),
+                cont: withStop(() => l.stream
+                    .tryNext()
+                    .then(doc => doc
+                    ? doc.ts
+                        ? next(step7({ ...l, ts: doc.ts }), 'nothing changed')
+                        : next(step1, 'restart')
+                    : step8(l))),
             };
         };
         return stop;
@@ -1810,4 +1823,4 @@ const makeCol = async (docs, database, name) => {
     }
 };
 
-export { $accumulator, $and, $countDict, $entries, $eq, $expr, $getField, $group, $groupId, $groupMerge, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map1, $match, $matchDelta, $merge, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $replaceWith, $set, $simpleInsert, $simpleMerge, $sum, $type, $unwind, $unwindDelta, Field, Machine, add, and, array, ceil, comp, concat$1 as concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first$1 as first, firstSure, floor, from, func, gt, gte, inArray, isArray, ite, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, notNull, now, or, pair, prepare, rand, range, root, set, setField, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };
+export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map1, $match, $matchDelta, $merge, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $replaceWith, $set, $simpleInsert, $simpleMerge, $sum, $type, $unwind, $unwindDelta, Field, Machine, add, and, array, ceil, comp, concat$1 as concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first$1 as first, firstSure, floor, from, func, gt, gte, inArray, isArray, ite, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, notNull, now, or, pair, prepare, rand, range, root, set, setField, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };

package/index.js

@@ -3,7 +3,6 @@
 var mongodb = require('mongodb');
 var synchronousPromise = require('synchronous-promise');
 var crypto$1 = require('crypto');
-var console$1 = require('console');
 var promises = require('fs/promises');
 
 const asExprRaw = (raw) => ({ get: () => raw });
@@ -902,6 +901,7 @@ const dateLt = comp('$lt');
 const $gte = comp('$gte');
 const $lte = comp('$lte');
 const $type = operator()('$type');
+const $exists = operator()('$exists');
 
 const $expr = (expr) => ({
     raw: f => ({ $expr: expr.raw(f).get() }),
@@ -1384,16 +1384,16 @@ const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
 const json = (a) => replace(JSON.stringify(a));
 const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
 
-const aggregate = (input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
+const aggregate = (streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
     const req = {
         aggregate: coll.collectionName,
         pipeline: input,
         cursor: {},
         ...(snapshot && { readConcern: { level: 'snapshot' } }),
     };
-    log('exec', req);
+    log('exec', streamName, req);
     return coll.s.db.command(req).then(result => {
-        log('execed', req, result, 'took', Date.now() - start);
+        log('execed', streamName, req, result, 'took', Date.now() - start);
         return result;
     }, err => {
         log('err', req, err);
@@ -1445,11 +1445,19 @@ const makeWatchStream = (db, { collection, projection: p, hardMatch: m }, startA
         },
     });
     pipeline.push({
-        $match: {
-            $or: [
-                { $expr: { $ne: ['$fullDocument', '$fullDocumentBeforeChange'] } },
-                Object.fromEntries(changeKeys.map(k => [k, null])),
-            ],
+        $replaceWith: {
+            ts: {
+                $cond: {
+                    if: {
+                        $or: [
+                            { $ne: ['$fullDocument', '$fullDocumentBeforeChange'] },
+                            { $and: changeKeys.map(k => ({ $eq: [k, null] })) },
+                        ],
+                    },
+                    then: '$clusterTime',
+                    else: null,
+                },
+            },
         },
     });
     const stream = db.collection(collection.collectionName).watch(pipeline, {
@@ -1462,7 +1470,7 @@ const makeWatchStream = (db, { collection, projection: p, hardMatch: m }, startA
         if (doc)
             await new Promise(resolve => setTimeout(resolve, 100));
         if (doc)
-            console$1.log('detected', streamName, collection.collectionName, doc);
+            log('detected', streamName, collection.collectionName, doc);
         return doc;
     };
     return { tryNext, close: () => stream.close() };
@@ -1485,12 +1493,7 @@ const executes$1 = (view, input, streamName) => {
     else if (streamNames[streamName] != hash)
         throw new Error(`streamName ${streamName} already used`);
     const { collection, projection, hardMatch: pre, match } = view;
-    const types = Array.from({ length: 19 }).map((_, i) => i + 1);
-    const removeNotYetSynchronizedFields = Object.values(mapExactToObject(projection, (_, k) => k.startsWith('_')
-        ? root()
-            .of(k)
-            .has($type(types.filter(x => x != 10)))
-        : null));
+    const removeNotYetSynchronizedFields = Object.values(mapExactToObject(projection, (_, k) => k.startsWith('_') ? root().of(k).has($exists(true)) : null));
     const hardMatch = $and(pre, ...removeNotYetSynchronizedFields);
     const job = {};
     const db = collection.s.db, coll = collection.collectionName;
@@ -1593,7 +1596,7 @@ const executes$1 = (view, input, streamName) => {
                 whenMatched: 'merge',
                 whenNotMatched: 'insert',
             })).stages;
-            const r = await aggregate(c => c({ coll: collection, input: cloneIntoNew }));
+            const r = await aggregate(streamName, c => c({ coll: collection, input: cloneIntoNew }));
             await snapshotCollection.deleteMany({ updated: true, after: null, before: null });
             return next(step4({ result: r, ts: lastTS?.ts }), 'run the aggregation');
         };
@@ -1601,7 +1604,7 @@ const executes$1 = (view, input, streamName) => {
         const step4 = ({ result, ts }) => async () => {
             const start = Date.now();
             await snapshotCollection.updateMany({ before: null }, { $set: { before: null } });
-            const aggResult = await aggregate(c => c({
+            const aggResult = await aggregate(streamName, c => c({
                 coll: snapshotCollection,
                 input: link()
                     .with($match_(root().of('updated').has($eq(true))))
@@ -1610,7 +1613,7 @@ const executes$1 = (view, input, streamName) => {
                     .with(finalInput.raw(ts === undefined)).stages,
             }), false, start);
             const stream = makeStream(result.cursor.atClusterTime);
-            return next(step5({ result, aggResult, stream }), 'remove handled deleted updated', () => stream.close());
+            return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream }), 'remove handled deleted updated', () => stream.close());
         };
         const step5 = (l) => async () => {
             log(`remove handled deleted updated db['${snapshotCollection.collectionName}'].deleteMany({ updated: true, after: null })`);
@@ -1635,7 +1638,7 @@ const executes$1 = (view, input, streamName) => {
         const step7 = (l) => async () => {
             await last.updateOne({ _id: streamName }, {
                 $set: {
-                    ts: l.result.cursor.atClusterTime,
+                    ts: l.ts,
                     data,
                 },
             }, { upsert: true });
@@ -1644,11 +1647,15 @@ const executes$1 = (view, input, streamName) => {
         const step8 = (l) => {
             return nextData(l.aggResult.cursor.firstBatch)(() => l.stream
                 .tryNext()
-                .catch(err => {
+                .catch((err) => {
                 log('restarting', err);
-                return 1;
+                return { ts: null };
             })
-                .then(doc => (doc ? next(step2, 'restart') : step8(l))), 'wait for change');
+                .then(doc => doc
+                ? doc.ts
+                    ? next(step7({ ...l, ts: doc.ts }), 'nothing changed')
+                    : next(step2, 'restart')
+                : step8(l)), 'wait for change');
         };
         return stop;
     };
@@ -1738,7 +1745,7 @@ const executes = (view, input, streamName) => {
         const makeStream = (startAt) => makeWatchStream(db, view, startAt, streamName);
         const step4 = (lastTS) => async () => {
             const hardQuery = $and(lastTS && root().of('touchedAt').has($gteTs(lastTS.ts)), hardMatch, notDeleted, match && $expr(match));
-            const aggResult = await aggregate(c => c({
+            const aggResult = await aggregate(streamName, c => c({
                 coll: collection,
                 input: link()
                     .with($match_(hardQuery))
@@ -1747,17 +1754,23 @@ const executes = (view, input, streamName) => {
                     .with(finalInput.raw(lastTS === null)).stages,
             }));
             const stream = makeStream(aggResult.cursor.atClusterTime);
-            return next(step7({ aggResult, result: aggResult, stream }), 'update __last', () => stream.close());
+            return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream }), 'update __last', () => stream.close());
         };
         const step7 = (l) => async () => {
-            await last.updateOne({ _id: streamName }, { $set: { ts: l.result.cursor.atClusterTime, data } }, { upsert: true });
+            await last.updateOne({ _id: streamName }, { $set: { ts: l.ts, data } }, { upsert: true });
             return step8(l);
         };
         const step8 = (l) => {
             return {
                 data: l.aggResult.cursor.firstBatch,
                 info: { job: undefined, debug: 'wait for change' },
-                cont: withStop(() => l.stream.tryNext().then(doc => (doc ? next(step1, 'restart') : step8(l)))),
+                cont: withStop(() => l.stream
+                    .tryNext()
+                    .then(doc => doc
+                    ? doc.ts
+                        ? next(step7({ ...l, ts: doc.ts }), 'nothing changed')
+                        : next(step1, 'restart')
+                    : step8(l))),
             };
         };
         return stop;
@@ -1817,6 +1830,7 @@ exports.$and = $and;
 exports.$countDict = $countDict;
 exports.$entries = $entries;
 exports.$eq = $eq;
+exports.$exists = $exists;
 exports.$expr = $expr;
 exports.$getField = $getField;
 exports.$group = $group;

package/package.json

@@ -3,7 +3,7 @@
     "module": "index.esm.js",
     "typings": "index.d.ts",
     "name": "@omegup/msync",
-    "version": "0.0.48",
+    "version": "0.0.49",
     "dependencies": {
         "dayjs": "^1.11.9",
         "dotenv": "^16.3.1",