@omegup/msync 0.1.21 → 0.1.22

package/index.d.ts

@@ -1,4 +1,4 @@
-import { Timestamp, Filter, UpdateFilter, BSON, Db, Collection, IndexSpecification, CreateIndexesOptions, MongoClient as MongoClient$1, OptionalUnlessRequiredId } from 'mongodb';
+import { Timestamp, Filter, UpdateFilter, BSON, MaxKey, Db, Collection, IndexSpecification, CreateIndexesOptions, MongoClient as MongoClient$1, OptionalUnlessRequiredId } from 'mongodb';
 export { Collection, Timestamp } from 'mongodb';
 
 type HasJob = {
@@ -220,7 +220,7 @@ declare const Type: unique symbol;
 
 type U = undefined;
 type N = null | U;
-type jsonPrim = number | null | string | boolean | Timestamp | Date;
+type jsonPrim = number | null | string | boolean | Timestamp | MaxKey | Date;
 type notObj = jsonPrim | U;
 type notArr = notObj | O;
 type jsonItem = unknown;
@@ -697,7 +697,7 @@ declare const wrap: <Result>(root: Machine<Result>) => Machine<Result>;
 declare const $eq: <T extends unknown>(operand: rawItem & T) => Predicate<T>;
 declare const $ne: <T extends unknown>(operand: rawItem & T) => Predicate<T>;
 type Numeric = number | Timestamp | Date;
-declare const comp: <D2 extends Numeric = Numeric>(op: "$lt" | "$gt" | "$lte" | "$gte") => <T extends Numeric>(operand: rawItem & D2) => Predicate<D2>;
+declare const comp: <D2 extends Numeric = Numeric>(op: "$lt" | "$lte" | "$gt" | "$gte") => <T extends Numeric>(operand: rawItem & D2) => Predicate<D2>;
 declare const $gt: <T extends Numeric>(operand: rawItem & Numeric) => Predicate<Numeric>;
 declare const $gtTs: <T extends Numeric>(operand: rawItem & Timestamp) => Predicate<Timestamp>;
 declare const $gteTs: <T extends Numeric>(operand: rawItem & Timestamp) => Predicate<Timestamp>;
@@ -730,12 +730,8 @@ declare const $and: Combiner;
 declare const $nor: Combiner;
 declare const $or: Combiner;
 
-declare const setF: (f: ({ input }: {
-    input: any;
-}) => Promise<void>) => void;
-
 declare const enablePreAndPostImages: <T extends doc>(coll: Collection<T>) => Promise<Document>;
 declare const prepare: (testName?: string) => Promise<MongoClient$1>;
 declare const makeCol: <T extends ID>(docs: readonly OptionalUnlessRequiredId<T>[], database: Db, name?: string) => Promise<Collection<T>>;
 
-export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, type Accumulators, type Arr, type AsLiteral, type Delta, type DeltaAccumulator, type DeltaAccumulators, type ExactKeys, Expr, type ExprHKT, type Exprs, type ExprsExact, type ExprsExactHKT, type ExprsPart, Field, type ID, type Loose, Machine, type Merge, type MergeArgs, type MergeInto, type MergeMapOArgs, type Model, type MongoTypeNames, type N, type NoRaw, type NullToOBJ, type O, type OPick, type OPickD, type Patch, type RONoRaw, type RORec, type RawStages, type Rec, type Replace, type SnapshotStreamExecutionResult, type StrKey, type Strict, type TS, Type, type WriteonlyCollection, add, and, anyElementTrue, array, ceil, comp, concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, type doc, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first, firstSure, floor, from, func, getWhenMatched, getWhenMatchedForMerge, gt, gte, inArray, isArray, ite, type jsonPrim, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, not, type notArr, notNull, now, or, pair, prepare, rand, range, regex, root, set, setF, setField, single, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };
+export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, type Accumulators, type Arr, type AsLiteral, type Delta, type DeltaAccumulator, type DeltaAccumulators, type ExactKeys, Expr, type ExprHKT, type Exprs, type ExprsExact, type ExprsExactHKT, type ExprsPart, Field, type ID, type Loose, Machine, type Merge, type MergeArgs, type MergeInto, type MergeMapOArgs, type Model, type MongoTypeNames, type N, type NoRaw, type NullToOBJ, type O, type OPick, type OPickD, type Patch, type RONoRaw, type RORec, type RawStages, type Rec, type Replace, type SnapshotStreamExecutionResult, type StrKey, type Strict, type TS, Type, type WriteonlyCollection, add, and, anyElementTrue, array, ceil, comp, concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, type doc, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first, firstSure, floor, from, func, getWhenMatched, getWhenMatchedForMerge, gt, gte, inArray, isArray, ite, type jsonPrim, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, not, type notArr, notNull, now, or, pair, prepare, rand, range, regex, root, set, setField, single, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };

package/index.esm.js

@@ -1,7 +1,7 @@
+import { MaxKey, MongoClient, UUID } from 'mongodb';
 import crypto$1 from 'crypto';
 import { canonicalize } from 'json-canonicalize';
 import { SynchronousPromise } from 'synchronous-promise';
-import { MongoClient, UUID } from 'mongodb';
 import { writeFile } from 'fs/promises';
 
 const asExprRaw = (raw) => ({ get: () => raw });
@@ -82,7 +82,7 @@ const val = (val) => asExpr({
         : val),
 });
 const current = asExpr({
-    raw: () => asExprRaw('$$CLUSTER_TIME'),
+    raw: () => asExprRaw(new MaxKey()),
 });
 const $let = (vars, inExpr) => asExpr({
     raw: f => asExprRaw({
@@ -1304,16 +1304,18 @@ const nextWinner = (previousWinner, previousWinnerNextFrame, sources, interrupt)
 };
 
 const mergeIterators = (params) => {
-    const { sources, interrupt, select = race } = params;
+    const { sources, interrupt, select = race, hooks } = params;
     const reiterate = (winner) => {
         const { frame, key } = winner;
         return {
             cont: () => {
                 const result = frame.cont();
+                hooks?.start?.(frame, result);
                 return mergeIterators({
                     sources: patch(sources, key, result),
                     interrupt,
                     select: sources => nextWinner(winner, result.next, sources, interrupt),
+                    hooks,
                 });
             },
             data: frame.data,
@@ -1321,7 +1323,7 @@ const mergeIterators = (params) => {
         };
     };
     return {
-        stop: () => mergeIterators({ sources: restart(sources), interrupt }),
+        stop: () => mergeIterators({ sources: restart(sources), interrupt, select, hooks }),
         next: select(sources).then(reiterate),
         clear: async () => {
             for (const key in sources) {
@@ -1331,56 +1333,37 @@ const mergeIterators = (params) => {
     };
 };
 
-const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
-const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
-const json = (a) => replace(JSON.stringify(a));
-const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
-
 const state = { steady: false, f: (_) => Promise.resolve() };
 let timeout = null;
-const setF = (f) => {
-    state.f = f;
-};
-const aggregate = (db, streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
-    const req = {
-        aggregate: coll.collectionName,
-        pipeline: input,
-        cursor: {},
-        ...(snapshot && { readConcern: { level: 'snapshot' } }),
-    };
-    if (timeout !== null) {
-        clearTimeout(timeout);
-        timeout = null;
-    }
-    log('exec', streamName, req);
-    const start2 = Date.now();
-    return db.then(d => d.command(req)).then(result => {
-        log('prepare', streamName, Date.now() - start);
-        log('prepare2', streamName, start2 - start);
-        const r = result;
-        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('"$$CLUSTER_TIME"', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
-        if (!state.steady) {
-            if (timeout !== null)
-                throw new Error('timeout should be null');
-            timeout = setTimeout(() => {
-                state.steady = true;
-                console.log('steady');
-            }, 10000);
-        }
-        return r;
-    }, err => {
-        log('err', req, err);
-        throw new Error(err);
-    });
-});
-
 const firstWorksMerge = (iters) => {
     const iterator = () => {
         const results = iters.map(iter => iter());
         const sources = { ...results };
         return mergeIterators({
             sources,
-            interrupt: key => state.steady
+            interrupt: key => false,
+            hooks: {
+                start: (frame, result) => {
+                    if (!frame.info.job)
+                        return;
+                    if (timeout !== null) {
+                        clearTimeout(timeout);
+                        timeout = null;
+                    }
+                    result.next.then(() => {
+                        if (!frame.info.job)
+                            return;
+                        if (!state.steady) {
+                            if (timeout !== null)
+                                clearTimeout(timeout);
+                            timeout = setTimeout(() => {
+                                state.steady = true;
+                                console.log('steady');
+                            }, 2000);
+                        }
+                    });
+                },
+            },
         });
     };
     return iterator;
@@ -1510,10 +1493,7 @@ const $insertX = (out, expr, map, ext, extExpr) => {
         teardown: c => c({
             collection: out,
             method: 'updateMany',
-            params: [
-                filter,
-                [{ $set: { deletedAt: '$$NOW', touchedAt: '$$CLUSTER_TIME' } }],
-            ],
+            params: [filter, [{ $set: { deletedAt: '$$NOW', touchedAt: '$$CLUSTER_TIME' } }]],
         }),
         raw: () => {
             const replacer = map(mergeObjects(expr, field(mergeExpr(extExpr, {
@@ -1544,6 +1524,32 @@ const $insertPart = (out, ext) => {
 const $insert = (out) => $insertPart(out, {});
 const assertNotNull = (expr) => expr;
 
+const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
+const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
+const json = (a) => replace(JSON.stringify(a));
+const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
+
+const aggregate = (db, streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
+    const req = {
+        aggregate: coll.collectionName,
+        pipeline: input,
+        cursor: {},
+        ...(snapshot && { readConcern: { level: 'snapshot' } }),
+    };
+    log('exec', streamName, req);
+    const start2 = Date.now();
+    return db.then(d => d.command(req)).then(result => {
+        log('prepare', streamName, Date.now() - start);
+        log('prepare2', streamName, start2 - start);
+        const r = result;
+        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('"$$CLUSTER_TIME"', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
+        return r;
+    }, err => {
+        log('err', req, err);
+        throw new Error(err);
+    });
+});
+
 const addTeardown = (it, tr) => {
     if (!tr)
         return it;
@@ -1571,7 +1577,7 @@ async function getLastCommittedTs(adminDb) {
     const st = await adminDb.command({ replSetGetStatus: 1 });
     return st?.optimes?.lastCommittedOpTime?.ts ?? null;
 }
-async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000, } = {}) {
+async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000 } = {}) {
     const adminDb = db.client.db('admin');
     const deadline = Date.now() + timeoutMs;
     while (true) {
@@ -1579,7 +1585,7 @@ async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000
         if (stable && stable.comp(oplogTs) >= 0)
             return;
         if (Date.now() > deadline) {
-            throw new Error("Timed out waiting for stable timestamp to reach oplog event time");
+            throw new Error('Timed out waiting for stable timestamp to reach oplog event time');
         }
         await sleep(pollMs);
     }
@@ -1601,29 +1607,37 @@ async function* tailOplog(db, opts) {
         try {
             for await (const doc of cursor) {
                 lastTs = doc.ts;
-                if (doc.op === 'i') {
-                    yield { ns: doc.ns, fields: new Set(Object.keys(doc.o)), doc };
+                if (doc.op === 'i' || '_id' in doc.o) {
+                    const fields = new Set(Object.keys(doc.o));
+                    fields.delete('_id');
+                    yield { fields, doc, changeTouched: doc.o['touchedAt'] instanceof MaxKey };
                 }
                 else {
+                    let changeTouched = false;
                     if (doc.o['$v'] !== 2) {
-                        throw new Error(`Expected update with $v: 2, got ${JSON.stringify(doc.o)}`);
+                        throw new Error(`Expected update with $v: 2, got ${JSON.stringify(doc)}`);
                     }
                     const updatedFields = [];
                     const diff = doc.o['diff'];
                     for (const updateOp in diff) {
                         if (['u', 'i', 'd'].includes(updateOp)) {
                             updatedFields.push(...Object.keys(diff[updateOp]));
+                            if (diff[updateOp]['touchedAt'] instanceof MaxKey) {
+                                changeTouched = true;
+                            }
                         }
                         else if (updateOp.startsWith('s')) {
                             updatedFields.push(updateOp.slice(1));
                         }
                     }
-                    yield { ns: doc.ns, fields: new Set(updatedFields), doc };
+                    yield { fields: new Set(updatedFields), doc, changeTouched };
                 }
             }
         }
         catch (e) {
-            log('oplog loop error', e);
+            log('oplog loop error, notifying watchers and reopening');
+            console.error(e);
+            yield null;
         }
         finally {
             log('oplog loop ended');
@@ -1634,15 +1648,76 @@ async function* tailOplog(db, opts) {
 }
 const watchers = new Map();
 let running = false;
+const makePromise = () => {
+    let resolve = () => { };
+    let promise = new Promise(r => (resolve = r));
+    return { promise, resolve };
+};
 const loop = async (db) => {
     log('starting oplog loop');
-    for await (const { ns, fields, doc } of tailOplog(db, {})) {
-        const m = watchers.get(ns);
-        if (!m)
+    let notify = makePromise();
+    let batch = [];
+    const last = db.collection('__last');
+    const run = async () => {
+        for await (const event of tailOplog(db, {
+            since: (await last.findOne({ _id: 'oplog' }))?.ts,
+        })) {
+            if (event?.fields.size === 0)
+                continue;
+            batch = event && batch ? [...batch, event] : null;
+            notify.resolve();
+        }
+    };
+    run();
+    const iter = async function* () {
+        while (true) {
+            await notify.promise;
+            const b = batch;
+            if (b?.length) {
+                last
+                    .updateOne({ _id: 'oplog' }, { $set: { ts: b[b.length - 1].doc.ts } }, { upsert: true })
+                    .catch(() => { });
+            }
+            batch = [];
+            notify = makePromise();
+            yield b;
+        }
+    };
+    for await (const events of iter()) {
+        if (!events) {
+            log('notifying watchers of oplog loop restart');
+            for (const m of watchers.values()) {
+                for (const { cb } of m.values()) {
+                    cb(null);
+                }
+            }
             continue;
-        for (const { cb, keys } of m.values()) {
-            if (!keys || keys.some(k => fields.has(k))) {
-                cb(doc);
+        }
+        const groups = Object.groupBy(events.filter(e => e.changeTouched), ev => ev.doc.ns);
+        for (const [ns, evs] of Object.entries(groups)) {
+            if (!evs)
+                continue;
+            const [dbName, collName] = ns.split('.');
+            if (dbName !== db.databaseName)
+                continue;
+            const coll = db.collection(collName);
+            coll
+                .bulkWrite(evs.map((e) => ({
+                updateOne: {
+                    filter: { _id: e.doc.o['_id'] ?? e.doc.o2?._id },
+                    update: { $set: { touchedAt: e.doc.ts } },
+                },
+            })))
+                .catch(() => { });
+        }
+        for (const { fields, doc } of events) {
+            const m = watchers.get(doc.ns);
+            if (!m)
+                continue;
+            for (const { cb, keys } of m.values()) {
+                if (!keys || keys.some(k => fields.has(k))) {
+                    cb(doc);
+                }
             }
         }
     }
@@ -1668,7 +1743,7 @@ const makeWatchStream = ({ collection, projection: p, hardMatch: m }, streamName
     const projection = { ...(p ? mapExactToObject(p, v => v) : {}), deletedAt: 1 };
     let resolve = (_) => { };
     const promise = new Promise(r => (resolve = r));
-    const close = register(collection, p ? Object.keys(projection) : null, (doc) => {
+    const close = register(collection, p ? Object.keys(projection) : null, doc => {
         log(streamName, 'change detected', doc);
         resolve(doc);
         close();
@@ -1677,9 +1752,10 @@ const makeWatchStream = ({ collection, projection: p, hardMatch: m }, streamName
         tryNext: async () => {
             const doc = await promise;
             const start = Date.now();
-            await waitUntilStablePast(collection.s.db, doc.ts);
+            if (doc)
+                await waitUntilStablePast(collection.s.db, doc.ts);
             log(streamName, 'stable past took', Date.now() - start);
-            return doc;
+            return doc ?? {};
         },
         close: async () => close(),
     };
@@ -2119,4 +2195,4 @@ const executes = (view, input, needs) => {
 };
 const single = (view, needs = {}) => pipe(input => executes(view, input, needs), emptyDelta(), concatDelta, emptyDelta);
 
-export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, Field, Machine, add, and, anyElementTrue, array, ceil, comp, concat$1 as concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first$1 as first, firstSure, floor, from, func, getWhenMatched, getWhenMatchedForMerge, gt, gte, inArray, isArray, ite, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, not, notNull, now, or, pair, prepare, rand, range, regex, root, set, setF, setField, single, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };
+export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, Field, Machine, add, and, anyElementTrue, array, ceil, comp, concat$1 as concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first$1 as first, firstSure, floor, from, func, getWhenMatched, getWhenMatchedForMerge, gt, gte, inArray, isArray, ite, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, not, notNull, now, or, pair, prepare, rand, range, regex, root, set, setField, single, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };