@omegup/msync 0.1.18 → 0.1.20

package/index.d.ts

@@ -730,8 +730,12 @@ declare const $and: Combiner;
 declare const $nor: Combiner;
 declare const $or: Combiner;
 
+declare const setF: (f: ({ input }: {
+    input: any;
+}) => Promise<void>) => void;
+
 declare const enablePreAndPostImages: <T extends doc>(coll: Collection<T>) => Promise<Document>;
 declare const prepare: (testName?: string) => Promise<MongoClient$1>;
 declare const makeCol: <T extends ID>(docs: readonly OptionalUnlessRequiredId<T>[], database: Db, name?: string) => Promise<Collection<T>>;
 
-export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, type Accumulators, type Arr, type AsLiteral, type Delta, type DeltaAccumulator, type DeltaAccumulators, type ExactKeys, Expr, type ExprHKT, type Exprs, type ExprsExact, type ExprsExactHKT, type ExprsPart, Field, type ID, type Loose, Machine, type Merge, type MergeArgs, type MergeInto, type MergeMapOArgs, type Model, type MongoTypeNames, type N, type NoRaw, type NullToOBJ, type O, type OPick, type OPickD, type Patch, type RONoRaw, type RORec, type RawStages, type Rec, type Replace, type SnapshotStreamExecutionResult, type StrKey, type Strict, type TS, Type, type WriteonlyCollection, add, and, anyElementTrue, array, ceil, comp, concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, type doc, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first, firstSure, floor, from, func, getWhenMatched, getWhenMatchedForMerge, gt, gte, inArray, isArray, ite, type jsonPrim, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, not, type notArr, notNull, now, or, pair, prepare, rand, range, regex, root, set, setField, single, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };
+export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, type Accumulators, type Arr, type AsLiteral, type Delta, type DeltaAccumulator, type DeltaAccumulators, type ExactKeys, Expr, type ExprHKT, type Exprs, type ExprsExact, type ExprsExactHKT, type ExprsPart, Field, type ID, type Loose, Machine, type Merge, type MergeArgs, type MergeInto, type MergeMapOArgs, type Model, type MongoTypeNames, type N, type NoRaw, type NullToOBJ, type O, type OPick, type OPickD, type Patch, type RONoRaw, type RORec, type RawStages, type Rec, type Replace, type SnapshotStreamExecutionResult, type StrKey, type Strict, type TS, Type, type WriteonlyCollection, add, and, anyElementTrue, array, ceil, comp, concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, type doc, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first, firstSure, floor, from, func, getWhenMatched, getWhenMatchedForMerge, gt, gte, inArray, isArray, ite, type jsonPrim, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, not, type notArr, notNull, now, or, pair, prepare, rand, range, regex, root, set, setF, setField, single, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };

package/index.esm.js

@@ -1,7 +1,7 @@
 import crypto$1 from 'crypto';
 import { canonicalize } from 'json-canonicalize';
 import { SynchronousPromise } from 'synchronous-promise';
-import { UUID, MongoClient } from 'mongodb';
+import { MongoClient, UUID } from 'mongodb';
 import { writeFile } from 'fs/promises';
 
 const asExprRaw = (raw) => ({ get: () => raw });
@@ -1336,9 +1336,12 @@ const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
 const json = (a) => replace(JSON.stringify(a));
 const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
 
-const state = { steady: false };
+const state = { steady: false, f: (_) => Promise.resolve() };
 let timeout = null;
-const aggregate = (streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
+const setF = (f) => {
+    state.f = f;
+};
+const aggregate = (db, streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
     const req = {
         aggregate: coll.collectionName,
         pipeline: input,
@@ -1350,9 +1353,12 @@ const aggregate = (streamName, input, snapshot = true, start = Date.now()) => in
         timeout = null;
     }
     log('exec', streamName, req);
-    return coll.s.db.command(req).then(result => {
+    const start2 = Date.now();
+    return db.then(d => d.command(req)).then(result => {
+        log('prepare', streamName, Date.now() - start);
+        log('prepare2', streamName, start2 - start);
         const r = result;
-        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('$$CLUSTER_TIME', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
+        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('"$$CLUSTER_TIME"', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
         if (!state.steady) {
             if (timeout !== null)
                 throw new Error('timeout should be null');
@@ -1555,67 +1561,129 @@ const addTeardown = (it, tr) => {
     };
 };
 
-const changeKeys = ['fullDocument', 'fullDocumentBeforeChange'];
-const subQ = (a, f) => ({ raw: g => a.raw(g.with(f)) });
-const makeWatchStream = (db, { collection, projection: p, hardMatch: m }, startAt, streamName) => {
-    const projection = p ? { ...mapExactToObject(p, v => v), deletedAt: 1 } : 1;
-    const pipeline = [];
-    if (m) {
-        const q = $or(...changeKeys.map((k) => subQ(m, root().of(k))));
-        if (q)
-            pipeline.push({
-                $match: {
-                    $or: [
-                        q.raw(root()),
-                        Object.fromEntries(changeKeys.map(k => [k, null])),
-                    ],
-                },
-            });
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+const getCurrentTimestamp = async (db) => {
+    const adminDb = db.admin();
+    const serverStatus = await adminDb.command({ serverStatus: 1 });
+    return serverStatus['operationTime'];
+};
+async function getLastCommittedTs(adminDb) {
+    const st = await adminDb.command({ replSetGetStatus: 1 });
+    return st?.optimes?.lastCommittedOpTime?.ts ?? null;
+}
+async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000, } = {}) {
+    const adminDb = db.client.db('admin');
+    const deadline = Date.now() + timeoutMs;
+    while (true) {
+        const stable = await getLastCommittedTs(adminDb);
+        if (stable && stable.comp(oplogTs) >= 0)
+            return;
+        if (Date.now() > deadline) {
+            throw new Error("Timed out waiting for stable timestamp to reach oplog event time");
+        }
+        await sleep(pollMs);
     }
-    pipeline.push({
-        $project: {
-            _id: 1,
-            fullDocument: projection,
-            fullDocumentBeforeChange: projection,
-            documentKey: 1,
-            clusterTime: 1,
-        },
-    });
-    pipeline.push({
-        $match: {
-            clusterTime: { $gt: startAt },
-            $or: [
-                {
-                    $expr: {
-                        $ne: [
-                            { $mergeObjects: ['$fullDocument', { touchedAt: null }] },
-                            { $mergeObjects: ['$fullDocumentBeforeChange', { touchedAt: null }] },
-                        ],
-                    },
-                },
-                Object.fromEntries(changeKeys.map(k => [k, null])),
-            ],
-        },
+}
+async function* tailOplog(db, opts) {
+    let lastTs = opts.since ?? (await getCurrentTimestamp(db));
+    const reopenDelayMs = opts.reopenDelayMs ?? 250;
+    const coll = db.client.db('local').collection('oplog.rs');
+    while (true) {
+        const cursor = coll.find({
+            ts: { $gt: lastTs },
+            ns: RegExp(`^${db.namespace}\\.(?!tmp_)(?!__).*(?<!_snapshot)$`),
+            op: { $in: ['i', 'u'] },
+        }, {
+            tailable: true,
+            awaitData: true,
+            noCursorTimeout: true,
+        });
+        try {
+            for await (const doc of cursor) {
+                lastTs = doc.ts;
+                if (doc.op === 'i') {
+                    yield { ns: doc.ns, fields: new Set(Object.keys(doc.o)), doc };
+                }
+                else {
+                    if (doc.o['$v'] !== 2) {
+                        throw new Error(`Expected update with $v: 2, got ${JSON.stringify(doc.o)}`);
+                    }
+                    const updatedFields = [];
+                    const diff = doc.o['diff'];
+                    for (const updateOp in diff) {
+                        if (['u', 'i', 'd'].includes(updateOp)) {
+                            updatedFields.push(...Object.keys(diff[updateOp]));
+                        }
+                        else if (updateOp.startsWith('s')) {
+                            updatedFields.push(updateOp.slice(1));
+                        }
+                    }
+                    yield { ns: doc.ns, fields: new Set(updatedFields), doc };
+                }
+            }
+        }
+        catch (e) {
+            log('oplog loop error', e);
+        }
+        finally {
+            log('oplog loop ended');
+            await cursor.close().catch(() => { });
+        }
+        await sleep(reopenDelayMs);
+    }
+}
+const watchers = new Map();
+let running = false;
+const loop = async (db) => {
+    log('starting oplog loop');
+    for await (const { ns, fields, doc } of tailOplog(db, {})) {
+        log('oplog event', ns, doc.op, [...fields]);
+        const m = watchers.get(ns);
+        if (!m)
+            continue;
+        for (const { cb, keys } of m.values()) {
+            if (!keys || keys.some(k => fields.has(k))) {
+                cb(doc);
+            }
+        }
+    }
+};
+const register = (coll, keys, cb) => {
+    const ns = coll.namespace;
+    let m = watchers.get(ns);
+    if (!m)
+        watchers.set(ns, (m = new Map()));
+    const id = crypto.randomUUID();
+    m.set(id, { cb, keys });
+    if (!running) {
+        running = true;
+        loop(coll.s.db);
+    }
+    return () => {
+        m.delete(id);
+        if (m.size === 0)
+            watchers.delete(ns);
+    };
+};
+const makeWatchStream = ({ collection, projection: p, hardMatch: m }, streamName) => {
+    const projection = { ...(p ? mapExactToObject(p, v => v) : {}), deletedAt: 1 };
+    let resolve = (_) => { };
+    const promise = new Promise(r => (resolve = r));
+    const close = register(collection, p ? Object.keys(projection) : null, (doc) => {
+        log(streamName, 'change detected', doc);
+        resolve(doc);
+        close();
     });
-    pipeline.push({
-        $project: {
-            _id: 1,
+    return {
+        tryNext: async () => {
+            const doc = await promise;
+            const start = Date.now();
+            await waitUntilStablePast(collection.s.db, doc.ts);
+            log(streamName, 'stable past took', Date.now() - start);
+            return doc;
         },
-    });
-    const stream = db.collection(collection.collectionName).watch(pipeline, {
-        fullDocument: 'required',
-        fullDocumentBeforeChange: 'required',
-        startAtOperationTime: startAt,
-    });
-    const tryNext = async () => {
-        const doc = await stream.tryNext();
-        if (doc)
-            await new Promise(resolve => setTimeout(resolve, 100));
-        if (doc)
-            log('detected', streamName, collection.collectionName, doc);
-        return doc;
+        close: async () => close(),
     };
-    return { tryNext, close: () => stream.close() };
 };
 
 const actions = {
@@ -1648,9 +1716,54 @@ const getFirstStages = (view, needs) => {
     return { firstStages, hardMatch };
 };
 
+require('dotenv').config();
+const uri = process.env['MONGO_URL'];
+
+const enablePreAndPostImages = (coll) => coll.s.db.command({
+    collMod: coll.collectionName,
+    changeStreamPreAndPostImages: { enabled: true },
+});
+const prepare = async (testName) => {
+    const client = new MongoClient(uri, testName ? { monitorCommands: true } : {});
+    if (testName) {
+        const handler = (c) => {
+            writeFile(`./out/${testName}.log`, JSON.stringify(c.command) + ',\n', { flag: 'w' });
+        };
+        client.on('commandStarted', handler);
+        client.on('commandSucceeded', handler);
+    }
+    await client.connect();
+    await client.db('admin').command({
+        setClusterParameter: {
+            changeStreamOptions: {
+                preAndPostImages: { expireAfterSeconds: 60 },
+            },
+        },
+    });
+    return client;
+};
+const makeCol = async (docs, database, name) => {
+    if (!name) {
+        (name = crypto.randomUUID());
+    }
+    try {
+        const col = await database.createCollection(name, {
+            changeStreamPreAndPostImages: { enabled: true },
+        });
+        if (docs.length)
+            await col.insertMany([...docs]);
+        return col;
+    }
+    catch {
+        return database.collection(name);
+    }
+};
+
 const streamNames = {};
 const executes$2 = (view, input, streamName, skip = false, after, needs = {}) => {
     const { collection, projection, match } = view;
+    const client = prepare();
+    const pdb = client.then(cl => cl.db(collection.dbName));
     const { firstStages, hardMatch } = getFirstStages(view, needs);
     const db = collection.s.db, coll = collection.collectionName;
     const hash = crypto$1
@@ -1673,10 +1786,22 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
         : {}).catch(e => e.code == 86 || Promise.reject(e));
     const last = db.collection('__last');
     const snapshotCollection = db.collection(coll + '_' + streamName + '_snapshot');
+    createIndex(snapshotCollection, { before: 1 }, {
+        partialFilterExpression: { before: null },
+        name: 'before_' + new UUID().toString('base64'),
+    });
     createIndex(snapshotCollection, { updated: 1 }, {
         partialFilterExpression: { updated: true },
         name: 'updated_' + new UUID().toString('base64'),
     });
+    createIndex(snapshotCollection, { updated: 1, after: 1, before: 1 }, {
+        partialFilterExpression: { updated: true, after: null, before: null },
+        name: 'updated_nulls_' + new UUID().toString('base64'),
+    });
+    createIndex(snapshotCollection, { updated: 1, after: 1 }, {
+        partialFilterExpression: { updated: true, after: null },
+        name: 'updated_no_after_' + new UUID().toString('base64'),
+    });
     createIndex(snapshotCollection, { updated: 1 }, {
         partialFilterExpression: { updated: true, after: null, before: null },
         name: 'updated_nulls_' + new UUID().toString('base64'),
@@ -1715,7 +1840,7 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
             return next(step2, 'get last update');
         };
         const step2 = () => Promise.all([
-            last.findOne({ _id: streamName, data }),
+            last.findOne({ _id: streamName, data, job: null }),
             last.findOne({ _id: streamName }),
         ]).then(ts => next(step2_5(ts), ts[0]
             ? `no teardown to handle, starting at ${ts[0].ts}`
@@ -1767,40 +1892,46 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
                 whenMatched: link().with($replaceWith_(ite(eq(root().of('before').expr())(ctx()('new').of('after').expr()), root().expr(), mergeObjects(root().expr(), ctx()('new').expr())))).stages,
                 whenNotMatched: 'insert',
             })).stages;
-            const r = await aggregate(streamName, c => c({ coll: collection, input: cloneIntoNew }));
-            await snapshotCollection.deleteMany({ updated: true, after: null, before: null });
+            const r = await aggregate(pdb, streamName, c => c({ coll: collection, input: cloneIntoNew }));
+            const start = Date.now();
+            const res = await snapshotCollection.deleteMany({ updated: true, after: null, before: null });
+            log('deleting from cloned into new collection', Date.now() - start, res, `db['${snapshotCollection.collectionName}'].deleteMany({ updated: true, after: null, before: null })`);
             return next(step4({ result: r, ts: lastTS?.ts }), 'run the aggregation');
         };
-        const makeStream = (startAt) => makeWatchStream(db, view, startAt, streamName);
+        const makeStream = () => makeWatchStream(view, streamName);
         const step4 = ({ result, ts }) => async () => {
             const start = Date.now();
-            await snapshotCollection.updateMany({ before: null }, { $set: { before: null } });
-            const stages = finalInput.raw(ts === undefined);
-            const aggResult = await aggregate(streamName, c => c({
+            log('snapshot', streamName, 'ensure before null', Date.now() - start);
+            const first = ts === undefined;
+            const stages = finalInput.raw(first);
+            await last.updateOne({ _id: streamName }, { $set: { job: 1 } }, { upsert: true });
+            const stream = makeStream();
+            const aggResult = await aggregate(pdb, streamName, c => c({
                 coll: snapshotCollection,
                 input: link()
                     .with($match_(root().of('updated').has($eq(true))))
+                    .with($set_(set()({
+                    before: [
+                        'before',
+                        to($ifNull(root().of('before').expr(), nil)),
+                    ],
+                })))
                     .with(input.delta)
                     .with(stages).stages,
             }), false, start);
-            const stream = makeStream(result.cursor.atClusterTime);
             const nextRes = stream.tryNext();
-            const intoColl = stages.at(-1).$merge.into.coll;
-            const startx = Date.now();
-            await db
-                .collection(intoColl)
-                .countDocuments({ touchedAt: { $gte: result.cursor.atClusterTime } })
-                .then(count => log(`documents updated ${intoColl}`, count, 'took', Date.now() - startx));
-            return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream, nextRes }), 'remove handled deleted updated', () => stream.close());
+            stages.at(-1).$merge.into.coll;
+            return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream, nextRes, first }), 'remove handled deleted updated', () => stream.close());
         };
         const step5 = (l) => async () => {
-            log(`remove handled deleted updated db['${snapshotCollection.collectionName}'].deleteMany({ updated: true, after: null })`);
+            log(streamName, `remove handled deleted updated db['${snapshotCollection.collectionName}'].deleteMany({ updated: true, after: null })`);
             await snapshotCollection.deleteMany({ updated: true, after: null });
             log('removed handled deleted updated');
             return next(step6(l), 'update snapshot aggregation');
         };
         const step6 = (l) => async () => {
             log('update snapshot aggregation', `db['${snapshotCollection.collectionName}'].updateMany({ updated: true }, [ { $set: { updated: false, after: null, before: '$after' } } ])`);
+            const start = Date.now();
             await snapshotCollection.updateMany({ updated: true }, [
                 {
                     $set: {
@@ -1810,25 +1941,25 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
                     },
                 },
             ]);
-            log('updated snapshot aggregation');
+            log('updated snapshot aggregation', Date.now() - start);
             return next(step7(l), 'update __last');
         };
         const step7 = (l) => async () => {
-            await last.updateOne({ _id: streamName }, {
+            const start = Date.now();
+            const patch = {
                 $set: {
                     ts: l.ts,
-                    data,
+                    job: null,
                 },
-            }, { upsert: true });
+            };
+            if (l.ts)
+                patch.$set = data;
+            await last.updateOne({ _id: streamName }, patch, { upsert: true });
+            log('updated __last', Date.now() - start, `db['${last.collectionName}'].updateOne({ _id: '${streamName}' }, `, patch, `, { upsert: true })`);
             return step8(l);
         };
         const step8 = (l) => {
-            return nextData(l.aggResult.cursor.firstBatch)(() => l.nextRes
-                .catch((err) => {
-                log('restarting', err);
-                return { ts: null };
-            })
-                .then(doc => doc
+            return nextData(l.aggResult.cursor.firstBatch)(() => l.nextRes.then(doc => doc
                 ? next(step3({ _id: streamName, ts: l.ts }), 'restart')
                 : step8({ ...l, nextRes: l.stream.tryNext() })), 'wait for change');
         };
@@ -1858,6 +1989,8 @@ const executes$1 = (view, input, streamName, needs) => {
     else if (streamNames[streamName] != hash)
         throw new Error('streamName already used');
     const { collection, projection, hardMatch: pre, match } = view;
+    const client = prepare();
+    const pdb = client.then(cl => cl.db(collection.dbName));
     const removeNotYetSynchronizedFields = projection &&
         Object.values(mapExactToObject(projection, (_, k) => (needs[k] ?? k.startsWith('_')) ? root().of(k).has($exists(true)) : null));
     const hardMatch = removeNotYetSynchronizedFields
@@ -1939,14 +2072,14 @@ const executes$1 = (view, input, streamName, needs) => {
                 info: { debug: 'wait for clone into new collection', job: undefined },
             };
         };
-        const makeStream = (startAt) => makeWatchStream(db, view, startAt, streamName);
+        const makeStream = () => makeWatchStream(view, streamName);
         const step4 = (lastTS) => async () => {
             const raw = stages(lastTS).with(finalInput.raw(lastTS === null)).stages;
-            const aggResult = await aggregate(streamName, c => c({
+            const stream = makeStream();
+            const aggResult = await aggregate(pdb, streamName, c => c({
                 coll: collection,
                 input: raw,
             }));
-            const stream = makeStream(aggResult.cursor.atClusterTime);
             const nextRes = stream.tryNext();
             return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream, nextRes }), 'update __last', () => stream.close());
         };
@@ -1958,12 +2091,7 @@ const executes$1 = (view, input, streamName, needs) => {
             return {
                 data: l.aggResult.cursor.firstBatch,
                 info: { job: undefined, debug: 'wait for change' },
-                cont: withStop(() => l.nextRes
-                    .catch((err) => {
-                    log('restarting', err);
-                    return { ts: null };
-                })
-                    .then(doc => doc
+                cont: withStop(() => l.nextRes.then(doc => doc
                     ? next(step4({ _id: streamName, ts: l.ts }), 'restart')
                     : step8({ ...l, nextRes: l.stream.tryNext() }))),
             };
@@ -1992,47 +2120,4 @@ const executes = (view, input, needs) => {
 };
 const single = (view, needs = {}) => pipe(input => executes(view, input, needs), emptyDelta(), concatDelta, emptyDelta);
 
-require('dotenv').config();
-const uri = process.env['MONGO_URL'];
-
-const enablePreAndPostImages = (coll) => coll.s.db.command({
-    collMod: coll.collectionName,
-    changeStreamPreAndPostImages: { enabled: true },
-});
-const prepare = async (testName) => {
-    const client = new MongoClient(uri, testName ? { monitorCommands: true } : {});
-    if (testName) {
-        const handler = (c) => {
-            writeFile(`./out/${testName}.log`, JSON.stringify(c.command) + ',\n', { flag: 'w' });
-        };
-        client.on('commandStarted', handler);
-        client.on('commandSucceeded', handler);
-    }
-    await client.connect();
-    await client.db('admin').command({
-        setClusterParameter: {
-            changeStreamOptions: {
-                preAndPostImages: { expireAfterSeconds: 60 },
-            },
-        },
-    });
-    return client;
-};
-const makeCol = async (docs, database, name) => {
-    if (!name) {
-        (name = crypto.randomUUID());
-    }
-    try {
-        const col = await database.createCollection(name, {
-            changeStreamPreAndPostImages: { enabled: true },
-        });
-        if (docs.length)
-            await col.insertMany([...docs]);
-        return col;
-    }
-    catch {
-        return database.collection(name);
-    }
-};
-
-export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, Field, Machine, add, and, anyElementTrue, array, ceil, comp, concat$1 as concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first$1 as first, firstSure, floor, from, func, getWhenMatched, getWhenMatchedForMerge, gt, gte, inArray, isArray, ite, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, not, notNull, now, or, pair, prepare, rand, range, regex, root, set, setField, single, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };
+export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, Field, Machine, add, and, anyElementTrue, array, ceil, comp, concat$1 as concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first$1 as first, firstSure, floor, from, func, getWhenMatched, getWhenMatchedForMerge, gt, gte, inArray, isArray, ite, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, not, notNull, now, or, pair, prepare, rand, range, regex, root, set, setF, setField, single, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };

package/index.js

@@ -1338,9 +1338,12 @@ const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
 const json = (a) => replace(JSON.stringify(a));
 const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
 
-const state = { steady: false };
+const state = { steady: false, f: (_) => Promise.resolve() };
 let timeout = null;
-const aggregate = (streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
+const setF = (f) => {
+    state.f = f;
+};
+const aggregate = (db, streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
     const req = {
         aggregate: coll.collectionName,
         pipeline: input,
@@ -1352,9 +1355,12 @@ const aggregate = (streamName, input, snapshot = true, start = Date.now()) => in
         timeout = null;
     }
     log('exec', streamName, req);
-    return coll.s.db.command(req).then(result => {
+    const start2 = Date.now();
+    return db.then(d => d.command(req)).then(result => {
+        log('prepare', streamName, Date.now() - start);
+        log('prepare2', streamName, start2 - start);
         const r = result;
-        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('$$CLUSTER_TIME', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
+        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('"$$CLUSTER_TIME"', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
         if (!state.steady) {
             if (timeout !== null)
                 throw new Error('timeout should be null');
@@ -1557,67 +1563,129 @@ const addTeardown = (it, tr) => {
     };
 };
 
-const changeKeys = ['fullDocument', 'fullDocumentBeforeChange'];
-const subQ = (a, f) => ({ raw: g => a.raw(g.with(f)) });
-const makeWatchStream = (db, { collection, projection: p, hardMatch: m }, startAt, streamName) => {
-    const projection = p ? { ...mapExactToObject(p, v => v), deletedAt: 1 } : 1;
-    const pipeline = [];
-    if (m) {
-        const q = $or(...changeKeys.map((k) => subQ(m, root().of(k))));
-        if (q)
-            pipeline.push({
-                $match: {
-                    $or: [
-                        q.raw(root()),
-                        Object.fromEntries(changeKeys.map(k => [k, null])),
-                    ],
-                },
-            });
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+const getCurrentTimestamp = async (db) => {
+    const adminDb = db.admin();
+    const serverStatus = await adminDb.command({ serverStatus: 1 });
+    return serverStatus['operationTime'];
+};
+async function getLastCommittedTs(adminDb) {
+    const st = await adminDb.command({ replSetGetStatus: 1 });
+    return st?.optimes?.lastCommittedOpTime?.ts ?? null;
+}
+async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000, } = {}) {
+    const adminDb = db.client.db('admin');
+    const deadline = Date.now() + timeoutMs;
+    while (true) {
+        const stable = await getLastCommittedTs(adminDb);
+        if (stable && stable.comp(oplogTs) >= 0)
+            return;
+        if (Date.now() > deadline) {
+            throw new Error("Timed out waiting for stable timestamp to reach oplog event time");
+        }
+        await sleep(pollMs);
     }
-    pipeline.push({
-        $project: {
-            _id: 1,
-            fullDocument: projection,
-            fullDocumentBeforeChange: projection,
-            documentKey: 1,
-            clusterTime: 1,
-        },
-    });
-    pipeline.push({
-        $match: {
-            clusterTime: { $gt: startAt },
-            $or: [
-                {
-                    $expr: {
-                        $ne: [
-                            { $mergeObjects: ['$fullDocument', { touchedAt: null }] },
-                            { $mergeObjects: ['$fullDocumentBeforeChange', { touchedAt: null }] },
-                        ],
-                    },
-                },
-                Object.fromEntries(changeKeys.map(k => [k, null])),
-            ],
-        },
+}
+async function* tailOplog(db, opts) {
+    let lastTs = opts.since ?? (await getCurrentTimestamp(db));
+    const reopenDelayMs = opts.reopenDelayMs ?? 250;
+    const coll = db.client.db('local').collection('oplog.rs');
+    while (true) {
+        const cursor = coll.find({
+            ts: { $gt: lastTs },
+            ns: RegExp(`^${db.namespace}\\.(?!tmp_)(?!__).*(?<!_snapshot)$`),
+            op: { $in: ['i', 'u'] },
+        }, {
+            tailable: true,
+            awaitData: true,
+            noCursorTimeout: true,
+        });
+        try {
+            for await (const doc of cursor) {
+                lastTs = doc.ts;
+                if (doc.op === 'i') {
+                    yield { ns: doc.ns, fields: new Set(Object.keys(doc.o)), doc };
+                }
+                else {
+                    if (doc.o['$v'] !== 2) {
+                        throw new Error(`Expected update with $v: 2, got ${JSON.stringify(doc.o)}`);
+                    }
+                    const updatedFields = [];
+                    const diff = doc.o['diff'];
+                    for (const updateOp in diff) {
+                        if (['u', 'i', 'd'].includes(updateOp)) {
+                            updatedFields.push(...Object.keys(diff[updateOp]));
+                        }
+                        else if (updateOp.startsWith('s')) {
+                            updatedFields.push(updateOp.slice(1));
+                        }
+                    }
+                    yield { ns: doc.ns, fields: new Set(updatedFields), doc };
+                }
+            }
+        }
+        catch (e) {
+            log('oplog loop error', e);
+        }
+        finally {
+            log('oplog loop ended');
+            await cursor.close().catch(() => { });
+        }
+        await sleep(reopenDelayMs);
+    }
+}
+const watchers = new Map();
+let running = false;
+const loop = async (db) => {
+    log('starting oplog loop');
+    for await (const { ns, fields, doc } of tailOplog(db, {})) {
+        log('oplog event', ns, doc.op, [...fields]);
+        const m = watchers.get(ns);
+        if (!m)
+            continue;
+        for (const { cb, keys } of m.values()) {
+            if (!keys || keys.some(k => fields.has(k))) {
+                cb(doc);
+            }
+        }
+    }
+};
+const register = (coll, keys, cb) => {
+    const ns = coll.namespace;
+    let m = watchers.get(ns);
+    if (!m)
+        watchers.set(ns, (m = new Map()));
+    const id = crypto.randomUUID();
+    m.set(id, { cb, keys });
+    if (!running) {
+        running = true;
+        loop(coll.s.db);
+    }
+    return () => {
+        m.delete(id);
+        if (m.size === 0)
+            watchers.delete(ns);
+    };
+};
+const makeWatchStream = ({ collection, projection: p, hardMatch: m }, streamName) => {
+    const projection = { ...(p ? mapExactToObject(p, v => v) : {}), deletedAt: 1 };
+    let resolve = (_) => { };
+    const promise = new Promise(r => (resolve = r));
+    const close = register(collection, p ? Object.keys(projection) : null, (doc) => {
+        log(streamName, 'change detected', doc);
+        resolve(doc);
+        close();
     });
-    pipeline.push({
-        $project: {
-            _id: 1,
+    return {
+        tryNext: async () => {
+            const doc = await promise;
+            const start = Date.now();
+            await waitUntilStablePast(collection.s.db, doc.ts);
+            log(streamName, 'stable past took', Date.now() - start);
+            return doc;
         },
-    });
-    const stream = db.collection(collection.collectionName).watch(pipeline, {
-        fullDocument: 'required',
-        fullDocumentBeforeChange: 'required',
-        startAtOperationTime: startAt,
-    });
-    const tryNext = async () => {
-        const doc = await stream.tryNext();
-        if (doc)
-            await new Promise(resolve => setTimeout(resolve, 100));
-        if (doc)
-            log('detected', streamName, collection.collectionName, doc);
-        return doc;
+        close: async () => close(),
     };
-    return { tryNext, close: () => stream.close() };
 };
 
 const actions = {
@@ -1650,9 +1718,54 @@ const getFirstStages = (view, needs) => {
     return { firstStages, hardMatch };
 };
 
+require('dotenv').config();
+const uri = process.env['MONGO_URL'];
+
+const enablePreAndPostImages = (coll) => coll.s.db.command({
+    collMod: coll.collectionName,
+    changeStreamPreAndPostImages: { enabled: true },
+});
+const prepare = async (testName) => {
+    const client = new mongodb.MongoClient(uri, testName ? { monitorCommands: true } : {});
+    if (testName) {
+        const handler = (c) => {
+            promises.writeFile(`./out/${testName}.log`, JSON.stringify(c.command) + ',\n', { flag: 'w' });
+        };
+        client.on('commandStarted', handler);
+        client.on('commandSucceeded', handler);
+    }
+    await client.connect();
+    await client.db('admin').command({
+        setClusterParameter: {
+            changeStreamOptions: {
+                preAndPostImages: { expireAfterSeconds: 60 },
+            },
+        },
+    });
+    return client;
+};
+const makeCol = async (docs, database, name) => {
+    if (!name) {
+        (name = crypto.randomUUID());
+    }
+    try {
+        const col = await database.createCollection(name, {
+            changeStreamPreAndPostImages: { enabled: true },
+        });
+        if (docs.length)
+            await col.insertMany([...docs]);
+        return col;
+    }
+    catch {
+        return database.collection(name);
+    }
+};
+
 const streamNames = {};
 const executes$2 = (view, input, streamName, skip = false, after, needs = {}) => {
     const { collection, projection, match } = view;
+    const client = prepare();
+    const pdb = client.then(cl => cl.db(collection.dbName));
     const { firstStages, hardMatch } = getFirstStages(view, needs);
     const db = collection.s.db, coll = collection.collectionName;
     const hash = crypto$1
@@ -1675,10 +1788,22 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
         : {}).catch(e => e.code == 86 || Promise.reject(e));
     const last = db.collection('__last');
     const snapshotCollection = db.collection(coll + '_' + streamName + '_snapshot');
+    createIndex(snapshotCollection, { before: 1 }, {
+        partialFilterExpression: { before: null },
+        name: 'before_' + new mongodb.UUID().toString('base64'),
+    });
     createIndex(snapshotCollection, { updated: 1 }, {
         partialFilterExpression: { updated: true },
         name: 'updated_' + new mongodb.UUID().toString('base64'),
     });
+    createIndex(snapshotCollection, { updated: 1, after: 1, before: 1 }, {
+        partialFilterExpression: { updated: true, after: null, before: null },
+        name: 'updated_nulls_' + new mongodb.UUID().toString('base64'),
+    });
+    createIndex(snapshotCollection, { updated: 1, after: 1 }, {
+        partialFilterExpression: { updated: true, after: null },
+        name: 'updated_no_after_' + new mongodb.UUID().toString('base64'),
+    });
     createIndex(snapshotCollection, { updated: 1 }, {
         partialFilterExpression: { updated: true, after: null, before: null },
         name: 'updated_nulls_' + new mongodb.UUID().toString('base64'),
@@ -1717,7 +1842,7 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
             return next(step2, 'get last update');
         };
         const step2 = () => Promise.all([
-            last.findOne({ _id: streamName, data }),
+            last.findOne({ _id: streamName, data, job: null }),
             last.findOne({ _id: streamName }),
         ]).then(ts => next(step2_5(ts), ts[0]
             ? `no teardown to handle, starting at ${ts[0].ts}`
@@ -1769,40 +1894,46 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
                 whenMatched: link().with($replaceWith_(ite(eq(root().of('before').expr())(ctx()('new').of('after').expr()), root().expr(), mergeObjects(root().expr(), ctx()('new').expr())))).stages,
                 whenNotMatched: 'insert',
             })).stages;
-            const r = await aggregate(streamName, c => c({ coll: collection, input: cloneIntoNew }));
-            await snapshotCollection.deleteMany({ updated: true, after: null, before: null });
+            const r = await aggregate(pdb, streamName, c => c({ coll: collection, input: cloneIntoNew }));
+            const start = Date.now();
+            const res = await snapshotCollection.deleteMany({ updated: true, after: null, before: null });
+            log('deleting from cloned into new collection', Date.now() - start, res, `db['${snapshotCollection.collectionName}'].deleteMany({ updated: true, after: null, before: null })`);
             return next(step4({ result: r, ts: lastTS?.ts }), 'run the aggregation');
         };
-        const makeStream = (startAt) => makeWatchStream(db, view, startAt, streamName);
+        const makeStream = () => makeWatchStream(view, streamName);
         const step4 = ({ result, ts }) => async () => {
             const start = Date.now();
-            await snapshotCollection.updateMany({ before: null }, { $set: { before: null } });
-            const stages = finalInput.raw(ts === undefined);
-            const aggResult = await aggregate(streamName, c => c({
+            log('snapshot', streamName, 'ensure before null', Date.now() - start);
+            const first = ts === undefined;
+            const stages = finalInput.raw(first);
+            await last.updateOne({ _id: streamName }, { $set: { job: 1 } }, { upsert: true });
+            const stream = makeStream();
+            const aggResult = await aggregate(pdb, streamName, c => c({
                 coll: snapshotCollection,
                 input: link()
                     .with($match_(root().of('updated').has($eq(true))))
+                    .with($set_(set()({
+                    before: [
+                        'before',
+                        to($ifNull(root().of('before').expr(), nil)),
+                    ],
+                })))
                     .with(input.delta)
                     .with(stages).stages,
             }), false, start);
-            const stream = makeStream(result.cursor.atClusterTime);
             const nextRes = stream.tryNext();
-            const intoColl = stages.at(-1).$merge.into.coll;
-            const startx = Date.now();
-            await db
-                .collection(intoColl)
-                .countDocuments({ touchedAt: { $gte: result.cursor.atClusterTime } })
-                .then(count => log(`documents updated ${intoColl}`, count, 'took', Date.now() - startx));
-            return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream, nextRes }), 'remove handled deleted updated', () => stream.close());
+            stages.at(-1).$merge.into.coll;
+            return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream, nextRes, first }), 'remove handled deleted updated', () => stream.close());
         };
         const step5 = (l) => async () => {
-            log(`remove handled deleted updated db['${snapshotCollection.collectionName}'].deleteMany({ updated: true, after: null })`);
+            log(streamName, `remove handled deleted updated db['${snapshotCollection.collectionName}'].deleteMany({ updated: true, after: null })`);
             await snapshotCollection.deleteMany({ updated: true, after: null });
             log('removed handled deleted updated');
             return next(step6(l), 'update snapshot aggregation');
         };
         const step6 = (l) => async () => {
             log('update snapshot aggregation', `db['${snapshotCollection.collectionName}'].updateMany({ updated: true }, [ { $set: { updated: false, after: null, before: '$after' } } ])`);
+            const start = Date.now();
             await snapshotCollection.updateMany({ updated: true }, [
                 {
                     $set: {
@@ -1812,25 +1943,25 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
                     },
                 },
             ]);
-            log('updated snapshot aggregation');
+            log('updated snapshot aggregation', Date.now() - start);
             return next(step7(l), 'update __last');
         };
         const step7 = (l) => async () => {
-            await last.updateOne({ _id: streamName }, {
+            const start = Date.now();
+            const patch = {
                 $set: {
                     ts: l.ts,
-                    data,
+                    job: null,
                 },
-            }, { upsert: true });
+            };
+            if (l.ts)
+                patch.$set = data;
+            await last.updateOne({ _id: streamName }, patch, { upsert: true });
+            log('updated __last', Date.now() - start, `db['${last.collectionName}'].updateOne({ _id: '${streamName}' }, `, patch, `, { upsert: true })`);
             return step8(l);
         };
         const step8 = (l) => {
-            return nextData(l.aggResult.cursor.firstBatch)(() => l.nextRes
-                .catch((err) => {
-                log('restarting', err);
-                return { ts: null };
-            })
-                .then(doc => doc
+            return nextData(l.aggResult.cursor.firstBatch)(() => l.nextRes.then(doc => doc
                 ? next(step3({ _id: streamName, ts: l.ts }), 'restart')
                 : step8({ ...l, nextRes: l.stream.tryNext() })), 'wait for change');
         };
@@ -1860,6 +1991,8 @@ const executes$1 = (view, input, streamName, needs) => {
     else if (streamNames[streamName] != hash)
         throw new Error('streamName already used');
     const { collection, projection, hardMatch: pre, match } = view;
+    const client = prepare();
+    const pdb = client.then(cl => cl.db(collection.dbName));
     const removeNotYetSynchronizedFields = projection &&
         Object.values(mapExactToObject(projection, (_, k) => (needs[k] ?? k.startsWith('_')) ? root().of(k).has($exists(true)) : null));
     const hardMatch = removeNotYetSynchronizedFields
@@ -1941,14 +2074,14 @@ const executes$1 = (view, input, streamName, needs) => {
                 info: { debug: 'wait for clone into new collection', job: undefined },
             };
         };
-        const makeStream = (startAt) => makeWatchStream(db, view, startAt, streamName);
+        const makeStream = () => makeWatchStream(view, streamName);
         const step4 = (lastTS) => async () => {
             const raw = stages(lastTS).with(finalInput.raw(lastTS === null)).stages;
-            const aggResult = await aggregate(streamName, c => c({
+            const stream = makeStream();
+            const aggResult = await aggregate(pdb, streamName, c => c({
                 coll: collection,
                 input: raw,
             }));
-            const stream = makeStream(aggResult.cursor.atClusterTime);
             const nextRes = stream.tryNext();
             return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream, nextRes }), 'update __last', () => stream.close());
         };
@@ -1960,12 +2093,7 @@ const executes$1 = (view, input, streamName, needs) => {
             return {
                 data: l.aggResult.cursor.firstBatch,
                 info: { job: undefined, debug: 'wait for change' },
-                cont: withStop(() => l.nextRes
-                    .catch((err) => {
-                    log('restarting', err);
-                    return { ts: null };
-                })
-                    .then(doc => doc
+                cont: withStop(() => l.nextRes.then(doc => doc
                     ? next(step4({ _id: streamName, ts: l.ts }), 'restart')
                     : step8({ ...l, nextRes: l.stream.tryNext() }))),
             };
@@ -1994,49 +2122,6 @@ const executes = (view, input, needs) => {
 };
 const single = (view, needs = {}) => pipe(input => executes(view, input, needs), emptyDelta(), concatDelta, emptyDelta);
 
-require('dotenv').config();
-const uri = process.env['MONGO_URL'];
-
-const enablePreAndPostImages = (coll) => coll.s.db.command({
-    collMod: coll.collectionName,
-    changeStreamPreAndPostImages: { enabled: true },
-});
-const prepare = async (testName) => {
-    const client = new mongodb.MongoClient(uri, testName ? { monitorCommands: true } : {});
-    if (testName) {
-        const handler = (c) => {
-            promises.writeFile(`./out/${testName}.log`, JSON.stringify(c.command) + ',\n', { flag: 'w' });
-        };
-        client.on('commandStarted', handler);
-        client.on('commandSucceeded', handler);
-    }
-    await client.connect();
-    await client.db('admin').command({
-        setClusterParameter: {
-            changeStreamOptions: {
-                preAndPostImages: { expireAfterSeconds: 60 },
-            },
-        },
-    });
-    return client;
-};
-const makeCol = async (docs, database, name) => {
-    if (!name) {
-        (name = crypto.randomUUID());
-    }
-    try {
-        const col = await database.createCollection(name, {
-            changeStreamPreAndPostImages: { enabled: true },
-        });
-        if (docs.length)
-            await col.insertMany([...docs]);
-        return col;
-    }
-    catch {
-        return database.collection(name);
-    }
-};
-
 exports.$accumulator = $accumulator;
 exports.$and = $and;
 exports.$countDict = $countDict;
@@ -2161,6 +2246,7 @@ exports.range = range;
 exports.regex = regex;
 exports.root = root;
 exports.set = set;
+exports.setF = setF;
 exports.setField = setField;
 exports.single = single;
 exports.size = size;

package/package.json

@@ -3,7 +3,7 @@
     "module": "index.esm.js",
     "typings": "index.d.ts",
     "name": "@omegup/msync",
-    "version": "0.1.18",
+    "version": "0.1.20",
     "dependencies": {
         "dayjs": "^1.11.9",
         "dotenv": "^16.3.1",