@omegup/msync 0.1.21 → 0.1.23

package/index.d.ts

@@ -1,4 +1,4 @@
-import { Timestamp, Filter, UpdateFilter, BSON, Db, Collection, IndexSpecification, CreateIndexesOptions, MongoClient as MongoClient$1, OptionalUnlessRequiredId } from 'mongodb';
+import { Timestamp, Filter, UpdateFilter, BSON, MaxKey, Db, Collection, IndexSpecification, CreateIndexesOptions, MongoClient as MongoClient$1, OptionalUnlessRequiredId } from 'mongodb';
 export { Collection, Timestamp } from 'mongodb';
 
 type HasJob = {
@@ -94,7 +94,7 @@ type StreamRunnerParam<in V, out Result> = {
     raw: (first: boolean) => RawStages<unknown, V, Result>;
     teardown: <R>(consume: <W, M extends keyof Actions<W>>(x: TeardownRecord<W, M>) => R) => R;
 };
-type StreamRunner<out V> = <Result>(input: StreamRunnerParam<V, Result>) => Runner<readonly Result[], HasJob>;
+type StreamRunner<out V> = <Result>(input: StreamRunnerParam<V, Result>, setup?: () => Promise<void>) => Runner<readonly Result[], HasJob>;
 type SimpleStreamExecutionResult<out Q, out V extends Q> = {
     readonly out: StreamRunner<V>;
 };
@@ -220,7 +220,7 @@ declare const Type: unique symbol;
 
 type U = undefined;
 type N = null | U;
-type jsonPrim = number | null | string | boolean | Timestamp | Date;
+type jsonPrim = number | null | string | boolean | Timestamp | MaxKey | Date;
 type notObj = jsonPrim | U;
 type notArr = notObj | O;
 type jsonItem = unknown;
@@ -536,7 +536,8 @@ declare const log: (...args: unknown[]) => void;
 
 declare const createIndex: (collection: {
     readonly createIndex: Collection["createIndex"];
-}, indexSpec: IndexSpecification, options?: CreateIndexesOptions) => Promise<void>;
+    collectionName: string;
+}, indexSpec: IndexSpecification, op?: CreateIndexesOptions) => Promise<void>;
 
 declare const noop: () => void;
 declare const map1: <K extends string, Im>(k: AsLiteral<K>, to: Im) => { readonly [P in K]: [P, Im]; } & {
@@ -697,7 +698,7 @@ declare const wrap: <Result>(root: Machine<Result>) => Machine<Result>;
 declare const $eq: <T extends unknown>(operand: rawItem & T) => Predicate<T>;
 declare const $ne: <T extends unknown>(operand: rawItem & T) => Predicate<T>;
 type Numeric = number | Timestamp | Date;
-declare const comp: <D2 extends Numeric = Numeric>(op: "$lt" | "$gt" | "$lte" | "$gte") => <T extends Numeric>(operand: rawItem & D2) => Predicate<D2>;
+declare const comp: <D2 extends Numeric = Numeric>(op: "$lt" | "$lte" | "$gt" | "$gte") => <T extends Numeric>(operand: rawItem & D2) => Predicate<D2>;
 declare const $gt: <T extends Numeric>(operand: rawItem & Numeric) => Predicate<Numeric>;
 declare const $gtTs: <T extends Numeric>(operand: rawItem & Timestamp) => Predicate<Timestamp>;
 declare const $gteTs: <T extends Numeric>(operand: rawItem & Timestamp) => Predicate<Timestamp>;
@@ -730,12 +731,8 @@ declare const $and: Combiner;
 declare const $nor: Combiner;
 declare const $or: Combiner;
 
-declare const setF: (f: ({ input }: {
-    input: any;
-}) => Promise<void>) => void;
-
 declare const enablePreAndPostImages: <T extends doc>(coll: Collection<T>) => Promise<Document>;
 declare const prepare: (testName?: string) => Promise<MongoClient$1>;
 declare const makeCol: <T extends ID>(docs: readonly OptionalUnlessRequiredId<T>[], database: Db, name?: string) => Promise<Collection<T>>;
 
-export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, type Accumulators, type Arr, type AsLiteral, type Delta, type DeltaAccumulator, type DeltaAccumulators, type ExactKeys, Expr, type ExprHKT, type Exprs, type ExprsExact, type ExprsExactHKT, type ExprsPart, Field, type ID, type Loose, Machine, type Merge, type MergeArgs, type MergeInto, type MergeMapOArgs, type Model, type MongoTypeNames, type N, type NoRaw, type NullToOBJ, type O, type OPick, type OPickD, type Patch, type RONoRaw, type RORec, type RawStages, type Rec, type Replace, type SnapshotStreamExecutionResult, type StrKey, type Strict, type TS, Type, type WriteonlyCollection, add, and, anyElementTrue, array, ceil, comp, concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, type doc, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first, firstSure, floor, from, func, getWhenMatched, getWhenMatchedForMerge, gt, gte, inArray, isArray, ite, type jsonPrim, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, not, type notArr, notNull, now, or, pair, prepare, rand, range, regex, root, set, setF, setField, single, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };
+export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, type Accumulators, type Arr, type AsLiteral, type Delta, type DeltaAccumulator, type DeltaAccumulators, type ExactKeys, Expr, type ExprHKT, type Exprs, type ExprsExact, type ExprsExactHKT, type ExprsPart, Field, type ID, type Loose, Machine, type Merge, type MergeArgs, type MergeInto, type MergeMapOArgs, type Model, type MongoTypeNames, type N, type NoRaw, type NullToOBJ, type O, type OPick, type OPickD, type Patch, type RONoRaw, type RORec, type RawStages, type Rec, type Replace, type SnapshotStreamExecutionResult, type StrKey, type Strict, type TS, Type, type WriteonlyCollection, add, and, anyElementTrue, array, ceil, comp, concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, type doc, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first, firstSure, floor, from, func, getWhenMatched, getWhenMatchedForMerge, gt, gte, inArray, isArray, ite, type jsonPrim, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, not, type notArr, notNull, now, or, pair, prepare, rand, range, regex, root, set, setField, single, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };

package/index.esm.js

@@ -1,7 +1,7 @@
+import { MaxKey, UUID, MongoClient } from 'mongodb';
 import crypto$1 from 'crypto';
 import { canonicalize } from 'json-canonicalize';
 import { SynchronousPromise } from 'synchronous-promise';
-import { MongoClient, UUID } from 'mongodb';
 import { writeFile } from 'fs/promises';
 
 const asExprRaw = (raw) => ({ get: () => raw });
@@ -82,7 +82,7 @@ const val = (val) => asExpr({
         : val),
 });
 const current = asExpr({
-    raw: () => asExprRaw('$$CLUSTER_TIME'),
+    raw: () => asExprRaw(new MaxKey()),
 });
 const $let = (vars, inExpr) => asExpr({
     raw: f => asExprRaw({
@@ -1251,19 +1251,48 @@ const $lookupRaw = ({ field1, field2 }, { coll, exec, input }, k2, k, includeNul
 
 const asBefore = (f) => f(() => root().of('before'));
 
-const createIndex = async (collection, indexSpec, options) => {
+const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
+const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
+const json = (a) => replace(JSON.stringify(a));
+const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
+
+const indexMap = new Map();
+const createIndex = async (collection, indexSpec, op) => {
+    const { name, ...options } = op ?? {};
+    const map = indexMap.get(collection.collectionName) ?? new Map();
+    indexMap.set(collection.collectionName, map);
+    const indexKey = `${JSON.stringify(indexSpec)}-${JSON.stringify(options)}`;
+    if (map.has(indexKey)) {
+        await map.get(indexKey);
+        return;
+    }
+    const promise = createIndexWithRetry(collection, indexSpec, op);
+    map.set(indexKey, promise);
+    await promise;
+};
+const createIndexWithRetry = async (collection, indexSpec, options) => {
+    const log = () => { };
+    log('Creating index', { collection: collection.collectionName, indexSpec, options });
     while (true) {
         try {
             await collection.createIndex(indexSpec, options);
+            log('Index created', { collection: collection.collectionName, indexSpec, options });
         }
         catch (e) {
             if ([85, 276].includes(e.code)) {
+                log('Index created with different name', e.code, { collection: collection.collectionName, indexSpec, options });
                 break;
             }
             if (e.code == 12587) {
                 await new Promise(res => setTimeout(res, 300));
                 continue;
             }
+            log('Error creating index', {
+                collection: collection.collectionName,
+                indexSpec,
+                options,
+                error: e,
+            });
             console.error('Error creating index', e);
             throw e;
         }
@@ -1304,16 +1333,18 @@ const nextWinner = (previousWinner, previousWinnerNextFrame, sources, interrupt)
 };
 
 const mergeIterators = (params) => {
-    const { sources, interrupt, select = race } = params;
+    const { sources, interrupt, select = race, hooks } = params;
     const reiterate = (winner) => {
         const { frame, key } = winner;
         return {
             cont: () => {
                 const result = frame.cont();
+                hooks?.start?.(frame, result);
                 return mergeIterators({
                     sources: patch(sources, key, result),
                     interrupt,
                     select: sources => nextWinner(winner, result.next, sources, interrupt),
+                    hooks,
                 });
             },
             data: frame.data,
@@ -1321,7 +1352,7 @@ const mergeIterators = (params) => {
         };
     };
     return {
-        stop: () => mergeIterators({ sources: restart(sources), interrupt }),
+        stop: () => mergeIterators({ sources: restart(sources), interrupt, select, hooks }),
         next: select(sources).then(reiterate),
         clear: async () => {
             for (const key in sources) {
@@ -1331,56 +1362,37 @@ const mergeIterators = (params) => {
     };
 };
 
-const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
-const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
-const json = (a) => replace(JSON.stringify(a));
-const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
-
-const state = { steady: false, f: (_) => Promise.resolve() };
+const state = { steady: false };
 let timeout = null;
-const setF = (f) => {
-    state.f = f;
-};
-const aggregate = (db, streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
-    const req = {
-        aggregate: coll.collectionName,
-        pipeline: input,
-        cursor: {},
-        ...(snapshot && { readConcern: { level: 'snapshot' } }),
-    };
-    if (timeout !== null) {
-        clearTimeout(timeout);
-        timeout = null;
-    }
-    log('exec', streamName, req);
-    const start2 = Date.now();
-    return db.then(d => d.command(req)).then(result => {
-        log('prepare', streamName, Date.now() - start);
-        log('prepare2', streamName, start2 - start);
-        const r = result;
-        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('"$$CLUSTER_TIME"', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
-        if (!state.steady) {
-            if (timeout !== null)
-                throw new Error('timeout should be null');
-            timeout = setTimeout(() => {
-                state.steady = true;
-                console.log('steady');
-            }, 10000);
-        }
-        return r;
-    }, err => {
-        log('err', req, err);
-        throw new Error(err);
-    });
-});
-
 const firstWorksMerge = (iters) => {
     const iterator = () => {
         const results = iters.map(iter => iter());
         const sources = { ...results };
         return mergeIterators({
             sources,
-            interrupt: key => state.steady
+            interrupt: key => state.steady,
+            hooks: {
+                start: (frame, result) => {
+                    if (!frame.info.job)
+                        return;
+                    if (timeout !== null) {
+                        clearTimeout(timeout);
+                        timeout = null;
+                    }
+                    result.next.then(() => {
+                        if (!frame.info.job)
+                            return;
+                        if (!state.steady) {
+                            if (timeout !== null)
+                                clearTimeout(timeout);
+                            timeout = setTimeout(() => {
+                                state.steady = true;
+                                console.log('steady');
+                            }, 2000);
+                        }
+                    });
+                },
+            },
         });
     };
     return iterator;
@@ -1426,8 +1438,6 @@ const runCont = async (it, cb) => {
 
 const merge = ({ lsource: L, rsource: R, }) => mergeIterators({ sources: { L, R } });
 const join = ({ lField, rField, left, right, as }, leftSnapshot, rightSnapshot, stagesUntilNextLookup, outerLeft) => {
-    createIndex(leftSnapshot.coll, { [`before.${lField.str()}`]: 1 }).catch(e => e.code == 86 || Promise.reject(e));
-    createIndex(rightSnapshot.coll, { [`before.${rField.str()}`]: 1 }).catch(e => e.code == 86 || Promise.reject(e));
     const rightJoinField = { field1: lField, field2: rField };
     const joinId = 'left';
     const joinR_Snapshot = asBefore($lookupRaw(rightJoinField, rightSnapshot, as, joinId, outerLeft));
@@ -1447,6 +1457,15 @@ const join = ({ lField, rField, left, right, as }, leftSnapshot, rightSnapshot,
             const getRunner = (f, stages, final) => f.out({
                 raw: first => concatStages(stages, final.raw(first)),
                 teardown: final.teardown,
+            }, async () => {
+                log('Creating indexes for lookup left', leftSnapshot.coll.collectionName, {
+                    [`before.${lField.str()}`]: 1,
+                });
+                await createIndex(leftSnapshot.coll, { [`before.${lField.str()}`]: 1 }, { name: 'left_' + new UUID().toString('base64') });
+                log('Creating indexes for lookup right', rightSnapshot.coll.collectionName, {
+                    [`before.${rField.str()}`]: 1,
+                });
+                await createIndex(rightSnapshot.coll, { [`before.${rField.str()}`]: 1 }, { name: 'right_' + new UUID().toString('base64') });
             });
             const lRunner = getRunner(left, lRunnerInput, finalInput);
             const rRunner = getRunner(right, rRunnerInput, finalInput);
@@ -1510,10 +1529,7 @@ const $insertX = (out, expr, map, ext, extExpr) => {
         teardown: c => c({
             collection: out,
             method: 'updateMany',
-            params: [
-                filter,
-                [{ $set: { deletedAt: '$$NOW', touchedAt: '$$CLUSTER_TIME' } }],
-            ],
+            params: [filter, [{ $set: { deletedAt: '$$NOW', touchedAt: '$$CLUSTER_TIME' } }]],
         }),
         raw: () => {
             const replacer = map(mergeObjects(expr, field(mergeExpr(extExpr, {
@@ -1544,6 +1560,27 @@ const $insertPart = (out, ext) => {
 const $insert = (out) => $insertPart(out, {});
 const assertNotNull = (expr) => expr;
 
+const aggregate = (db, streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
+    const req = {
+        aggregate: coll.collectionName,
+        pipeline: input,
+        cursor: {},
+        ...(snapshot && { readConcern: { level: 'snapshot' } }),
+    };
+    log('exec', streamName, req);
+    const start2 = Date.now();
+    return db.then(d => d.command(req)).then(result => {
+        log('prepare', streamName, Date.now() - start);
+        log('prepare2', streamName, start2 - start);
+        const r = result;
+        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('"$$CLUSTER_TIME"', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
+        return r;
+    }, err => {
+        log('err', req, err);
+        throw new Error(err);
+    });
+});
+
 const addTeardown = (it, tr) => {
     if (!tr)
         return it;
@@ -1571,7 +1608,7 @@ async function getLastCommittedTs(adminDb) {
     const st = await adminDb.command({ replSetGetStatus: 1 });
     return st?.optimes?.lastCommittedOpTime?.ts ?? null;
 }
-async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000, } = {}) {
+async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000 } = {}) {
     const adminDb = db.client.db('admin');
     const deadline = Date.now() + timeoutMs;
     while (true) {
@@ -1579,7 +1616,7 @@ async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000
         if (stable && stable.comp(oplogTs) >= 0)
             return;
         if (Date.now() > deadline) {
-            throw new Error("Timed out waiting for stable timestamp to reach oplog event time");
+            throw new Error('Timed out waiting for stable timestamp to reach oplog event time');
         }
         await sleep(pollMs);
     }
@@ -1601,29 +1638,37 @@ async function* tailOplog(db, opts) {
         try {
             for await (const doc of cursor) {
                 lastTs = doc.ts;
-                if (doc.op === 'i') {
-                    yield { ns: doc.ns, fields: new Set(Object.keys(doc.o)), doc };
+                if (doc.op === 'i' || '_id' in doc.o) {
+                    const fields = new Set(Object.keys(doc.o));
+                    fields.delete('_id');
+                    yield { fields, doc, changeTouched: doc.o['touchedAt'] instanceof MaxKey };
                 }
                 else {
+                    let changeTouched = false;
                     if (doc.o['$v'] !== 2) {
-                        throw new Error(`Expected update with $v: 2, got ${JSON.stringify(doc.o)}`);
+                        throw new Error(`Expected update with $v: 2, got ${JSON.stringify(doc)}`);
                     }
                     const updatedFields = [];
                     const diff = doc.o['diff'];
                     for (const updateOp in diff) {
                         if (['u', 'i', 'd'].includes(updateOp)) {
                             updatedFields.push(...Object.keys(diff[updateOp]));
+                            if (diff[updateOp]['touchedAt'] instanceof MaxKey) {
+                                changeTouched = true;
+                            }
                         }
                         else if (updateOp.startsWith('s')) {
                             updatedFields.push(updateOp.slice(1));
                         }
                     }
-                    yield { ns: doc.ns, fields: new Set(updatedFields), doc };
+                    yield { fields: new Set(updatedFields), doc, changeTouched };
                 }
             }
         }
         catch (e) {
-            log('oplog loop error', e);
+            log('oplog loop error, notifying watchers and reopening');
+            console.error(e);
+            yield null;
         }
         finally {
             log('oplog loop ended');
@@ -1634,15 +1679,68 @@ async function* tailOplog(db, opts) {
 }
 const watchers = new Map();
 let running = false;
+const makePromise = () => {
+    let resolve = () => { };
+    let promise = new Promise(r => (resolve = r));
+    return { promise, resolve };
+};
 const loop = async (db) => {
     log('starting oplog loop');
-    for await (const { ns, fields, doc } of tailOplog(db, {})) {
-        const m = watchers.get(ns);
-        if (!m)
+    let notify = makePromise();
+    let batch = [];
+    const run = async () => {
+        for await (const event of tailOplog(db, {})) {
+            if (event?.fields.size === 0)
+                continue;
+            batch = event && batch ? [...batch, event] : null;
+            notify.resolve();
+        }
+    };
+    run();
+    const iter = async function* () {
+        while (true) {
+            await notify.promise;
+            const b = batch;
+            batch = [];
+            notify = makePromise();
+            yield b;
+        }
+    };
+    for await (const events of iter()) {
+        if (!events) {
+            log('notifying watchers of oplog loop restart');
+            for (const m of watchers.values()) {
+                for (const { cb } of m.values()) {
+                    cb(null);
+                }
+            }
             continue;
-        for (const { cb, keys } of m.values()) {
-            if (!keys || keys.some(k => fields.has(k))) {
-                cb(doc);
+        }
+        const groups = Object.groupBy(events.filter(e => e.changeTouched), ev => ev.doc.ns);
+        for (const [ns, evs] of Object.entries(groups)) {
+            if (!evs)
+                continue;
+            const [dbName, collName] = ns.split('.');
+            if (dbName !== db.databaseName)
+                continue;
+            const coll = db.collection(collName);
+            coll
+                .bulkWrite(evs.map((e) => ({
+                updateOne: {
+                    filter: { _id: e.doc.o['_id'] ?? e.doc.o2?._id },
+                    update: { $set: { touchedAt: e.doc.ts } },
+                },
+            })))
+                .catch(() => { });
+        }
+        for (const { fields, doc } of events) {
+            const m = watchers.get(doc.ns);
+            if (!m)
+                continue;
+            for (const { cb, keys } of m.values()) {
+                if (!keys || keys.some(k => fields.has(k))) {
+                    cb(doc);
+                }
             }
         }
     }
@@ -1664,11 +1762,16 @@ const register = (coll, keys, cb) => {
             watchers.delete(ns);
     };
 };
-const makeWatchStream = ({ collection, projection: p, hardMatch: m }, streamName) => {
+let maxKeysRemoved = null;
+const makeWatchStream = async ({ collection, projection: p, hardMatch: m }, streamName) => {
+    const { db } = collection.s;
+    await (maxKeysRemoved ??= Promise.all((await db.listCollections({}, { nameOnly: true }).toArray()).map(x => void db
+        .collection(x.name)
+        .updateMany({ touchedAt: new MaxKey() }, [{ $set: { touchedAt: '$$CLUSTER_TIME' } }]))).then(() => { }));
     const projection = { ...(p ? mapExactToObject(p, v => v) : {}), deletedAt: 1 };
     let resolve = (_) => { };
     const promise = new Promise(r => (resolve = r));
-    const close = register(collection, p ? Object.keys(projection) : null, (doc) => {
+    const close = register(collection, p ? Object.keys(projection) : null, doc => {
         log(streamName, 'change detected', doc);
         resolve(doc);
         close();
@@ -1677,9 +1780,10 @@ const makeWatchStream = ({ collection, projection: p, hardMatch: m }, streamName
         tryNext: async () => {
             const doc = await promise;
             const start = Date.now();
-            await waitUntilStablePast(collection.s.db, doc.ts);
+            if (doc)
+                await waitUntilStablePast(collection.s.db, doc.ts);
             log(streamName, 'stable past took', Date.now() - start);
-            return doc;
+            return doc ?? {};
         },
         close: async () => close(),
     };
@@ -1773,44 +1877,17 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
         streamNames[streamName] = hash;
     else if (streamNames[streamName] != hash)
         throw new Error(`streamName ${streamName} already used`);
-    db.command({
-        collMod: coll,
-        changeStreamPreAndPostImages: { enabled: true },
-    });
-    createIndex(collection, { touchedAt: 1 }, hardMatch
-        ? {
-            partialFilterExpression: hardMatch.raw(root()),
-            name: 'touchedAt_hard_' + new UUID().toString('base64'),
-        }
-        : {}).catch(e => e.code == 86 || Promise.reject(e));
     const last = db.collection('__last');
     const snapshotCollection = db.collection(coll + '_' + streamName + '_snapshot');
-    createIndex(snapshotCollection, { before: 1 }, {
-        partialFilterExpression: { before: null },
-        name: 'before_' + new UUID().toString('base64'),
-    });
-    createIndex(snapshotCollection, { updated: 1 }, {
-        partialFilterExpression: { updated: true },
-        name: 'updated_' + new UUID().toString('base64'),
-    });
-    createIndex(snapshotCollection, { updated: 1, after: 1, before: 1 }, {
-        partialFilterExpression: { updated: true, after: null, before: null },
-        name: 'updated_nulls_' + new UUID().toString('base64'),
-    });
-    createIndex(snapshotCollection, { updated: 1, after: 1 }, {
-        partialFilterExpression: { updated: true, after: null },
-        name: 'updated_no_after_' + new UUID().toString('base64'),
-    });
-    createIndex(snapshotCollection, { updated: 1 }, {
-        partialFilterExpression: { updated: true, after: null, before: null },
-        name: 'updated_nulls_' + new UUID().toString('base64'),
-    });
     const job = {};
-    const run = (finalInput) => {
-        const clear = async () => Promise.all([
-            snapshotCollection.drop().catch(noop).catch(noop),
-            last.deleteOne({ _id: streamName }),
-        ]);
+    const run = (finalInput, setup) => {
+        const dropSnapshot = async () => {
+            await snapshotCollection.drop().catch(noop);
+            log('snapshot collection dropped', streamName, `db['${snapshotCollection.collectionName}'].drop()`);
+            log('with', [...(indexMap.get(snapshotCollection.collectionName)?.keys() ?? [])], 'indexes in map before deletion');
+            indexMap.delete(snapshotCollection.collectionName);
+        };
+        const clear = async () => Promise.all([dropSnapshot(), last.deleteOne({ _id: streamName })]);
         const withStop = (next, tr) => {
             return addTeardown(() => ({ stop, next: next(), clear }), tr);
         };
@@ -1835,7 +1912,9 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
         const step0 = () => SynchronousPromise.resolve(next(step1, 'empty new collection'));
         const stop = withStop(step0);
         const step1 = async () => {
+            log('reset collection', streamName, `db['${snapshotCollection.collectionName}'].updateMany( updated: true }, { $set: { updated: false, after: null } })`);
             await snapshotCollection.updateMany({ updated: true }, { $set: { updated: false, after: null } });
+            log('reset collection done', streamName);
             return next(step2, 'get last update');
         };
         const step2 = () => Promise.all([
@@ -1858,13 +1937,45 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
                 };
                 const [action, out] = actions[method](collection, params);
                 log('teardown', `db['${snapshotCollection.collectionName}'].drop()`, ...out);
-                await Promise.all([snapshotCollection.drop().catch(noop), action]);
+                await Promise.all([dropSnapshot(), action]);
                 log('teardown done', `db['${snapshotCollection.collectionName}'].drop()`, ...out);
             };
             if (!same) {
                 log('not same, new data', streamName, data);
                 await handleTeardown(exists ?? { data });
             }
+            log('creating indexes');
+            await createIndex(snapshotCollection, { before: 1 }, {
+                partialFilterExpression: { before: null },
+                name: 'before_' + new UUID().toString('base64'),
+            });
+            await createIndex(snapshotCollection, { updated: 1 }, {
+                partialFilterExpression: { updated: true },
+                name: 'updated_' + new UUID().toString('base64'),
+            });
+            await createIndex(snapshotCollection, { updated: 1, after: 1, before: 1 }, {
+                partialFilterExpression: { updated: true, after: null, before: null },
+                name: 'updated_nulls_' + new UUID().toString('base64'),
+            });
+            await createIndex(snapshotCollection, { updated: 1, after: 1 }, {
+                partialFilterExpression: { updated: true, after: null },
+                name: 'updated_no_after_' + new UUID().toString('base64'),
+            });
+            await createIndex(snapshotCollection, { updated: 1 }, {
+                partialFilterExpression: { updated: true, after: null, before: null },
+                name: 'updated_nulls_' + new UUID().toString('base64'),
+            });
+            await db.command({
+                collMod: coll,
+                changeStreamPreAndPostImages: { enabled: true },
+            });
+            await createIndex(collection, { touchedAt: 1 }, hardMatch
+                ? {
+                    partialFilterExpression: hardMatch.raw(root()),
+                    name: 'touchedAt_hard_' + new UUID().toString('base64'),
+                }
+                : {});
+            await setup?.();
             await after?.();
             return nextData([])(async () => {
                 await new Promise(resolve => setTimeout(resolve, 1000));
@@ -1904,7 +2015,8 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
             const first = ts === undefined;
             const stages = finalInput.raw(first);
             await last.updateOne({ _id: streamName }, { $set: { job: 1 } }, { upsert: true });
-            const stream = makeStream();
+            const stream = await makeStream();
+            const nextRes = stream.tryNext();
             const aggResult = await aggregate(pdb, streamName, c => c({
                 coll: snapshotCollection,
                 input: link()
@@ -1918,7 +2030,6 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
                     .with(input.delta)
                     .with(stages).stages,
             }), false, start);
-            const nextRes = stream.tryNext();
             stages.at(-1).$merge.into.coll;
             return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream, nextRes, first }), 'remove handled deleted updated', () => stream.close());
         };
@@ -1997,14 +2108,6 @@ const executes$1 = (view, input, streamName, needs) => {
         : pre;
     const job = {};
     const db = collection.s.db, coll = collection.collectionName;
-    db.command({
-        collMod: coll,
-        changeStreamPreAndPostImages: { enabled: true },
-    });
-    createIndex(collection, { touchedAt: 1 }, {
-        partialFilterExpression: { deletedAt: { $eq: null } },
-        name: 'touchedAt_' + new UUID().toString('base64'),
-    });
     const last = db.collection('__last');
     const projectInput = projection &&
         $project_(spread(projection, {
@@ -2041,10 +2144,26 @@ const executes$1 = (view, input, streamName, needs) => {
         };
         const step0 = () => SynchronousPromise.resolve(next(step1, 'get last update'));
         const stop = withStop(step0);
-        const step1 = () => Promise.all([
-            last.findOne({ _id: streamName, data }),
-            last.findOne({ _id: streamName }),
-        ]).then(ts => next(step2_5(ts), 'handle teardown'));
+        const step1 = async () => {
+            log('creating indexes');
+            await db.command({
+                collMod: coll,
+                changeStreamPreAndPostImages: { enabled: true },
+            });
+            await createIndex(collection, { touchedAt: 1 }, {
+                partialFilterExpression: { deletedAt: { $eq: null } },
+                name: 'touchedAt_' + new UUID().toString('base64'),
+            });
+            log('start stream', { streamName, data });
+            await last.findOne();
+            console.log('got last update');
+            const p = last.findOne({ _id: streamName, data });
+            await p;
+            log('stream started', { streamName, data });
+            const ts = await Promise.all([p, last.findOne({ _id: streamName })]);
+            log('got last update', { streamName, ts });
+            return next(step2_5(ts), 'handle teardown');
+        };
         const step2_5 = ([same, exists]) => async () => {
             const handleTeardown = async (last) => {
                 if (!last.data)
@@ -2074,7 +2193,7 @@ const executes$1 = (view, input, streamName, needs) => {
         const makeStream = () => makeWatchStream(view, streamName);
         const step4 = (lastTS) => async () => {
             const raw = stages(lastTS).with(finalInput.raw(lastTS === null)).stages;
-            const stream = makeStream();
+            const stream = await makeStream();
             const aggResult = await aggregate(pdb, streamName, c => c({
                 coll: collection,
                 input: raw,
@@ -2119,4 +2238,4 @@ const executes = (view, input, needs) => {
 };
 const single = (view, needs = {}) => pipe(input => executes(view, input, needs), emptyDelta(), concatDelta, emptyDelta);
 
-export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, Field, Machine, add, and, anyElementTrue, array, ceil, comp, concat$1 as concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first$1 as first, firstSure, floor, from, func, getWhenMatched, getWhenMatchedForMerge, gt, gte, inArray, isArray, ite, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, not, notNull, now, or, pair, prepare, rand, range, regex, root, set, setF, setField, single, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };
+export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, Field, Machine, add, and, anyElementTrue, array, ceil, comp, concat$1 as concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first$1 as first, firstSure, floor, from, func, getWhenMatched, getWhenMatchedForMerge, gt, gte, inArray, isArray, ite, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, not, notNull, now, or, pair, prepare, rand, range, regex, root, set, setField, single, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };

package/index.js

@@ -1,9 +1,9 @@
 'use strict';
 
+var mongodb = require('mongodb');
 var crypto$1 = require('crypto');
 var jsonCanonicalize = require('json-canonicalize');
 var synchronousPromise = require('synchronous-promise');
-var mongodb = require('mongodb');
 var promises = require('fs/promises');
 
 const asExprRaw = (raw) => ({ get: () => raw });
@@ -84,7 +84,7 @@ const val = (val) => asExpr({
         : val),
 });
 const current = asExpr({
-    raw: () => asExprRaw('$$CLUSTER_TIME'),
+    raw: () => asExprRaw(new mongodb.MaxKey()),
 });
 const $let = (vars, inExpr) => asExpr({
     raw: f => asExprRaw({
@@ -1253,19 +1253,48 @@ const $lookupRaw = ({ field1, field2 }, { coll, exec, input }, k2, k, includeNul
 
 const asBefore = (f) => f(() => root().of('before'));
 
-const createIndex = async (collection, indexSpec, options) => {
+const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
+const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
+const json = (a) => replace(JSON.stringify(a));
+const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
+
+const indexMap = new Map();
+const createIndex = async (collection, indexSpec, op) => {
+    const { name, ...options } = op ?? {};
+    const map = indexMap.get(collection.collectionName) ?? new Map();
+    indexMap.set(collection.collectionName, map);
+    const indexKey = `${JSON.stringify(indexSpec)}-${JSON.stringify(options)}`;
+    if (map.has(indexKey)) {
+        await map.get(indexKey);
+        return;
+    }
+    const promise = createIndexWithRetry(collection, indexSpec, op);
+    map.set(indexKey, promise);
+    await promise;
+};
+const createIndexWithRetry = async (collection, indexSpec, options) => {
+    const log = () => { };
+    log('Creating index', { collection: collection.collectionName, indexSpec, options });
     while (true) {
         try {
             await collection.createIndex(indexSpec, options);
+            log('Index created', { collection: collection.collectionName, indexSpec, options });
         }
         catch (e) {
             if ([85, 276].includes(e.code)) {
+                log('Index created with different name', e.code, { collection: collection.collectionName, indexSpec, options });
                 break;
             }
             if (e.code == 12587) {
                 await new Promise(res => setTimeout(res, 300));
                 continue;
             }
+            log('Error creating index', {
+                collection: collection.collectionName,
+                indexSpec,
+                options,
+                error: e,
+            });
             console.error('Error creating index', e);
             throw e;
         }
@@ -1306,16 +1335,18 @@ const nextWinner = (previousWinner, previousWinnerNextFrame, sources, interrupt)
 };
 
 const mergeIterators = (params) => {
-    const { sources, interrupt, select = race } = params;
+    const { sources, interrupt, select = race, hooks } = params;
     const reiterate = (winner) => {
         const { frame, key } = winner;
         return {
             cont: () => {
                 const result = frame.cont();
+                hooks?.start?.(frame, result);
                 return mergeIterators({
                     sources: patch(sources, key, result),
                     interrupt,
                     select: sources => nextWinner(winner, result.next, sources, interrupt),
+                    hooks,
                 });
             },
             data: frame.data,
@@ -1323,7 +1354,7 @@ const mergeIterators = (params) => {
         };
     };
     return {
-        stop: () => mergeIterators({ sources: restart(sources), interrupt }),
+        stop: () => mergeIterators({ sources: restart(sources), interrupt, select, hooks }),
         next: select(sources).then(reiterate),
         clear: async () => {
             for (const key in sources) {
@@ -1333,56 +1364,37 @@ const mergeIterators = (params) => {
     };
 };
 
-const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
-const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
-const json = (a) => replace(JSON.stringify(a));
-const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
-
-const state = { steady: false, f: (_) => Promise.resolve() };
+const state = { steady: false };
 let timeout = null;
-const setF = (f) => {
-    state.f = f;
-};
-const aggregate = (db, streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
-    const req = {
-        aggregate: coll.collectionName,
-        pipeline: input,
-        cursor: {},
-        ...(snapshot && { readConcern: { level: 'snapshot' } }),
-    };
-    if (timeout !== null) {
-        clearTimeout(timeout);
-        timeout = null;
-    }
-    log('exec', streamName, req);
-    const start2 = Date.now();
-    return db.then(d => d.command(req)).then(result => {
-        log('prepare', streamName, Date.now() - start);
-        log('prepare2', streamName, start2 - start);
-        const r = result;
-        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('"$$CLUSTER_TIME"', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
-        if (!state.steady) {
-            if (timeout !== null)
-                throw new Error('timeout should be null');
-            timeout = setTimeout(() => {
-                state.steady = true;
-                console.log('steady');
-            }, 10000);
-        }
-        return r;
-    }, err => {
-        log('err', req, err);
-        throw new Error(err);
-    });
-});
-
 const firstWorksMerge = (iters) => {
     const iterator = () => {
         const results = iters.map(iter => iter());
         const sources = { ...results };
         return mergeIterators({
             sources,
-            interrupt: key => state.steady
+            interrupt: key => state.steady,
+            hooks: {
+                start: (frame, result) => {
+                    if (!frame.info.job)
+                        return;
+                    if (timeout !== null) {
+                        clearTimeout(timeout);
+                        timeout = null;
+                    }
+                    result.next.then(() => {
+                        if (!frame.info.job)
+                            return;
+                        if (!state.steady) {
+                            if (timeout !== null)
+                                clearTimeout(timeout);
+                            timeout = setTimeout(() => {
+                                state.steady = true;
+                                console.log('steady');
+                            }, 2000);
+                        }
+                    });
+                },
+            },
         });
     };
     return iterator;
@@ -1428,8 +1440,6 @@ const runCont = async (it, cb) => {
 
 const merge = ({ lsource: L, rsource: R, }) => mergeIterators({ sources: { L, R } });
 const join = ({ lField, rField, left, right, as }, leftSnapshot, rightSnapshot, stagesUntilNextLookup, outerLeft) => {
-    createIndex(leftSnapshot.coll, { [`before.${lField.str()}`]: 1 }).catch(e => e.code == 86 || Promise.reject(e));
-    createIndex(rightSnapshot.coll, { [`before.${rField.str()}`]: 1 }).catch(e => e.code == 86 || Promise.reject(e));
     const rightJoinField = { field1: lField, field2: rField };
     const joinId = 'left';
     const joinR_Snapshot = asBefore($lookupRaw(rightJoinField, rightSnapshot, as, joinId, outerLeft));
@@ -1449,6 +1459,15 @@ const join = ({ lField, rField, left, right, as }, leftSnapshot, rightSnapshot,
             const getRunner = (f, stages, final) => f.out({
                 raw: first => concatStages(stages, final.raw(first)),
                 teardown: final.teardown,
+            }, async () => {
+                log('Creating indexes for lookup left', leftSnapshot.coll.collectionName, {
+                    [`before.${lField.str()}`]: 1,
+                });
+                await createIndex(leftSnapshot.coll, { [`before.${lField.str()}`]: 1 }, { name: 'left_' + new mongodb.UUID().toString('base64') });
+                log('Creating indexes for lookup right', rightSnapshot.coll.collectionName, {
+                    [`before.${rField.str()}`]: 1,
+                });
+                await createIndex(rightSnapshot.coll, { [`before.${rField.str()}`]: 1 }, { name: 'right_' + new mongodb.UUID().toString('base64') });
             });
             const lRunner = getRunner(left, lRunnerInput, finalInput);
             const rRunner = getRunner(right, rRunnerInput, finalInput);
@@ -1512,10 +1531,7 @@ const $insertX = (out, expr, map, ext, extExpr) => {
         teardown: c => c({
             collection: out,
             method: 'updateMany',
-            params: [
-                filter,
-                [{ $set: { deletedAt: '$$NOW', touchedAt: '$$CLUSTER_TIME' } }],
-            ],
+            params: [filter, [{ $set: { deletedAt: '$$NOW', touchedAt: '$$CLUSTER_TIME' } }]],
         }),
         raw: () => {
             const replacer = map(mergeObjects(expr, field(mergeExpr(extExpr, {
@@ -1546,6 +1562,27 @@ const $insertPart = (out, ext) => {
 const $insert = (out) => $insertPart(out, {});
 const assertNotNull = (expr) => expr;
 
+const aggregate = (db, streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
+    const req = {
+        aggregate: coll.collectionName,
+        pipeline: input,
+        cursor: {},
+        ...(snapshot && { readConcern: { level: 'snapshot' } }),
+    };
+    log('exec', streamName, req);
+    const start2 = Date.now();
+    return db.then(d => d.command(req)).then(result => {
+        log('prepare', streamName, Date.now() - start);
+        log('prepare2', streamName, start2 - start);
+        const r = result;
+        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('"$$CLUSTER_TIME"', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
+        return r;
+    }, err => {
+        log('err', req, err);
+        throw new Error(err);
+    });
+});
+
 const addTeardown = (it, tr) => {
     if (!tr)
         return it;
@@ -1573,7 +1610,7 @@ async function getLastCommittedTs(adminDb) {
     const st = await adminDb.command({ replSetGetStatus: 1 });
     return st?.optimes?.lastCommittedOpTime?.ts ?? null;
 }
-async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000, } = {}) {
+async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000 } = {}) {
     const adminDb = db.client.db('admin');
     const deadline = Date.now() + timeoutMs;
     while (true) {
@@ -1581,7 +1618,7 @@ async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000
         if (stable && stable.comp(oplogTs) >= 0)
             return;
         if (Date.now() > deadline) {
-            throw new Error("Timed out waiting for stable timestamp to reach oplog event time");
+            throw new Error('Timed out waiting for stable timestamp to reach oplog event time');
         }
         await sleep(pollMs);
     }
@@ -1603,29 +1640,37 @@ async function* tailOplog(db, opts) {
         try {
             for await (const doc of cursor) {
                 lastTs = doc.ts;
-                if (doc.op === 'i') {
-                    yield { ns: doc.ns, fields: new Set(Object.keys(doc.o)), doc };
+                if (doc.op === 'i' || '_id' in doc.o) {
+                    const fields = new Set(Object.keys(doc.o));
+                    fields.delete('_id');
+                    yield { fields, doc, changeTouched: doc.o['touchedAt'] instanceof mongodb.MaxKey };
                 }
                 else {
+                    let changeTouched = false;
                     if (doc.o['$v'] !== 2) {
-                        throw new Error(`Expected update with $v: 2, got ${JSON.stringify(doc.o)}`);
+                        throw new Error(`Expected update with $v: 2, got ${JSON.stringify(doc)}`);
                     }
                     const updatedFields = [];
                     const diff = doc.o['diff'];
                     for (const updateOp in diff) {
                         if (['u', 'i', 'd'].includes(updateOp)) {
                             updatedFields.push(...Object.keys(diff[updateOp]));
+                            if (diff[updateOp]['touchedAt'] instanceof mongodb.MaxKey) {
+                                changeTouched = true;
+                            }
                         }
                         else if (updateOp.startsWith('s')) {
                             updatedFields.push(updateOp.slice(1));
                         }
                     }
-                    yield { ns: doc.ns, fields: new Set(updatedFields), doc };
+                    yield { fields: new Set(updatedFields), doc, changeTouched };
                 }
             }
         }
         catch (e) {
-            log('oplog loop error', e);
+            log('oplog loop error, notifying watchers and reopening');
+            console.error(e);
+            yield null;
         }
         finally {
             log('oplog loop ended');
@@ -1636,15 +1681,68 @@ async function* tailOplog(db, opts) {
 }
 const watchers = new Map();
 let running = false;
+const makePromise = () => {
+    let resolve = () => { };
+    let promise = new Promise(r => (resolve = r));
+    return { promise, resolve };
+};
 const loop = async (db) => {
     log('starting oplog loop');
-    for await (const { ns, fields, doc } of tailOplog(db, {})) {
-        const m = watchers.get(ns);
-        if (!m)
+    let notify = makePromise();
+    let batch = [];
+    const run = async () => {
+        for await (const event of tailOplog(db, {})) {
+            if (event?.fields.size === 0)
+                continue;
+            batch = event && batch ? [...batch, event] : null;
+            notify.resolve();
+        }
+    };
+    run();
+    const iter = async function* () {
+        while (true) {
+            await notify.promise;
+            const b = batch;
+            batch = [];
+            notify = makePromise();
+            yield b;
+        }
+    };
+    for await (const events of iter()) {
+        if (!events) {
+            log('notifying watchers of oplog loop restart');
+            for (const m of watchers.values()) {
+                for (const { cb } of m.values()) {
+                    cb(null);
+                }
+            }
             continue;
-        for (const { cb, keys } of m.values()) {
-            if (!keys || keys.some(k => fields.has(k))) {
-                cb(doc);
+        }
+        const groups = Object.groupBy(events.filter(e => e.changeTouched), ev => ev.doc.ns);
+        for (const [ns, evs] of Object.entries(groups)) {
+            if (!evs)
+                continue;
+            const [dbName, collName] = ns.split('.');
+            if (dbName !== db.databaseName)
+                continue;
+            const coll = db.collection(collName);
+            coll
+                .bulkWrite(evs.map((e) => ({
+                updateOne: {
+                    filter: { _id: e.doc.o['_id'] ?? e.doc.o2?._id },
+                    update: { $set: { touchedAt: e.doc.ts } },
+                },
+            })))
+                .catch(() => { });
+        }
+        for (const { fields, doc } of events) {
+            const m = watchers.get(doc.ns);
+            if (!m)
+                continue;
+            for (const { cb, keys } of m.values()) {
+                if (!keys || keys.some(k => fields.has(k))) {
+                    cb(doc);
+                }
             }
         }
     }
@@ -1666,11 +1764,16 @@ const register = (coll, keys, cb) => {
             watchers.delete(ns);
     };
 };
-const makeWatchStream = ({ collection, projection: p, hardMatch: m }, streamName) => {
+let maxKeysRemoved = null;
+const makeWatchStream = async ({ collection, projection: p, hardMatch: m }, streamName) => {
+    const { db } = collection.s;
+    await (maxKeysRemoved ??= Promise.all((await db.listCollections({}, { nameOnly: true }).toArray()).map(x => void db
+        .collection(x.name)
+        .updateMany({ touchedAt: new mongodb.MaxKey() }, [{ $set: { touchedAt: '$$CLUSTER_TIME' } }]))).then(() => { }));
     const projection = { ...(p ? mapExactToObject(p, v => v) : {}), deletedAt: 1 };
     let resolve = (_) => { };
     const promise = new Promise(r => (resolve = r));
-    const close = register(collection, p ? Object.keys(projection) : null, (doc) => {
+    const close = register(collection, p ? Object.keys(projection) : null, doc => {
         log(streamName, 'change detected', doc);
         resolve(doc);
         close();
@@ -1679,9 +1782,10 @@ const makeWatchStream = ({ collection, projection: p, hardMatch: m }, streamName
         tryNext: async () => {
             const doc = await promise;
             const start = Date.now();
-            await waitUntilStablePast(collection.s.db, doc.ts);
+            if (doc)
+                await waitUntilStablePast(collection.s.db, doc.ts);
             log(streamName, 'stable past took', Date.now() - start);
-            return doc;
+            return doc ?? {};
         },
         close: async () => close(),
     };
@@ -1775,44 +1879,17 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
         streamNames[streamName] = hash;
     else if (streamNames[streamName] != hash)
         throw new Error(`streamName ${streamName} already used`);
-    db.command({
-        collMod: coll,
-        changeStreamPreAndPostImages: { enabled: true },
-    });
-    createIndex(collection, { touchedAt: 1 }, hardMatch
-        ? {
-            partialFilterExpression: hardMatch.raw(root()),
-            name: 'touchedAt_hard_' + new mongodb.UUID().toString('base64'),
-        }
-        : {}).catch(e => e.code == 86 || Promise.reject(e));
     const last = db.collection('__last');
     const snapshotCollection = db.collection(coll + '_' + streamName + '_snapshot');
-    createIndex(snapshotCollection, { before: 1 }, {
-        partialFilterExpression: { before: null },
-        name: 'before_' + new mongodb.UUID().toString('base64'),
-    });
-    createIndex(snapshotCollection, { updated: 1 }, {
-        partialFilterExpression: { updated: true },
-        name: 'updated_' + new mongodb.UUID().toString('base64'),
-    });
-    createIndex(snapshotCollection, { updated: 1, after: 1, before: 1 }, {
-        partialFilterExpression: { updated: true, after: null, before: null },
-        name: 'updated_nulls_' + new mongodb.UUID().toString('base64'),
-    });
-    createIndex(snapshotCollection, { updated: 1, after: 1 }, {
-        partialFilterExpression: { updated: true, after: null },
-        name: 'updated_no_after_' + new mongodb.UUID().toString('base64'),
-    });
-    createIndex(snapshotCollection, { updated: 1 }, {
-        partialFilterExpression: { updated: true, after: null, before: null },
-        name: 'updated_nulls_' + new mongodb.UUID().toString('base64'),
-    });
     const job = {};
-    const run = (finalInput) => {
-        const clear = async () => Promise.all([
-            snapshotCollection.drop().catch(noop).catch(noop),
-            last.deleteOne({ _id: streamName }),
-        ]);
+    const run = (finalInput, setup) => {
+        const dropSnapshot = async () => {
+            await snapshotCollection.drop().catch(noop);
+            log('snapshot collection dropped', streamName, `db['${snapshotCollection.collectionName}'].drop()`);
+            log('with', [...(indexMap.get(snapshotCollection.collectionName)?.keys() ?? [])], 'indexes in map before deletion');
+            indexMap.delete(snapshotCollection.collectionName);
+        };
+        const clear = async () => Promise.all([dropSnapshot(), last.deleteOne({ _id: streamName })]);
         const withStop = (next, tr) => {
             return addTeardown(() => ({ stop, next: next(), clear }), tr);
         };
@@ -1837,7 +1914,9 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
         const step0 = () => synchronousPromise.SynchronousPromise.resolve(next(step1, 'empty new collection'));
         const stop = withStop(step0);
         const step1 = async () => {
+            log('reset collection', streamName, `db['${snapshotCollection.collectionName}'].updateMany( updated: true }, { $set: { updated: false, after: null } })`);
             await snapshotCollection.updateMany({ updated: true }, { $set: { updated: false, after: null } });
+            log('reset collection done', streamName);
             return next(step2, 'get last update');
         };
         const step2 = () => Promise.all([
@@ -1860,13 +1939,45 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
                 };
                 const [action, out] = actions[method](collection, params);
                 log('teardown', `db['${snapshotCollection.collectionName}'].drop()`, ...out);
-                await Promise.all([snapshotCollection.drop().catch(noop), action]);
+                await Promise.all([dropSnapshot(), action]);
                 log('teardown done', `db['${snapshotCollection.collectionName}'].drop()`, ...out);
             };
             if (!same) {
                 log('not same, new data', streamName, data);
                 await handleTeardown(exists ?? { data });
             }
+            log('creating indexes');
+            await createIndex(snapshotCollection, { before: 1 }, {
+                partialFilterExpression: { before: null },
+                name: 'before_' + new mongodb.UUID().toString('base64'),
+            });
+            await createIndex(snapshotCollection, { updated: 1 }, {
+                partialFilterExpression: { updated: true },
+                name: 'updated_' + new mongodb.UUID().toString('base64'),
+            });
+            await createIndex(snapshotCollection, { updated: 1, after: 1, before: 1 }, {
+                partialFilterExpression: { updated: true, after: null, before: null },
+                name: 'updated_nulls_' + new mongodb.UUID().toString('base64'),
+            });
+            await createIndex(snapshotCollection, { updated: 1, after: 1 }, {
+                partialFilterExpression: { updated: true, after: null },
+                name: 'updated_no_after_' + new mongodb.UUID().toString('base64'),
+            });
+            await createIndex(snapshotCollection, { updated: 1 }, {
+                partialFilterExpression: { updated: true, after: null, before: null },
+                name: 'updated_nulls_' + new mongodb.UUID().toString('base64'),
+            });
+            await db.command({
+                collMod: coll,
+                changeStreamPreAndPostImages: { enabled: true },
+            });
+            await createIndex(collection, { touchedAt: 1 }, hardMatch
+                ? {
+                    partialFilterExpression: hardMatch.raw(root()),
+                    name: 'touchedAt_hard_' + new mongodb.UUID().toString('base64'),
+                }
+                : {});
+            await setup?.();
             await after?.();
             return nextData([])(async () => {
                 await new Promise(resolve => setTimeout(resolve, 1000));
@@ -1906,7 +2017,8 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
             const first = ts === undefined;
             const stages = finalInput.raw(first);
             await last.updateOne({ _id: streamName }, { $set: { job: 1 } }, { upsert: true });
-            const stream = makeStream();
+            const stream = await makeStream();
+            const nextRes = stream.tryNext();
             const aggResult = await aggregate(pdb, streamName, c => c({
                 coll: snapshotCollection,
                 input: link()
@@ -1920,7 +2032,6 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
                     .with(input.delta)
                     .with(stages).stages,
             }), false, start);
-            const nextRes = stream.tryNext();
             stages.at(-1).$merge.into.coll;
             return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream, nextRes, first }), 'remove handled deleted updated', () => stream.close());
         };
@@ -1999,14 +2110,6 @@ const executes$1 = (view, input, streamName, needs) => {
         : pre;
     const job = {};
     const db = collection.s.db, coll = collection.collectionName;
-    db.command({
-        collMod: coll,
-        changeStreamPreAndPostImages: { enabled: true },
-    });
-    createIndex(collection, { touchedAt: 1 }, {
-        partialFilterExpression: { deletedAt: { $eq: null } },
-        name: 'touchedAt_' + new mongodb.UUID().toString('base64'),
-    });
     const last = db.collection('__last');
     const projectInput = projection &&
         $project_(spread(projection, {
@@ -2043,10 +2146,26 @@ const executes$1 = (view, input, streamName, needs) => {
         };
         const step0 = () => synchronousPromise.SynchronousPromise.resolve(next(step1, 'get last update'));
         const stop = withStop(step0);
-        const step1 = () => Promise.all([
-            last.findOne({ _id: streamName, data }),
-            last.findOne({ _id: streamName }),
-        ]).then(ts => next(step2_5(ts), 'handle teardown'));
+        const step1 = async () => {
+            log('creating indexes');
+            await db.command({
+                collMod: coll,
+                changeStreamPreAndPostImages: { enabled: true },
+            });
+            await createIndex(collection, { touchedAt: 1 }, {
+                partialFilterExpression: { deletedAt: { $eq: null } },
+                name: 'touchedAt_' + new mongodb.UUID().toString('base64'),
+            });
+            log('start stream', { streamName, data });
+            await last.findOne();
+            console.log('got last update');
+            const p = last.findOne({ _id: streamName, data });
+            await p;
+            log('stream started', { streamName, data });
+            const ts = await Promise.all([p, last.findOne({ _id: streamName })]);
+            log('got last update', { streamName, ts });
+            return next(step2_5(ts), 'handle teardown');
+        };
         const step2_5 = ([same, exists]) => async () => {
             const handleTeardown = async (last) => {
                 if (!last.data)
@@ -2076,7 +2195,7 @@ const executes$1 = (view, input, streamName, needs) => {
         const makeStream = () => makeWatchStream(view, streamName);
         const step4 = (lastTS) => async () => {
             const raw = stages(lastTS).with(finalInput.raw(lastTS === null)).stages;
-            const stream = makeStream();
+            const stream = await makeStream();
             const aggResult = await aggregate(pdb, streamName, c => c({
                 coll: collection,
                 input: raw,
@@ -2245,7 +2364,6 @@ exports.range = range;
 exports.regex = regex;
 exports.root = root;
 exports.set = set;
-exports.setF = setF;
 exports.setField = setField;
 exports.single = single;
 exports.size = size;

package/package.json

@@ -3,7 +3,7 @@
     "module": "index.esm.js",
     "typings": "index.d.ts",
     "name": "@omegup/msync",
-    "version": "0.1.21",
+    "version": "0.1.23",
     "dependencies": {
         "dayjs": "^1.11.9",
         "dotenv": "^16.3.1",