@omegup/msync 0.1.22 → 0.1.24

package/index.d.ts

@@ -94,7 +94,7 @@ type StreamRunnerParam<in V, out Result> = {
     raw: (first: boolean) => RawStages<unknown, V, Result>;
     teardown: <R>(consume: <W, M extends keyof Actions<W>>(x: TeardownRecord<W, M>) => R) => R;
 };
-type StreamRunner<out V> = <Result>(input: StreamRunnerParam<V, Result>) => Runner<readonly Result[], HasJob>;
+type StreamRunner<out V> = <Result>(input: StreamRunnerParam<V, Result>, setup?: () => Promise<void>) => Runner<readonly Result[], HasJob>;
 type SimpleStreamExecutionResult<out Q, out V extends Q> = {
     readonly out: StreamRunner<V>;
 };
@@ -536,7 +536,8 @@ declare const log: (...args: unknown[]) => void;
 
 declare const createIndex: (collection: {
     readonly createIndex: Collection["createIndex"];
-}, indexSpec: IndexSpecification, options?: CreateIndexesOptions) => Promise<void>;
+    collectionName: string;
+}, indexSpec: IndexSpecification, op?: CreateIndexesOptions) => Promise<void>;
 
 declare const noop: () => void;
 declare const map1: <K extends string, Im>(k: AsLiteral<K>, to: Im) => { readonly [P in K]: [P, Im]; } & {

package/index.esm.js

@@ -1,4 +1,4 @@
-import { MaxKey, MongoClient, UUID } from 'mongodb';
+import { MaxKey, UUID, MongoClient } from 'mongodb';
 import crypto$1 from 'crypto';
 import { canonicalize } from 'json-canonicalize';
 import { SynchronousPromise } from 'synchronous-promise';
@@ -1251,19 +1251,48 @@ const $lookupRaw = ({ field1, field2 }, { coll, exec, input }, k2, k, includeNul
 
 const asBefore = (f) => f(() => root().of('before'));
 
-const createIndex = async (collection, indexSpec, options) => {
+const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
+const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
+const json = (a) => replace(JSON.stringify(a));
+const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
+
+const indexMap = new Map();
+const createIndex = async (collection, indexSpec, op) => {
+    const { name, ...options } = op ?? {};
+    const map = indexMap.get(collection.collectionName) ?? new Map();
+    indexMap.set(collection.collectionName, map);
+    const indexKey = `${JSON.stringify(indexSpec)}-${JSON.stringify(options)}`;
+    if (map.has(indexKey)) {
+        await map.get(indexKey);
+        return;
+    }
+    const promise = createIndexWithRetry(collection, indexSpec, op);
+    map.set(indexKey, promise);
+    await promise;
+};
+const createIndexWithRetry = async (collection, indexSpec, options) => {
+    const log = () => { };
+    log('Creating index', { collection: collection.collectionName, indexSpec, options });
     while (true) {
         try {
             await collection.createIndex(indexSpec, options);
+            log('Index created', { collection: collection.collectionName, indexSpec, options });
         }
         catch (e) {
             if ([85, 276].includes(e.code)) {
+                log('Index created with different name', e.code, { collection: collection.collectionName, indexSpec, options });
                 break;
             }
             if (e.code == 12587) {
                 await new Promise(res => setTimeout(res, 300));
                 continue;
             }
+            log('Error creating index', {
+                collection: collection.collectionName,
+                indexSpec,
+                options,
+                error: e,
+            });
             console.error('Error creating index', e);
             throw e;
         }
@@ -1333,7 +1362,7 @@ const mergeIterators = (params) => {
     };
 };
 
-const state = { steady: false, f: (_) => Promise.resolve() };
+const state = { steady: false };
 let timeout = null;
 const firstWorksMerge = (iters) => {
     const iterator = () => {
@@ -1341,7 +1370,7 @@ const firstWorksMerge = (iters) => {
         const sources = { ...results };
         return mergeIterators({
             sources,
-            interrupt: key => false,
+            interrupt: key => state.steady,
             hooks: {
                 start: (frame, result) => {
                     if (!frame.info.job)
@@ -1409,8 +1438,6 @@ const runCont = async (it, cb) => {
 
 const merge = ({ lsource: L, rsource: R, }) => mergeIterators({ sources: { L, R } });
 const join = ({ lField, rField, left, right, as }, leftSnapshot, rightSnapshot, stagesUntilNextLookup, outerLeft) => {
-    createIndex(leftSnapshot.coll, { [`before.${lField.str()}`]: 1 }).catch(e => e.code == 86 || Promise.reject(e));
-    createIndex(rightSnapshot.coll, { [`before.${rField.str()}`]: 1 }).catch(e => e.code == 86 || Promise.reject(e));
     const rightJoinField = { field1: lField, field2: rField };
     const joinId = 'left';
     const joinR_Snapshot = asBefore($lookupRaw(rightJoinField, rightSnapshot, as, joinId, outerLeft));
@@ -1430,6 +1457,15 @@ const join = ({ lField, rField, left, right, as }, leftSnapshot, rightSnapshot,
             const getRunner = (f, stages, final) => f.out({
                 raw: first => concatStages(stages, final.raw(first)),
                 teardown: final.teardown,
+            }, async () => {
+                log('Creating indexes for lookup left', leftSnapshot.coll.collectionName, {
+                    [`before.${lField.str()}`]: 1,
+                });
+                await createIndex(leftSnapshot.coll, { [`before.${lField.str()}`]: 1 }, { name: 'left_' + new UUID().toString('base64') });
+                log('Creating indexes for lookup right', rightSnapshot.coll.collectionName, {
+                    [`before.${rField.str()}`]: 1,
+                });
+                await createIndex(rightSnapshot.coll, { [`before.${rField.str()}`]: 1 }, { name: 'right_' + new UUID().toString('base64') });
             });
             const lRunner = getRunner(left, lRunnerInput, finalInput);
             const rRunner = getRunner(right, rRunnerInput, finalInput);
@@ -1524,11 +1560,6 @@ const $insertPart = (out, ext) => {
 const $insert = (out) => $insertPart(out, {});
 const assertNotNull = (expr) => expr;
 
-const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
-const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
-const json = (a) => replace(JSON.stringify(a));
-const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
-
 const aggregate = (db, streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
     const req = {
         aggregate: coll.collectionName,
@@ -1657,11 +1688,8 @@ const loop = async (db) => {
     log('starting oplog loop');
     let notify = makePromise();
     let batch = [];
-    const last = db.collection('__last');
     const run = async () => {
-        for await (const event of tailOplog(db, {
-            since: (await last.findOne({ _id: 'oplog' }))?.ts,
-        })) {
+        for await (const event of tailOplog(db, {})) {
             if (event?.fields.size === 0)
                 continue;
             batch = event && batch ? [...batch, event] : null;
@@ -1673,11 +1701,6 @@ const loop = async (db) => {
         while (true) {
             await notify.promise;
             const b = batch;
-            if (b?.length) {
-                last
-                    .updateOne({ _id: 'oplog' }, { $set: { ts: b[b.length - 1].doc.ts } }, { upsert: true })
-                    .catch(() => { });
-            }
             batch = [];
             notify = makePromise();
             yield b;
@@ -1739,7 +1762,12 @@ const register = (coll, keys, cb) => {
             watchers.delete(ns);
     };
 };
-const makeWatchStream = ({ collection, projection: p, hardMatch: m }, streamName) => {
+let maxKeysRemoved = null;
+const makeWatchStream = async ({ collection, projection: p, hardMatch: m }, streamName) => {
+    const { db } = collection.s;
+    await (maxKeysRemoved ??= Promise.all((await db.listCollections({}, { nameOnly: true }).toArray()).map(x => void db
+        .collection(x.name)
+        .updateMany({ touchedAt: new MaxKey() }, [{ $set: { touchedAt: '$$CLUSTER_TIME' } }]))).then(() => { }));
     const projection = { ...(p ? mapExactToObject(p, v => v) : {}), deletedAt: 1 };
     let resolve = (_) => { };
     const promise = new Promise(r => (resolve = r));
@@ -1849,44 +1877,17 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
         streamNames[streamName] = hash;
     else if (streamNames[streamName] != hash)
         throw new Error(`streamName ${streamName} already used`);
-    db.command({
-        collMod: coll,
-        changeStreamPreAndPostImages: { enabled: true },
-    });
-    createIndex(collection, { touchedAt: 1 }, hardMatch
-        ? {
-            partialFilterExpression: hardMatch.raw(root()),
-            name: 'touchedAt_hard_' + new UUID().toString('base64'),
-        }
-        : {}).catch(e => e.code == 86 || Promise.reject(e));
     const last = db.collection('__last');
     const snapshotCollection = db.collection(coll + '_' + streamName + '_snapshot');
-    createIndex(snapshotCollection, { before: 1 }, {
-        partialFilterExpression: { before: null },
-        name: 'before_' + new UUID().toString('base64'),
-    });
-    createIndex(snapshotCollection, { updated: 1 }, {
-        partialFilterExpression: { updated: true },
-        name: 'updated_' + new UUID().toString('base64'),
-    });
-    createIndex(snapshotCollection, { updated: 1, after: 1, before: 1 }, {
-        partialFilterExpression: { updated: true, after: null, before: null },
-        name: 'updated_nulls_' + new UUID().toString('base64'),
-    });
-    createIndex(snapshotCollection, { updated: 1, after: 1 }, {
-        partialFilterExpression: { updated: true, after: null },
-        name: 'updated_no_after_' + new UUID().toString('base64'),
-    });
-    createIndex(snapshotCollection, { updated: 1 }, {
-        partialFilterExpression: { updated: true, after: null, before: null },
-        name: 'updated_nulls_' + new UUID().toString('base64'),
-    });
     const job = {};
-    const run = (finalInput) => {
-        const clear = async () => Promise.all([
-            snapshotCollection.drop().catch(noop).catch(noop),
-            last.deleteOne({ _id: streamName }),
-        ]);
+    const run = (finalInput, setup) => {
+        const dropSnapshot = async () => {
+            await snapshotCollection.drop().catch(noop);
+            log('snapshot collection dropped', streamName, `db['${snapshotCollection.collectionName}'].drop()`);
+            log('with', [...(indexMap.get(snapshotCollection.collectionName)?.keys() ?? [])], 'indexes in map before deletion');
+            indexMap.delete(snapshotCollection.collectionName);
+        };
+        const clear = async () => Promise.all([dropSnapshot(), last.deleteOne({ _id: streamName })]);
         const withStop = (next, tr) => {
             return addTeardown(() => ({ stop, next: next(), clear }), tr);
         };
@@ -1911,7 +1912,9 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
         const step0 = () => SynchronousPromise.resolve(next(step1, 'empty new collection'));
         const stop = withStop(step0);
         const step1 = async () => {
+            log('reset collection', streamName, `db['${snapshotCollection.collectionName}'].updateMany( updated: true }, { $set: { updated: false, after: null } })`);
             await snapshotCollection.updateMany({ updated: true }, { $set: { updated: false, after: null } });
+            log('reset collection done', streamName);
             return next(step2, 'get last update');
         };
         const step2 = () => Promise.all([
@@ -1934,13 +1937,45 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
                 };
                 const [action, out] = actions[method](collection, params);
                 log('teardown', `db['${snapshotCollection.collectionName}'].drop()`, ...out);
-                await Promise.all([snapshotCollection.drop().catch(noop), action]);
+                await Promise.all([dropSnapshot(), action]);
                 log('teardown done', `db['${snapshotCollection.collectionName}'].drop()`, ...out);
             };
             if (!same) {
                 log('not same, new data', streamName, data);
                 await handleTeardown(exists ?? { data });
             }
+            log('creating indexes');
+            await createIndex(snapshotCollection, { before: 1 }, {
+                partialFilterExpression: { before: null },
+                name: 'before_' + new UUID().toString('base64'),
+            });
+            await createIndex(snapshotCollection, { updated: 1 }, {
+                partialFilterExpression: { updated: true },
+                name: 'updated_' + new UUID().toString('base64'),
+            });
+            await createIndex(snapshotCollection, { updated: 1, after: 1, before: 1 }, {
+                partialFilterExpression: { updated: true, after: null, before: null },
+                name: 'updated_nulls_' + new UUID().toString('base64'),
+            });
+            await createIndex(snapshotCollection, { updated: 1, after: 1 }, {
+                partialFilterExpression: { updated: true, after: null },
+                name: 'updated_no_after_' + new UUID().toString('base64'),
+            });
+            await createIndex(snapshotCollection, { updated: 1 }, {
+                partialFilterExpression: { updated: true, after: null, before: null },
+                name: 'updated_nulls_' + new UUID().toString('base64'),
+            });
+            await db.command({
+                collMod: coll,
+                changeStreamPreAndPostImages: { enabled: true },
+            });
+            await createIndex(collection, { touchedAt: 1 }, hardMatch
+                ? {
+                    partialFilterExpression: hardMatch.raw(root()),
+                    name: 'touchedAt_hard_' + new UUID().toString('base64'),
+                }
+                : {});
+            await setup?.();
             await after?.();
             return nextData([])(async () => {
                 await new Promise(resolve => setTimeout(resolve, 1000));
@@ -1980,7 +2015,8 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
             const first = ts === undefined;
             const stages = finalInput.raw(first);
             await last.updateOne({ _id: streamName }, { $set: { job: 1 } }, { upsert: true });
-            const stream = makeStream();
+            const stream = await makeStream();
+            const nextRes = stream.tryNext();
             const aggResult = await aggregate(pdb, streamName, c => c({
                 coll: snapshotCollection,
                 input: link()
@@ -1994,7 +2030,6 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
                     .with(input.delta)
                     .with(stages).stages,
             }), false, start);
-            const nextRes = stream.tryNext();
             stages.at(-1).$merge.into.coll;
             return next(step5({ ts: result.cursor.atClusterTime, aggResult, stream, nextRes, first }), 'remove handled deleted updated', () => stream.close());
         };
@@ -2073,14 +2108,6 @@ const executes$1 = (view, input, streamName, needs) => {
         : pre;
     const job = {};
     const db = collection.s.db, coll = collection.collectionName;
-    db.command({
-        collMod: coll,
-        changeStreamPreAndPostImages: { enabled: true },
-    });
-    createIndex(collection, { touchedAt: 1 }, {
-        partialFilterExpression: { deletedAt: { $eq: null } },
-        name: 'touchedAt_' + new UUID().toString('base64'),
-    });
     const last = db.collection('__last');
     const projectInput = projection &&
         $project_(spread(projection, {
@@ -2117,10 +2144,24 @@ const executes$1 = (view, input, streamName, needs) => {
         };
         const step0 = () => SynchronousPromise.resolve(next(step1, 'get last update'));
         const stop = withStop(step0);
-        const step1 = () => Promise.all([
-            last.findOne({ _id: streamName, data }),
-            last.findOne({ _id: streamName }),
-        ]).then(ts => next(step2_5(ts), 'handle teardown'));
+        const step1 = async () => {
+            log('creating indexes');
+            await db.command({
+                collMod: coll,
+                changeStreamPreAndPostImages: { enabled: true },
+            });
+            await createIndex(collection, { touchedAt: 1 }, {
+                partialFilterExpression: { deletedAt: { $eq: null } },
+                name: 'touchedAt_' + new UUID().toString('base64'),
+            });
+            log('stream started', { streamName, data });
+            const ts = await Promise.all([
+                last.findOne({ _id: streamName, data }),
+                last.findOne({ _id: streamName }),
+            ]);
+            log('got last update', { streamName, ts });
+            return next(step2_5(ts), 'handle teardown');
+        };
         const step2_5 = ([same, exists]) => async () => {
             const handleTeardown = async (last) => {
                 if (!last.data)
@@ -2150,12 +2191,12 @@ const executes$1 = (view, input, streamName, needs) => {
         const makeStream = () => makeWatchStream(view, streamName);
         const step4 = (lastTS) => async () => {
             const raw = stages(lastTS).with(finalInput.raw(lastTS === null)).stages;
-            const stream = makeStream();
+            const stream = await makeStream();
+            const nextRes = stream.tryNext();
             const aggResult = await aggregate(pdb, streamName, c => c({
                 coll: collection,
                 input: raw,
             }));
-            const nextRes = stream.tryNext();
             return next(step7({ aggResult, ts: aggResult.cursor.atClusterTime, stream, nextRes }), 'update __last', () => stream.close());
         };
         const step7 = (l) => async () => {