@omegup/msync 0.1.20 → 0.1.22

package/index.js

@@ -1,9 +1,9 @@
 'use strict';
 
+var mongodb = require('mongodb');
 var crypto$1 = require('crypto');
 var jsonCanonicalize = require('json-canonicalize');
 var synchronousPromise = require('synchronous-promise');
-var mongodb = require('mongodb');
 var promises = require('fs/promises');
 
 const asExprRaw = (raw) => ({ get: () => raw });
@@ -84,7 +84,7 @@ const val = (val) => asExpr({
         : val),
 });
 const current = asExpr({
-    raw: () => asExprRaw('$$CLUSTER_TIME'),
+    raw: () => asExprRaw(new mongodb.MaxKey()),
 });
 const $let = (vars, inExpr) => asExpr({
     raw: f => asExprRaw({
@@ -1306,16 +1306,18 @@ const nextWinner = (previousWinner, previousWinnerNextFrame, sources, interrupt)
 };
 
 const mergeIterators = (params) => {
-    const { sources, interrupt, select = race } = params;
+    const { sources, interrupt, select = race, hooks } = params;
     const reiterate = (winner) => {
         const { frame, key } = winner;
         return {
             cont: () => {
                 const result = frame.cont();
+                hooks?.start?.(frame, result);
                 return mergeIterators({
                     sources: patch(sources, key, result),
                     interrupt,
                     select: sources => nextWinner(winner, result.next, sources, interrupt),
+                    hooks,
                 });
             },
             data: frame.data,
@@ -1323,7 +1325,7 @@ const mergeIterators = (params) => {
         };
     };
     return {
-        stop: () => mergeIterators({ sources: restart(sources), interrupt }),
+        stop: () => mergeIterators({ sources: restart(sources), interrupt, select, hooks }),
         next: select(sources).then(reiterate),
         clear: async () => {
             for (const key in sources) {
@@ -1333,56 +1335,37 @@ const mergeIterators = (params) => {
     };
 };
 
-const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
-const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
-const json = (a) => replace(JSON.stringify(a));
-const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
-
 const state = { steady: false, f: (_) => Promise.resolve() };
 let timeout = null;
-const setF = (f) => {
-    state.f = f;
-};
-const aggregate = (db, streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
-    const req = {
-        aggregate: coll.collectionName,
-        pipeline: input,
-        cursor: {},
-        ...(snapshot && { readConcern: { level: 'snapshot' } }),
-    };
-    if (timeout !== null) {
-        clearTimeout(timeout);
-        timeout = null;
-    }
-    log('exec', streamName, req);
-    const start2 = Date.now();
-    return db.then(d => d.command(req)).then(result => {
-        log('prepare', streamName, Date.now() - start);
-        log('prepare2', streamName, start2 - start);
-        const r = result;
-        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('"$$CLUSTER_TIME"', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
-        if (!state.steady) {
-            if (timeout !== null)
-                throw new Error('timeout should be null');
-            timeout = setTimeout(() => {
-                state.steady = true;
-                console.log('steady');
-            }, 10000);
-        }
-        return r;
-    }, err => {
-        log('err', req, err);
-        throw new Error(err);
-    });
-});
-
 const firstWorksMerge = (iters) => {
     const iterator = () => {
         const results = iters.map(iter => iter());
         const sources = { ...results };
         return mergeIterators({
             sources,
-            interrupt: key => state.steady
+            interrupt: key => false,
+            hooks: {
+                start: (frame, result) => {
+                    if (!frame.info.job)
+                        return;
+                    if (timeout !== null) {
+                        clearTimeout(timeout);
+                        timeout = null;
+                    }
+                    result.next.then(() => {
+                        if (!frame.info.job)
+                            return;
+                        if (!state.steady) {
+                            if (timeout !== null)
+                                clearTimeout(timeout);
+                            timeout = setTimeout(() => {
+                                state.steady = true;
+                                console.log('steady');
+                            }, 2000);
+                        }
+                    });
+                },
+            },
         });
     };
     return iterator;
@@ -1512,10 +1495,7 @@ const $insertX = (out, expr, map, ext, extExpr) => {
         teardown: c => c({
             collection: out,
             method: 'updateMany',
-            params: [
-                filter,
-                [{ $set: { deletedAt: '$$NOW', touchedAt: '$$CLUSTER_TIME' } }],
-            ],
+            params: [filter, [{ $set: { deletedAt: '$$NOW', touchedAt: '$$CLUSTER_TIME' } }]],
         }),
         raw: () => {
             const replacer = map(mergeObjects(expr, field(mergeExpr(extExpr, {
@@ -1546,6 +1526,32 @@ const $insertPart = (out, ext) => {
 const $insert = (out) => $insertPart(out, {});
 const assertNotNull = (expr) => expr;
 
+const T = (s) => `Timestamp(${parseInt(`${BigInt(s) / 2n ** 32n}`)}, ${parseInt(`${BigInt(s) % 2n ** 32n}`)})`;
+const replace = (s) => s.replace(/\{"\$timestamp":"(\d+)"\}/g, (_, d) => T(d));
+const json = (a) => replace(JSON.stringify(a));
+const log = (...args) => console.log(new Date(), ...args.map(a => (typeof a === 'function' ? a(replace) : a && typeof a === 'object' ? json(a) : a)));
+
+const aggregate = (db, streamName, input, snapshot = true, start = Date.now()) => input(({ coll, input }) => {
+    const req = {
+        aggregate: coll.collectionName,
+        pipeline: input,
+        cursor: {},
+        ...(snapshot && { readConcern: { level: 'snapshot' } }),
+    };
+    log('exec', streamName, req);
+    const start2 = Date.now();
+    return db.then(d => d.command(req)).then(result => {
+        log('prepare', streamName, Date.now() - start);
+        log('prepare2', streamName, start2 - start);
+        const r = result;
+        log('execed', streamName, (replace) => replace(JSON.stringify(req).replaceAll('"$$CLUSTER_TIME"', JSON.stringify(r.cursor.atClusterTime))), result, 'took', Date.now() - start);
+        return r;
+    }, err => {
+        log('err', req, err);
+        throw new Error(err);
+    });
+});
+
 const addTeardown = (it, tr) => {
     if (!tr)
         return it;
@@ -1573,7 +1579,7 @@ async function getLastCommittedTs(adminDb) {
     const st = await adminDb.command({ replSetGetStatus: 1 });
     return st?.optimes?.lastCommittedOpTime?.ts ?? null;
 }
-async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000, } = {}) {
+async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000 } = {}) {
     const adminDb = db.client.db('admin');
     const deadline = Date.now() + timeoutMs;
     while (true) {
@@ -1581,7 +1587,7 @@ async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000
         if (stable && stable.comp(oplogTs) >= 0)
             return;
         if (Date.now() > deadline) {
-            throw new Error("Timed out waiting for stable timestamp to reach oplog event time");
+            throw new Error('Timed out waiting for stable timestamp to reach oplog event time');
         }
         await sleep(pollMs);
     }
@@ -1603,29 +1609,37 @@ async function* tailOplog(db, opts) {
         try {
             for await (const doc of cursor) {
                 lastTs = doc.ts;
-                if (doc.op === 'i') {
-                    yield { ns: doc.ns, fields: new Set(Object.keys(doc.o)), doc };
+                if (doc.op === 'i' || '_id' in doc.o) {
+                    const fields = new Set(Object.keys(doc.o));
+                    fields.delete('_id');
+                    yield { fields, doc, changeTouched: doc.o['touchedAt'] instanceof mongodb.MaxKey };
                 }
                 else {
+                    let changeTouched = false;
                     if (doc.o['$v'] !== 2) {
-                        throw new Error(`Expected update with $v: 2, got ${JSON.stringify(doc.o)}`);
+                        throw new Error(`Expected update with $v: 2, got ${JSON.stringify(doc)}`);
                     }
                     const updatedFields = [];
                     const diff = doc.o['diff'];
                     for (const updateOp in diff) {
                         if (['u', 'i', 'd'].includes(updateOp)) {
                             updatedFields.push(...Object.keys(diff[updateOp]));
+                            if (diff[updateOp]['touchedAt'] instanceof mongodb.MaxKey) {
+                                changeTouched = true;
+                            }
                         }
                         else if (updateOp.startsWith('s')) {
                             updatedFields.push(updateOp.slice(1));
                         }
                     }
-                    yield { ns: doc.ns, fields: new Set(updatedFields), doc };
+                    yield { fields: new Set(updatedFields), doc, changeTouched };
                 }
             }
         }
         catch (e) {
-            log('oplog loop error', e);
+            log('oplog loop error, notifying watchers and reopening');
+            console.error(e);
+            yield null;
         }
         finally {
             log('oplog loop ended');
@@ -1636,16 +1650,76 @@ async function* tailOplog(db, opts) {
 }
 const watchers = new Map();
 let running = false;
+const makePromise = () => {
+    let resolve = () => { };
+    let promise = new Promise(r => (resolve = r));
+    return { promise, resolve };
+};
 const loop = async (db) => {
     log('starting oplog loop');
-    for await (const { ns, fields, doc } of tailOplog(db, {})) {
-        log('oplog event', ns, doc.op, [...fields]);
-        const m = watchers.get(ns);
-        if (!m)
+    let notify = makePromise();
+    let batch = [];
+    const last = db.collection('__last');
+    const run = async () => {
+        for await (const event of tailOplog(db, {
+            since: (await last.findOne({ _id: 'oplog' }))?.ts,
+        })) {
+            if (event?.fields.size === 0)
+                continue;
+            batch = event && batch ? [...batch, event] : null;
+            notify.resolve();
+        }
+    };
+    run();
+    const iter = async function* () {
+        while (true) {
+            await notify.promise;
+            const b = batch;
+            if (b?.length) {
+                last
+                    .updateOne({ _id: 'oplog' }, { $set: { ts: b[b.length - 1].doc.ts } }, { upsert: true })
+                    .catch(() => { });
+            }
+            batch = [];
+            notify = makePromise();
+            yield b;
+        }
+    };
+    for await (const events of iter()) {
+        if (!events) {
+            log('notifying watchers of oplog loop restart');
+            for (const m of watchers.values()) {
+                for (const { cb } of m.values()) {
+                    cb(null);
+                }
+            }
             continue;
-        for (const { cb, keys } of m.values()) {
-            if (!keys || keys.some(k => fields.has(k))) {
-                cb(doc);
+        }
+        const groups = Object.groupBy(events.filter(e => e.changeTouched), ev => ev.doc.ns);
+        for (const [ns, evs] of Object.entries(groups)) {
+            if (!evs)
+                continue;
+            const [dbName, collName] = ns.split('.');
+            if (dbName !== db.databaseName)
+                continue;
+            const coll = db.collection(collName);
+            coll
+                .bulkWrite(evs.map((e) => ({
+                updateOne: {
+                    filter: { _id: e.doc.o['_id'] ?? e.doc.o2?._id },
+                    update: { $set: { touchedAt: e.doc.ts } },
+                },
+            })))
+                .catch(() => { });
+        }
+        for (const { fields, doc } of events) {
+            const m = watchers.get(doc.ns);
+            if (!m)
+                continue;
+            for (const { cb, keys } of m.values()) {
+                if (!keys || keys.some(k => fields.has(k))) {
+                    cb(doc);
+                }
             }
         }
     }
@@ -1671,7 +1745,7 @@ const makeWatchStream = ({ collection, projection: p, hardMatch: m }, streamName
     const projection = { ...(p ? mapExactToObject(p, v => v) : {}), deletedAt: 1 };
     let resolve = (_) => { };
     const promise = new Promise(r => (resolve = r));
-    const close = register(collection, p ? Object.keys(projection) : null, (doc) => {
+    const close = register(collection, p ? Object.keys(projection) : null, doc => {
         log(streamName, 'change detected', doc);
         resolve(doc);
         close();
@@ -1680,9 +1754,10 @@ const makeWatchStream = ({ collection, projection: p, hardMatch: m }, streamName
         tryNext: async () => {
             const doc = await promise;
             const start = Date.now();
-            await waitUntilStablePast(collection.s.db, doc.ts);
+            if (doc)
+                await waitUntilStablePast(collection.s.db, doc.ts);
             log(streamName, 'stable past took', Date.now() - start);
-            return doc;
+            return doc ?? {};
         },
         close: async () => close(),
     };
@@ -1865,7 +1940,7 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
                 log('teardown done', `db['${snapshotCollection.collectionName}'].drop()`, ...out);
             };
             if (!same) {
-                log('not same, new data', data);
+                log('not same, new data', streamName, data);
                 await handleTeardown(exists ?? { data });
             }
             await after?.();
@@ -1954,8 +2029,8 @@ const executes$2 = (view, input, streamName, skip = false, after, needs = {}) =>
                     job: null,
                 },
             };
-            if (l.ts)
-                patch.$set = data;
+            if (l.first)
+                patch.$set = { ...patch.$set, data };
             await last.updateOne({ _id: streamName }, patch, { upsert: true });
             log('updated __last', Date.now() - start, `db['${last.collectionName}'].updateOne({ _id: '${streamName}' }, `, patch, `, { upsert: true })`);
             return step8(l);
@@ -2246,7 +2321,6 @@ exports.range = range;
 exports.regex = regex;
 exports.root = root;
 exports.set = set;
-exports.setF = setF;
 exports.setField = setField;
 exports.single = single;
 exports.size = size;

package/lib/accumulators/index.d.ts

@@ -0,0 +1,10 @@
+import type { AppMap, Arr, N, O, Rec } from '../../types';
+import type { ExprHKT } from '../expression/concat';
+import { type NoRaw } from '../expression/val';
+import type { DeltaAccumulator, Expr, Part } from '../types';
+export declare const $sum: <D extends O, C = unknown>(expr: Expr<number | N, D, C>) => DeltaAccumulator<D, number, C>;
+export declare const $accumulator: <D, T, Ctx, A extends readonly unknown[]>(init: () => NoRaw<T>, accumulateArgs: AppMap<ExprHKT<Part<D>, Ctx>, A>, accumulate: (a: NoRaw<T>, ...args: NoRaw<A>) => NoRaw<T>, merge: (...args: NoRaw<[T | N, T]>) => NoRaw<T>) => DeltaAccumulator<D, T, Ctx>;
+export declare const $countDict: <D extends O, C = unknown>(expr: Expr<string, D, C>) => DeltaAccumulator<D, Rec<string, number>, C>;
+export declare const $pushDict: <D extends O, V, C = unknown>(key: Expr<string, D, C>, value: Expr<V, D, C>) => DeltaAccumulator<D, Rec<string, Rec<"0" | "1", Arr<V>>>, C>;
+export declare const $keys: <D extends O, C = unknown>(expr: Expr<Rec<string, number>, D, C>) => Expr<Arr<string>, D, C>;
+export declare const $entries: <D extends O, V, C = unknown>(expr: Expr<Rec<string, Rec<"1" | "0", Arr<V>>>, D, C>) => Expr<Arr<Rec<"k", string> & Rec<"v", V>>, D, C>;

package/lib/aggregate/$merge.d.ts

@@ -0,0 +1,28 @@
+import type { IdHKT, OPick, RWCollection, Timestamp } from '../../types';
+import type { doc, ID, N, O, rawItem, Rec, Replace, RORec, StrKey } from '../../types/json';
+import { type ExprsExact } from '../expression/concat';
+import type { Delta, Expr, IsDeleted, Model, RawStages, StreamRunnerParam, TS } from '../types';
+import { type Exact } from '../utils/map-object';
+type OutInputE<T, E, A = T | null> = ID & Rec<'after', A> & E;
+type Allowed<K extends string> = Exclude<K, keyof (TS & ID)>;
+export type Patch<V, KK extends StrKey<V> = StrKey<V>> = ((OPick<V, Allowed<KK>> & ID) | (Rec<Allowed<KK>, N> & ID)) & TS;
+type TakeDoc<V, E = ID, KK extends StrKey<V> = StrKey<V>> = OPick<V, Allowed<KK>> & E;
+type ND = {
+    readonly deletedAt?: null;
+};
+type SafeE<E> = Omit<E, `$${string}` | keyof ID>;
+export declare const getWhenMatchedForMerge: <Out extends Model, P extends Model, K extends keyof IsDeleted>(whenNotMatched: "discard" | "fail" | "insert") => RawStages<O, Out, Out | Replace<Out, P>, RORec<"new", Replace<P, RORec<K, Timestamp>>>>;
+export declare const getWhenMatched: <Out extends Model, P extends Model, K extends keyof IsDeleted>(whenNotMatched: "discard" | "fail" | "insert") => RawStages<O, Rec<"old" | "merged", Out | Replace<Out, P>>, Out | Replace<Out, P>>;
+type MergeCollection<V extends O, Out extends Model> = {
+    coll: RWCollection<Out | Replace<Out, Patch<V>> | Replace<Patch<V>, IsDeleted>, Out>;
+    whenNotMatched: 'discard';
+} | {
+    coll: RWCollection<Out | Replace<Out, Patch<V>>, Out>;
+    whenNotMatched: 'fail';
+};
+export declare const $mergeId: <V extends O>() => <SourcePart extends doc, Out extends Model, E = unknown, EEE extends RORec<string, rawItem> = {}>(out: MergeCollection<V, Out>, keys: ExprsExact<TakeDoc<V, unknown>, SourcePart>, id: Expr<string, OutInputE<TakeDoc<V>, E, null>>, ext: Exact<Omit<SafeE<EEE>, keyof (ND & TS)>, IdHKT>) => StreamRunnerParam<OutInputE<SourcePart, E>, "out">;
+export declare const $simpleMergePart: <V extends O>() => <Source extends doc, Out extends Model, EEE extends RORec<string, rawItem>>(out: MergeCollection<V, Out>, keys: ExprsExact<TakeDoc<V, unknown>, Source>, ext: Exact<Omit<SafeE<EEE>, keyof (ND & TS)>, IdHKT>) => StreamRunnerParam<Source, "out">;
+export declare const $simpleMerge: <V extends O>() => <Source extends doc, Out extends Model>(out: RWCollection<Out | Replace<Out, Patch<V>> | Replace<Patch<V>, IsDeleted>, Out>, keys: ExprsExact<TakeDoc<V, unknown>, Source>, whenNotMatched?: "fail" | "discard") => StreamRunnerParam<Source, "out">;
+export declare const $mergePart: <V extends O>() => <Out extends Model, SourcePart extends doc, EEE extends RORec<string, rawItem>>(out: RWCollection<Out | Replace<Out, Patch<V>>, Out>, keys: ExprsExact<TakeDoc<V, unknown>, SourcePart>, ext: Exact<Omit<SafeE<EEE>, keyof (ND & TS)>, IdHKT>) => StreamRunnerParam<Delta<SourcePart>, "out">;
+export declare const $merge: <V extends O>() => <Out extends Model, SourcePart extends doc>(out: RWCollection<Out | Replace<Out, Patch<V>>, Out>, keys: ExprsExact<TakeDoc<V, unknown>, SourcePart>) => StreamRunnerParam<Delta<SourcePart>, "out">;
+export {};

package/lib/aggregate/$upsert.d.ts

@@ -0,0 +1,15 @@
+import type { IdHKT, RWCollection } from '../../types';
+import type { ID, O, RORec, doc, rawItem } from '../../types/json';
+import { type ExprsExact } from '../expression/concat';
+import type { Del, Delta, Expr, StreamRunnerParam, TS } from '../types';
+import { type Exact } from '../utils/map-object';
+type ND = {
+    readonly deletedAt?: null;
+};
+type SafeE<E> = Omit<E, `$${string}` | keyof ID>;
+export type Merge<T extends doc, E> = Omit<SafeE<E>, keyof (ND & TS)> & ((T & ND & TS) | Del);
+export declare const $insertX: <T extends doc, D extends O, EEE extends RORec<string, rawItem>>(out: RWCollection<Merge<T, EEE>>, expr: Expr<T, D>, map: (x: Expr<T & ND & TS & Omit<SafeE<EEE>, keyof (ND & TS)>, D>) => Expr<Merge<T, EEE>, D>, ext: Exact<Omit<SafeE<EEE>, keyof (ND & TS)>, IdHKT>, extExpr: ExprsExact<Omit<SafeE<EEE>, keyof (ND & TS)>, unknown>) => StreamRunnerParam<D, "out">;
+export declare const $simpleInsert: <T extends doc>(out: RWCollection<Merge<T, {}>>) => StreamRunnerParam<T, "out">;
+export declare const $insertPart: <T extends doc, EEE extends RORec<string, rawItem>>(out: RWCollection<Merge<T, EEE>>, ext: Exact<Omit<SafeE<EEE>, keyof (ND & TS)>, IdHKT>) => StreamRunnerParam<Delta<T>, "out">;
+export declare const $insert: <T extends doc>(out: RWCollection<Merge<T, {}>>) => StreamRunnerParam<Delta<T>, "out">;
+export {};

package/lib/aggregate/group/$group-merge.d.ts

@@ -0,0 +1,12 @@
+import type { RWCollection, WriteonlyCollection } from '../../../types';
+import type { AsLiteral, ID, O, Rec, doc, notArr } from '../../../types/json';
+import { type ExprsExact } from '../../expression/concat';
+import type { Delta, DeltaAccumulators, Expr, StreamRunnerParam, TS } from '../../types';
+import type { MergeInto } from '../out';
+import { type Extra, type IdAndTsKeys, type Loose, type MergedInput, type Strict, type V_Grp } from './utils/sub-merge';
+type Denied<GID = never> = keyof (TS & ID) | GID;
+type GI<GG> = Exclude<GG, keyof TS>;
+export declare const $groupMerge: <T extends O, Grp extends notArr, V extends O, GG extends string, EE = {}, Out extends Loose<Grp, V, GG> = Loose<Grp, V, GG>>(id: Expr<Grp, T>, args: DeltaAccumulators<T, O & Omit<V, Denied<GI<GG>>>>, out: MergeInto<Strict<Grp, V, GG, EE>, Out, WriteonlyCollection<MergedInput<Out, V, Grp, GG, EE>>>, gid: AsLiteral<GI<GG>>, extra: ExprsExact<Extra<EE, V, GG>, V_Grp<V, GG, Grp>>, idPrefix?: string) => StreamRunnerParam<Delta<T>, "out">;
+export declare const $groupId: <T extends O, V extends O, EE = {}, Out extends Loose<string, V, "_id"> = Loose<string, V, "_id">>(id: Expr<string, T>, args: DeltaAccumulators<T, O & Omit<V, Denied>>, out: RWCollection<MergedInput<Out, V, string, "_id", EE>, Out>, extra: ExprsExact<Omit<EE, IdAndTsKeys | keyof Omit<V, IdAndTsKeys>>, doc & Omit<V, IdAndTsKeys>>) => StreamRunnerParam<Delta<T>, "out">;
+export declare const $group: <T extends O, Grp extends notArr, V extends O, EE = {}, Out extends Loose<Grp, V, "_grp"> = Loose<Grp, V, "_grp">>(id: Expr<Grp, T>, args: DeltaAccumulators<T, O & Omit<V, Denied<"_grp">>>, out: RWCollection<MergedInput<Out, V, Grp, "_grp", EE> | Strict<Grp, V, "_grp", EE>, Out>, extra: ExprsExact<Omit<EE, IdAndTsKeys | "_grp" | Exclude<keyof V, IdAndTsKeys | "_grp">>, Rec<"_grp", Grp> & Omit<V, IdAndTsKeys | "_grp">>, idPrefix?: string) => StreamRunnerParam<Delta<T>, "out">;
+export {};

package/lib/aggregate/group/index.d.ts

@@ -0,0 +1,2 @@
+export { $groupMerge, $groupId, $group } from './$group-merge';
+export type { Loose, Strict } from './utils/sub-merge';

package/lib/aggregate/group/utils/sub-group.d.ts

@@ -0,0 +1,6 @@
+import type { O, RORec, Rec } from '../../../../types';
+import { type ExprsExact } from '../../../expression/concat';
+import type { Delta, Expr, RawStages } from '../../../types';
+import type { DeltaAccumulators } from '../../../types/accumulator';
+export type WithItem<V, Grp> = Rec<'_id', Grp> & Rec<'item', O<V>>;
+export declare const subGroup: <T extends O, Grp, V extends O, GID extends string>(id: Expr<Grp, T>, args: DeltaAccumulators<T, V>, addGrp: <D extends Rec<"_id", Grp>>(src: ExprsExact<V, D>) => ExprsExact<RORec<GID, Grp> & V, D>) => RawStages<unknown, Delta<T>, Rec<GID, Grp> & V>;

package/lib/aggregate/group/utils/sub-merge.d.ts

@@ -0,0 +1,20 @@
+import type { AsLiteral, ID, O, Rec, Replace, WriteonlyCollection, notArr } from '../../../../types';
+import { type ExprsExact } from '../../../expression/concat';
+import type { Model, RawStages, TS } from '../../../types';
+import type { DeltaAccumulators } from '../../../types/accumulator';
+import { type MergeInto } from '../../out';
+type GI<GG> = Exclude<GG, keyof TS>;
+export type IdAndTsKeys = keyof (TS & ID);
+type V<VV, GG extends string> = Omit<VV, IdAndTsKeys | GI<GG>>;
+export type Prepare<Grp, GG extends string> = TS & ID & Rec<GI<GG>, Grp>;
+type Par<T> = {
+    [P in keyof T]?: T[P] | null;
+};
+export type Loose<Grp, VV, GG extends string> = Prepare<Grp, GG> & Par<V<VV, GG>>;
+export type Strict<Grp, VV, GG extends string, EE> = Prepare<Grp, GG> & V<VV, GG> & Omit<EE, IdAndTsKeys | GI<GG> | keyof V<VV, GG>>;
+export type V_Grp<VV, GG extends string, Grp> = Rec<GI<GG>, Grp> & V<VV, GG>;
+export type Extra<EE, VV, GG extends string> = Omit<EE, IdAndTsKeys | GI<GG> | keyof V<VV, GG>>;
+type OrReplace<T, V> = T | Replace<T, V>;
+export type MergedInput<Out, VV, Grp, GG extends string, EE> = OrReplace<Replace<Replace<Out, V<VV, GG>>, Extra<EE, VV, GG>> & Model, TS & ID & V_Grp<VV, GG, Grp> & Extra<EE, VV, GG>>;
+export declare const subMerge: <T extends O, Grp extends notArr, VV extends O, GG extends string, EE = {}, Out extends Loose<Grp, VV, GG> = Loose<Grp, VV, GG>>(args: DeltaAccumulators<T, V<VV, GG>>, out: MergeInto<Strict<Grp, VV, GG, EE>, Out, WriteonlyCollection<MergedInput<Out, VV, Grp, GG, EE>>>, gid: AsLiteral<GI<GG>>, extra: ExprsExact<Extra<EE, VV, GG>, V_Grp<VV, GG, Grp>>, idPrefix: string, first: boolean) => RawStages<unknown, V_Grp<VV, GG, Grp>, "out">;
+export {};

package/lib/aggregate/index.d.ts

@@ -0,0 +1,9 @@
+export * from './group';
+export * from './lookup';
+export * from './match';
+export * from './set';
+export * from './unwind';
+export * from './$merge';
+export * from './$upsert';
+export * from './out';
+export { $group_ } from './mongo-stages';

package/lib/aggregate/lookup/$lookup-delta.d.ts

@@ -0,0 +1,10 @@
+import type { AsLiteral, ID, N, O, Rec, doc, rawItem } from '../../../types';
+import { Field } from '../../field';
+import type { BA, Before, Delta, RawStages, TStages, UBefore } from '../../types';
+type s = string;
+type Both<K1 extends s, LE, KK2 extends s, RE, N1 extends null = never, N2 extends null = never> = Delta<Rec<K1, LE | N1> & Rec<Exclude<KK2, BA | K1>, RE | N2> & ID>;
+export declare const $lookupDelta: <LQ extends O, LE extends LQ & doc, RQ extends O, RE extends RQ & doc, BRB extends UBefore<RQ>, RS extends UBefore<RQ>, S extends rawItem, K1 extends s, KK2 extends s, N1 extends null = never, N2 extends null = never>({ field1, field2 }: {
+    field1: Field<LQ, S | N>;
+    field2: Field<RQ, S | N>;
+}, { coll, exec, input }: TStages<RS, UBefore<RQ>, BRB, Before<RE>>, k1: AsLiteral<K1>, k2: AsLiteral<Exclude<KK2, BA | K1>>, k: ([N1] extends [never] ? K1 : never) | ([N2] extends [never] ? Exclude<KK2, BA | K1> : never) | false, includeNull1?: N1, includeNull2?: N2) => RawStages<unknown, Delta<LE>, Both<K1, LE, KK2, RE, N1, N2>>;
+export {};

package/lib/aggregate/lookup/$lookup-raw.d.ts

@@ -0,0 +1,9 @@
+import type { App, AsLiteral, HKT, ID, O, Rec } from '../../../types';
+import { type Field } from '../../field';
+import type { Before, RawStages, TStages } from '../../types';
+type s = string;
+export declare const $lookupRaw: <LQ extends O, LE extends LQ & ID, RQ extends O, RE extends RQ & ID, BRB extends Before<RQ>, RS, S, As extends s, Null extends null = never>({ field1, field2 }: {
+    field1: Field<LQ, S>;
+    field2: Field<RQ, S>;
+}, { coll, exec, input }: TStages<RS, Before<RQ>, BRB, Before<RE>>, k2: AsLiteral<As>, k: "left", includeNull?: Null) => <F extends HKT<O, O>>(f: <T extends O>() => Field<App<F, T>, T>) => RawStages<App<F, LQ>, App<F, LE>, App<F, LE & Rec<As, RE> & ID>>;
+export {};

package/lib/aggregate/lookup/index.d.ts

@@ -0,0 +1,12 @@
+import type { AsLiteral, O, RORec, doc, notArr } from '../../../types';
+import { type Field } from '../../field';
+import type { SnapshotStream, SnapshotStreamExecutionResult } from '../../types/stream';
+type Params<As extends string, LQ extends O, RQ extends O, RE extends RQ, S extends notArr> = {
+    localField: Field<LQ, S>;
+    foreignField: Field<RQ, S>;
+    from: SnapshotStreamExecutionResult<RQ, RE>;
+    as: AsLiteral<As>;
+};
+export declare const $lookup: <As extends string, LQ extends doc, RQ extends O, RE extends RQ & doc, S extends notArr>(p: Params<As, LQ, RQ, RE, S>) => <LE extends LQ>(l: SnapshotStream<LQ, LE>) => SnapshotStream<LQ, LE & RORec<As, RE>>;
+export declare const $outerLookup: <As extends string, LQ extends doc, RQ extends O, RE extends RQ & doc, S extends notArr>(p: Params<As, LQ, RQ, RE, S>) => <LE extends LQ>(l: SnapshotStream<LQ, LE>) => SnapshotStream<LQ, LE & RORec<As, RE | null>>;
+export {};

package/lib/aggregate/match/$match-delta.d.ts

@@ -0,0 +1,3 @@
+import type { doc } from '../../../types';
+import type { Delta, Expr } from '../../types';
+export declare const $matchDelta: <T extends doc>(query: Expr<boolean, T>) => import("../../types").RawStages<unknown, Delta<T>, Delta<T>, unknown, number>;

package/lib/aggregate/match/$match.d.ts

@@ -0,0 +1,3 @@
+import type { doc } from '../../../types';
+import type { DeltaStages, Expr } from '../../types';
+export declare const $match: <T extends doc>(query: Expr<boolean, T>) => DeltaStages<T, T, T>;

package/lib/aggregate/match/index.d.ts

@@ -0,0 +1,2 @@
+export { $matchDelta } from './$match-delta';
+export { $match } from './$match';

package/lib/aggregate/mongo-stages.d.ts

@@ -0,0 +1,13 @@
+import type { Arr, O, Rec } from '../../types';
+import type { Accumulators, Expr, LookupArgs, Query, RawStages } from '../types';
+import type { Updater } from '../update/updater';
+export * from './raws';
+type s = string;
+export declare const $match_: <Q, T extends Q & O, C = unknown>(query?: Query<T, C>) => RawStages<O<T>, T, T, C>;
+export declare const $set_: <Q, T extends Q & O, V extends Q & O, C = unknown>(updater: Updater<T, T, V, C>) => RawStages<O<Q>, T, V, C, 1>;
+export declare const $replaceWith_: <T extends O, V extends O, C = unknown>(expr: Expr<V, T, C>) => RawStages<O, T, V, C, 1>;
+export declare const $unwind_: <T extends O, K extends s, U, Null extends null = never>(k: K, includeNull?: Null) => RawStages<O<T>, T & Rec<K, Arr<U>>, T & Rec<K, Null | U>>;
+export declare const $group_: <V extends O>() => <ID, T extends O, C = unknown>(id: Expr<ID, T, C>, args: Accumulators<T, V, C>) => RawStages<O, T, Rec<"_id", ID> & V, C, 1>;
+export declare const $documents_: <T extends O, C>(docs: Expr<Arr<T>, unknown, C>) => RawStages<unknown, null, T, C, 1>;
+export declare const $project_: <T extends O, K extends import("../..").StrKey<T>>(projection: import("../utils").ExactKeys<K>) => RawStages<T, T, T, unknown, 1>;
+export declare const $simpleLookup_: <T extends O, U extends O, R, K extends s, Ctx, C = unknown, S = string>(args: LookupArgs<T, U, R, K, Ctx, C, S>) => RawStages<O<T>, T, T & Rec<K, Arr<U>>, C, 1>;

package/lib/aggregate/out.d.ts

@@ -0,0 +1,37 @@
+import type { O, RWCollection, ReadonlyCollection, Replace, jsonItem } from '../../types';
+import type { ExprsExact } from '../expression/concat';
+import { type Field } from '../field';
+import type { RawStages } from '../types';
+export type MergeInto<T extends O, Out extends O, E = unknown> = {
+    whenNotMatched: 'insert';
+    into: RWCollection<T, Out> & E;
+} | {
+    whenNotMatched: 'discard' | 'fail';
+    into: ReadonlyCollection<Out> & E;
+};
+export type MergeArgs<T extends O, Out extends O, Ctx, In extends O> = {
+    on: Field<T, jsonItem> & Field<Out, jsonItem>;
+} & MergeInto<T, Out> & (({
+    stages?: never;
+} & ({
+    whenMatched: 'keepExisting' | 'fail';
+} | {
+    whenMatched: 'replace';
+    into: RWCollection<T, Out>;
+} | {
+    whenMatched: 'merge';
+    into: RWCollection<Replace<Out, T>, Out>;
+})) | {
+    stages: true;
+    into: RWCollection<In, Out>;
+    whenMatched: RawStages<unknown, Out, In, {
+        new: T;
+    }>;
+} | {
+    stages: 'ctx';
+    vars: ExprsExact<Ctx, T>;
+    into: RWCollection<In, Out>;
+    whenMatched: RawStages<unknown, Out, In, Ctx>;
+});
+export declare const $merge_: <T extends O, Out extends O = T, Ctx = unknown, In extends O = Out>({ into, on, whenNotMatched, ...notMatched }: MergeArgs<T, Out, Ctx, In>) => RawStages<unknown, T, "out", unknown, number>;
+export declare const $merge2: <T extends O, Out extends O = T, Ctx = unknown, In extends O = Out>(args: MergeArgs<T, Out, Ctx, In>) => RawStages<unknown, T, "out", unknown, number>;

package/lib/aggregate/prefix.d.ts

@@ -0,0 +1,33 @@
+import type { App, HKT, O2, O3, O, RawObj } from '../../types';
+import type { Delta, DeltaStages, FRawStages, RawStages, Stream, TStages } from '../types';
+import type { Equal } from '../utils/guard';
+type n = number;
+export declare const asStages: <Q, T extends Q, V extends Q, C = unknown, M extends n = number>(x: readonly RawObj[]) => RawStages<Q, T, V, C, M>;
+export declare const same: <T extends Dom, V extends Dom, F extends HKT<Dom, Q>, Dom = unknown, Q = unknown, C = unknown, M extends n = number>(_: Equal<Dom, T, V>) => RawStages<Q, App<F, T>, App<F, V>, C, M>;
+export declare const concatStages: <Q, T extends Q, V extends Q, W extends Q, C, M extends n = number>(part1: RawStages<Q, T, V, C, M>, part2: RawStages<Q, V, W, C, M>) => RawStages<Q, T, W, C, M>;
+export declare const concatFStages: <Q, T extends Q & O, V extends Q & O, W extends Q & O, C, M extends n = number>(part1: FRawStages<Q, T, V, C, M>, part2: FRawStages<Q, V, W, C, M>) => FRawStages<Q, T, W, C, M>;
+export declare const concatDelta: <Q extends O, T extends Q, V extends Q, W extends Q>(part1: DeltaStages<Q, T, V>, part2: DeltaStages<Q, V, W>) => DeltaStages<Q, T, W>;
+export type Concat<out Q, in T extends Q, out V extends Q, in out C, in out M extends n = n> = {
+    with: <Q2, W extends Q2>(extra: RawStages<Q | Q2, V, W, C, M>) => Concat<Q | Q2, T, W, C, M>;
+    stages: RawStages<Q, T, V, C, M>;
+};
+type FConcat<out Q, in T extends Q & O, out V extends Q & O, in out C, in out M extends n = n> = {
+    with: <Q2, W extends Q2 & O>(extra: FRawStages<Q | Q2, V, W, C, M>) => FConcat<Q | Q2, T, W, C, M>;
+    stages: FRawStages<Q, T, V, C, M>;
+};
+export type DeltaPipe<Q extends O, T extends Q, F extends HKT<O2>, G extends HKT<O3>> = {
+    with: <Q2 extends O, V extends Q2>(map: (a: Stream<Q, T, F, G>) => Stream<Q | Q2, V, F, G>) => DeltaPipe<Q | Q2, V, F, G>;
+    then: <Q2 extends O, V extends Q2>(next: App<G, [Q2 | T, T, V]>) => DeltaPipe<Q | Q2, V, F, G>;
+    get: () => App<F, [Q, T]>;
+};
+export declare const pipe: <Q extends O, S extends Q, T extends Q, F extends HKT<O2>, G extends HKT<O3>>(stream: Stream<Q, S, F, G>, s: App<G, [Q, S, T]>, concat: <Q_1 extends O, T_1 extends Q_1, V extends Q_1, W extends Q_1>(part1: App<G, [Q_1, T_1, V]>, part2: App<G, [Q_1, V, W]>) => App<G, [Q_1, T_1, W]>, empty: <T_2 extends Q>() => App<G, [Q, T_2, T_2]>) => DeltaPipe<Q, T, F, G>;
+type Link = <T, C = unknown, M extends n = n>() => Concat<T, T, T, C, M>;
+type FLink = <T extends O, C = unknown, M extends n = n>() => FConcat<T, T, T, C, M>;
+export declare const link: Link;
+export declare const flink: FLink;
+export declare const emptyDelta: <T extends O>() => {
+    delta: RawStages<Delta<T>, Delta<T>, Delta<T>, unknown, number>;
+    raw: <F extends HKT<O>>() => RawStages<App<F, T>, App<F, T>, App<F, T>, unknown, number>;
+};
+export declare const concatTStages: <S, Q, B extends Q, T extends Q, V extends Q>({ coll, exec, input }: TStages<S, Q, B, T>, stages: RawStages<Q, T, V>) => TStages<S, Q, B, V>;
+export {};