@livequery/mongodb 2.0.0 → 2.0.145

package/README.md

@@ -24,6 +24,7 @@ For local development in this workspace, `@livequery/core` is installed as a dev
 ```ts
 export * from './MongoDatasource.js'
 export * from './DataChangePayload.js'
+export * from './MongodbRealtime.js'
 export * from './types.js'
 ```
 
@@ -305,6 +306,76 @@ Returns:
 
 The collection cache key includes connection, database, and collection name to avoid reusing collection handles across tenants or connections.
 
+### `MongodbRealtime`
+
+MongoDB change stream watcher for realtime Livequery updates. This replaces the need to use the separate `@livequery/mongodb-mapper` package in native MongoDB projects.
+
+```ts
+import { WebsocketGateway } from '@livequery/core'
+import { MongoDatasource, MongodbRealtime } from '@livequery/mongodb'
+
+const routes = [
+  {
+    method: 'GET',
+    path: '/products',
+    collection: 'products',
+    realtime: true,
+  },
+]
+
+const datasource = new MongoDatasource({
+  connections: { default: client },
+  databases: ['main'],
+})
+
+await datasource.init(routes)
+
+const websocketGateway = new WebsocketGateway(server)
+
+new MongodbRealtime()
+  .watch(datasource.config, routes)
+  .subscribe(websocketGateway)
+```
+
+Realtime route requirements:
+
+- `method` must be `GET` or legacy method `0`.
+- `realtime` must be `true`.
+- `collection` must be a static string. Dynamic collection, database, or connection resolver functions are skipped because database watchers must be known up front.
+- When watching a `MongoClient`, `db` or `config.databases` decides which database names to watch. When watching a `Db`, that database is used directly.
+
+By default, `MongodbRealtime` enables MongoDB pre/post images with `collMod` and watches with `fullDocument: 'updateLookup'` and `fullDocumentBeforeChange: 'whenAvailable'`.
+
+Disable the `collMod` call when your deployment manages pre/post images separately:
+
+```ts
+new MongodbRealtime({ enablePreAndPostImages: false })
+```
+
+For nested collection refs, use `refFields` to map route params to document fields:
+
+```ts
+{
+  method: 'GET',
+  path: '/users/:id/posts',
+  collection: 'posts',
+  realtime: true,
+  refFields: {
+    id: 'userId',
+  },
+}
+```
+
+An inserted `{ _id: 'post1', userId: 'user1', title: 'Hello' }` emits:
+
+```ts
+{
+  ref: 'users/user1/posts',
+  type: 'added',
+  data: { id: 'post1', userId: 'user1', title: 'Hello' },
+}
+```
+
 ### `DataChangePayload<T>`
 
 Type-only realtime/change payload contract.
@@ -354,16 +425,18 @@ type RouteOptions = {
   collection: string | ((req: LivequeryRequest) => Promise<string> | string)
   db?: string | ((req: LivequeryRequest) => Promise<string> | string)
   connection?: string | ((req: LivequeryRequest) => Promise<string> | string)
+  refFields?: Record<string, string | { field: string, array?: boolean }>
   objectIdFields?: string[]
 }
 ```
 
 Fields:
 
-- `realtime`: optional marker for realtime routes. The current adapter does not use it for query execution.
+- `realtime`: marks a static collection route for `MongodbRealtime.watch()`. Query execution itself is unchanged.
 - `collection`: required collection name or resolver function.
 - `db`: optional database name or resolver function.
 - `connection`: optional connection name or resolver function.
+- `refFields`: optional mapping from nested route params to document fields for realtime ref formatting.
 - `objectIdFields`: top-level request fields that should be converted from valid string ids to `ObjectId`.
 
 Use function values when tenant, database, or collection depends on request keys.

package/build/src/MongoDatasource.d.ts

@@ -28,6 +28,10 @@ export type RouteOptions = {
     collection: string | ((req: LivequeryRequest) => Promise<string> | string);
     db?: string | ((req: LivequeryRequest) => Promise<string> | string);
     connection?: string | ((req: LivequeryRequest) => Promise<string> | string);
+    refFields?: Record<string, string | {
+        field: string;
+        array?: boolean;
+    }>;
     objectIdFields?: string[];
 };
 export declare class MongoDatasource extends Subject<WebsocketSyncPayload<LivequeryBaseEntity>> implements LivequeryDatasource<MongoDatasourceConfig, RouteOptions>, CoreLivequeryDatasource<RouteOptions> {
@@ -39,7 +43,7 @@ export declare class MongoDatasource extends Subject<WebsocketSyncPayload<Livequ
     init(routes: Array<LivequeryDatasourceInitConfig<RouteOptions>>): Promise<void>;
     init(config: MongoDatasourceConfig, routes: Array<LegacyRouteConfig<RouteOptions>>): Promise<void>;
     handle(ctx: LivequeryContext): Promise<{}>;
-    query(req: LivequeryRequest, options: RouteOptions): Promise<import("mongodb").DeleteResult | {
+    query(req: LivequeryRequest, options: RouteOptions): Promise<{
         items: any[];
         summary: any;
         cursor: {
@@ -62,6 +66,6 @@ export declare class MongoDatasource extends Subject<WebsocketSyncPayload<Livequ
         };
     } | {
         item: any;
-    } | import("mongodb").UpdateResult<any>>;
+    }>;
 }
 export {};

package/build/src/MongoDatasource.js

@@ -186,13 +186,27 @@ export class MongoDatasource extends Subject {
         };
     }
     async #put(req, collection) {
-        return await collection.updateOne(this.#keys(req), this.#update(req.body));
+        await collection.updateOne(this.#keys(req), this.#update(req.body));
+        return { item: this.#writtenItem(req) };
     }
     async #patch(req, collection) {
-        return await collection.updateOne(this.#keys(req), this.#update(req.body));
+        await collection.updateOne(this.#keys(req), this.#update(req.body));
+        return { item: this.#writtenItem(req) };
     }
     async #del(req, collection) {
-        return await collection.deleteOne(this.#keys(req));
+        await collection.deleteOne(this.#keys(req));
+        return { item: this.#writtenItem(req) };
+    }
+    #writtenItem(req) {
+        const keys = req.keys || {};
+        const isPlainBody = req.body && typeof req.body === 'object'
+            && !Object.keys(req.body).some(k => k.startsWith('$'));
+        const id = keys.id ?? req.doc_id;
+        return {
+            ...keys,
+            ...isPlainBody ? req.body : {},
+            ...id ? { id } : {}
+        };
     }
     #keys(req) {
         return Object.entries(req.keys).reduce((p, [k, c]) => {

package/build/src/MongoQuery.js

@@ -91,6 +91,9 @@ export class MongoQuery {
         }
         return stack.pop();
     }
+    static #escape_regex(value) {
+        return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    }
     static #parse_array(value) {
         if (Array.isArray(value))
             return value;
@@ -201,19 +204,25 @@ export class MongoQuery {
         };
     }
     static #parse_conditions(filters) {
+        const $match = this.#build_match(filters);
+        return Object.keys($match).length > 0 ? [{ $match }] : [];
+    }
+    static #build_match(filters) {
         if (!filters)
-            return [];
+            return {};
         const { ':and': and, ':or': or, ':not': not, ...rest } = filters;
         const $or = Object.entries(rest).filter(([k]) => k.endsWith(':like')).map(([k, v]) => {
             const key = k.split(':like')[0];
             const value = `${v}`;
             return {
-                [key]: { $regex: value }
+                [key]: { $regex: this.#escape_regex(value), $options: 'i' }
             };
         });
-        const $match = Object.entries(rest).reduce((p, [k, value]) => {
+        const fields = Object.entries(rest).reduce((p, [k, value]) => {
             if (k.startsWith('::'))
                 return p;
+            if (k.endsWith(':like'))
+                return p;
             const [key, expression] = k.split(':');
             const map = {
                 eq: () => ({ $eq: value }),
@@ -245,13 +254,28 @@ export class MongoQuery {
                     ...fn()
                 }
             };
-        }, $or.length > 0 ? { $or } : {});
-        return [
-            ...Object.keys($match).length > 0 ? [{ $match }] : [],
-            ...and && Object.keys(and).length > 0 ? [{ $expr: { $and: this.#parse_conditions(and) } }] : [],
-            ...or && Object.keys(and).length > 0 ? [{ $expr: { $or: this.#parse_conditions(or) } }] : [],
-            ...not && Object.keys(and).length > 0 ? [{ $not: { $and: this.#parse_conditions(not) } }] : [],
-        ];
+        }, {});
+        const clauses = [];
+        if (Object.keys(fields).length > 0)
+            clauses.push(fields);
+        if ($or.length > 0)
+            clauses.push({ $or });
+        const andMatch = and ? this.#build_match(and) : {};
+        if (Object.keys(andMatch).length > 0)
+            clauses.push(andMatch);
+        const orMatch = or ? this.#build_match(or) : {};
+        if (Object.keys(orMatch).length > 0) {
+            clauses.push({ $or: Object.entries(orMatch).map(([k, v]) => ({ [k]: v })) });
+        }
+        const notMatch = not ? this.#build_match(not) : {};
+        if (Object.keys(notMatch).length > 0) {
+            clauses.push({ $nor: Object.entries(notMatch).map(([k, v]) => ({ [k]: v })) });
+        }
+        if (clauses.length === 0)
+            return {};
+        if (clauses.length === 1)
+            return clauses[0];
+        return { $and: clauses };
     }
     static #build_search_query(req) {
         const search = req.options[":search"];
@@ -384,7 +408,40 @@ export class MongoQuery {
         ];
     }
     static #build_offset_paging(req) {
-        return [];
+        const limit = this.#get_limit(req);
+        const page = Math.max(1, Math.floor(Number(req.options[':page'])) || 1);
+        const skip = (page - 1) * limit;
+        const { pipelines, summary } = this.#parse_summary(req);
+        return [
+            {
+                $facet: {
+                    ...pipelines,
+                    items: [{ $skip: skip }, { $limit: limit }],
+                    total: [{ $count: 'count' }],
+                }
+            },
+            {
+                $project: {
+                    summary,
+                    items: 1,
+                    total: { $ifNull: [{ $arrayElemAt: ['$total.count', 0] }, 0] },
+                }
+            },
+            {
+                $project: {
+                    summary: 1,
+                    items: 1,
+                    has: {
+                        prev: { $gt: [skip, 0] },
+                        next: { $gt: ['$total', skip + limit] },
+                    },
+                    count: {
+                        prev: { $literal: skip },
+                        next: { $max: [{ $subtract: ['$total', skip + limit] }, 0] },
+                    },
+                }
+            }
+        ];
     }
     static #build_query_filter(req) {
         const { ":after": after, ":before": before, ':around': around, ":limit": _limit, ":page": _page, ":search": search, ...rest } = req.options;
@@ -436,7 +493,7 @@ export class MongoQuery {
                 summary: {}
             };
         }
-        const is_cursor_paging = req.options[':after'] || req.options[':before'] || req.options[':around'] || !req.options['page'];
+        const is_cursor_paging = req.options[':after'] || req.options[':before'] || req.options[':around'] || !req.options[':page'];
         const $sort = this.#get_sorter(req);
         const pipelines = [
             { $sort },

package/build/src/MongodbRealtime.d.ts

@@ -0,0 +1,23 @@
+import { Observable } from 'rxjs';
+import type { WebsocketSyncPayload } from './types.js';
+import type { MongoDatasourceConfig, RouteOptions } from './MongoDatasource.js';
+export type MongoRealtimeChangeType = 'added' | 'modified' | 'removed';
+export type MongoRealtimeRefField = string | {
+    field: string;
+    array?: boolean;
+};
+export type MongoRealtimeOptions = {
+    enablePreAndPostImages?: boolean;
+};
+export type MongoRealtimeRoute = {
+    path: string;
+    method: string | number;
+    options?: RouteOptions;
+    config?: RouteOptions;
+} & Partial<RouteOptions>;
+export declare class MongodbRealtime {
+    #private;
+    private options;
+    constructor(options?: MongoRealtimeOptions);
+    watch(config: MongoDatasourceConfig, routes: MongoRealtimeRoute[]): Observable<WebsocketSyncPayload>;
+}

package/build/src/MongodbRealtime.js

@@ -0,0 +1,221 @@
+import { EMPTY, Observable, from, map, mergeAll, mergeMap, retry } from 'rxjs';
+const changeTypes = {
+    insert: 'added',
+    update: 'modified',
+    replace: 'modified',
+    delete: 'removed',
+};
+export class MongodbRealtime {
+    options;
+    constructor(options = {}) {
+        this.options = options;
+    }
+    #reformatId(obj) {
+        if (!obj)
+            return undefined;
+        const { _id, __v, id, ...rest } = obj;
+        return {
+            id: _id ? String(_id) : id || '#',
+            ...Object.entries(rest).reduce((acc, [key, value]) => {
+                if (key.startsWith('_'))
+                    return acc;
+                return { ...acc, [key]: value };
+            }, {}),
+        };
+    }
+    #isGetRoute(method) {
+        return method === 0 || String(method).toUpperCase() === 'GET';
+    }
+    #getRouteOptions(route) {
+        if (route.options)
+            return route.options;
+        if (route.config)
+            return route.config;
+        const { method: _method, path: _path, options: _options, config: _config, ...options } = route;
+        return options.collection ? options : undefined;
+    }
+    #isDb(connection) {
+        return typeof connection.collection == 'function';
+    }
+    #collection(source) {
+        if (this.#isDb(source.connection))
+            return source.connection.collection(source.collection);
+        return source.connection.db(source.dbName).collection(source.collection);
+    }
+    #watchSources(config, routes) {
+        const sources = new Map();
+        for (const route of routes) {
+            const options = this.#getRouteOptions(route);
+            if (!options?.realtime || !this.#isGetRoute(route.method))
+                continue;
+            if (typeof options.collection != 'string')
+                continue;
+            if (typeof options.connection == 'function' || typeof options.db == 'function')
+                continue;
+            const connectionName = options.connection || Object.keys(config.connections)[0] || 'default';
+            const connection = config.connections[connectionName];
+            if (!connection)
+                continue;
+            const dbNames = this.#isDb(connection)
+                ? [undefined]
+                : options.db ? [options.db] : config.databases || ['main'];
+            for (const dbName of dbNames) {
+                const key = `${connectionName}|${dbName || ''}|${options.collection}`;
+                sources.set(key, { connection, dbName, collection: options.collection });
+            }
+        }
+        return [...sources.values()];
+    }
+    #listenRawChanges(config, routes) {
+        const sources = this.#watchSources(config, routes);
+        if (sources.length === 0)
+            return EMPTY;
+        return from(sources).pipe(mergeMap(async (source) => {
+            const collection = this.#collection(source);
+            if (this.options.enablePreAndPostImages !== false) {
+                await collection.db.command({
+                    collMod: source.collection,
+                    changeStreamPreAndPostImages: { enabled: true },
+                });
+            }
+            return new Observable(observer => {
+                const stream = collection.watch([], {
+                    fullDocument: 'updateLookup',
+                    fullDocumentBeforeChange: 'whenAvailable',
+                });
+                stream
+                    .on('error', error => observer.error(error))
+                    .on('change', (change) => {
+                    const type = changeTypes[change.operationType];
+                    if (!type)
+                        return;
+                    const fields = new Set(type == 'modified'
+                        ? [
+                            ...Object.keys(change.updateDescription?.updatedFields || {}).map(field => field.split('.')[0]),
+                            ...(change.updateDescription?.removedFields || []).map(field => field.split('.')[0]),
+                        ].filter(field => !field.startsWith('_'))
+                        : []);
+                    observer.next({
+                        table: change.ns.coll,
+                        type,
+                        new_data: this.#reformatId(change.fullDocument),
+                        old_data: this.#reformatId(change.fullDocumentBeforeChange),
+                        fields,
+                    });
+                });
+                return () => {
+                    void stream.close();
+                };
+            });
+        }), mergeMap(stream => stream), retry());
+    }
+    #routeRefMetadata(route) {
+        const options = this.#getRouteOptions(route);
+        if (!options?.realtime || !this.#isGetRoute(route.method))
+            return;
+        if (typeof options.collection != 'string')
+            return;
+        const segments = route.path.split('/').filter(Boolean);
+        const collectionSegments = segments.length % 2 === 0 ? segments.slice(0, -1) : segments;
+        const ref = collectionSegments.map(segment => segment.startsWith(':') ? '' : segment).filter(Boolean).join('/');
+        if (!ref)
+            return;
+        const metadata = collectionSegments
+            .map((collection, index) => {
+            if (index % 2 === 1)
+                return [];
+            const param = collectionSegments[index + 1];
+            if (!param?.startsWith(':'))
+                return [{ collection }];
+            const paramName = param.slice(1);
+            const mapped = options.refFields?.[paramName] || paramName;
+            const field = typeof mapped == 'string' ? mapped : mapped.field;
+            const array = typeof mapped == 'string' ? undefined : mapped.array;
+            return [{ collection, field: field == 'id' ? '_id' : field, array }];
+        })
+            .flat();
+        return [ref, metadata];
+    }
+    #paths(routes) {
+        return routes.reduce((paths, route) => {
+            const options = this.#getRouteOptions(route);
+            const entry = this.#routeRefMetadata(route);
+            if (!options || !entry || typeof options.collection != 'string')
+                return paths;
+            const [ref, metadata] = entry;
+            const refs = paths.get(options.collection) || new Map();
+            refs.set(ref, metadata);
+            paths.set(options.collection, refs);
+            return paths;
+        }, new Map());
+    }
+    #format(paths, event) {
+        const refs = paths.get(event.table);
+        if (!refs)
+            return [];
+        const merged = {
+            ...event.old_data || {},
+            ...event.new_data || {},
+        };
+        const changes = Object.keys(merged)
+            .filter(key => event.fields.has(key))
+            .reduce((acc, key) => ({ ...acc, [key]: event.new_data?.[key] }), { id: merged.id });
+        const typeForField = (field, pathValue) => {
+            const oldValue = event.old_data?.[field];
+            const newValue = event.new_data?.[field];
+            const value = String(pathValue);
+            if (Array.isArray(oldValue) || Array.isArray(newValue)) {
+                const oldArray = (Array.isArray(oldValue) ? oldValue : []).map(item => String(item));
+                const newArray = (Array.isArray(newValue) ? newValue : []).map(item => String(item));
+                if (oldArray.includes(value) && !newArray.includes(value))
+                    return 'removed';
+                if (!oldArray.includes(value) && newArray.includes(value))
+                    return 'added';
+                return 'modified';
+            }
+            if (String(newValue) == value && String(oldValue) != value)
+                return 'added';
+            if (String(oldValue) == value && String(newValue) != value)
+                return 'removed';
+            return 'modified';
+        };
+        const buildRefs = ([{ array, collection, field }, ...fields]) => {
+            if (fields.length === 0 || !field)
+                return [{ refs: [collection], type: event.type }];
+            const values = array
+                ? Array.isArray(merged[field])
+                    ? [...new Set([
+                            ...(event.old_data?.[field]?.map((item) => String(item)) || []),
+                            ...(event.new_data?.[field]?.map((item) => String(item)) || []),
+                        ])]
+                    : ['-']
+                : [merged[field] ?? '-'];
+            return values.flatMap(value => {
+                return buildRefs(fields).map(next => ({
+                    type: event.type == 'added' || event.type == 'removed'
+                        ? event.type
+                        : next.type == 'added' || next.type == 'removed'
+                            ? next.type
+                            : typeForField(field, value),
+                    refs: [collection, String(value), ...next.refs],
+                }));
+            });
+        };
+        return [...refs.values()].flatMap(metadata => {
+            return buildRefs(metadata).map(({ refs, type }) => ({
+                ref: refs.join('/'),
+                type,
+                data: type == 'added' ? merged : {
+                    ...type == 'modified' ? changes : {},
+                    id: merged.id,
+                },
+            }));
+        });
+    }
+    watch(config, routes) {
+        const paths = this.#paths(routes);
+        if (paths.size === 0)
+            return EMPTY;
+        return this.#listenRawChanges(config, routes).pipe(map(event => this.#format(paths, event)), mergeAll());
+    }
+}

package/build/src/SmartCache.d.ts

@@ -1,4 +1,4 @@
 export declare class SmartCache {
     #private;
-    get<T>(key: any, reslover: () => Promise<T>): Promise<any>;
+    get<T>(key: any, resolver: () => Promise<T>): Promise<any>;
 }

package/build/src/SmartCache.js

@@ -1,10 +1,10 @@
 export class SmartCache {
     #storage = new Map();
-    async get(key, reslover) {
+    async get(key, resolver) {
         const cache = this.#storage.get(key);
         if (cache)
             return await cache;
-        const value = reslover();
+        const value = resolver();
         this.#storage.set(key, value);
         return await value;
     }

package/build/src/index.d.ts

@@ -1,3 +1,4 @@
 export * from './MongoDatasource.js';
 export * from './DataChangePayload.js';
+export * from './MongodbRealtime.js';
 export * from './types.js';

package/build/src/index.js

@@ -1,3 +1,4 @@
 export * from './MongoDatasource.js';
 export * from './DataChangePayload.js';
+export * from './MongodbRealtime.js';
 export * from './types.js';

package/build/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"root":["../src/cursor.ts","../src/datachangepayload.ts","../src/mongodatasource.ts","../src/mongoquery.ts","../src/smartcache.ts","../src/index.ts","../src/types.ts"],"version":"5.9.3"}
+{"root":["../src/cursor.ts","../src/datachangepayload.ts","../src/mongodatasource.ts","../src/mongoquery.ts","../src/mongodbrealtime.ts","../src/smartcache.ts","../src/index.ts","../src/types.ts"],"version":"5.9.3"}

package/package.json

@@ -6,7 +6,7 @@
     "repository": {
         "url": "git@github.com:livequery/mongodb.git"
     },
-    "version": "2.0.0",
+    "version": "2.0.145",
     "description": "MongoDB datasource mapping for @livequery ecosystem",
     "main": "./build/src/index.js",
     "types": "./build/src/index.d.ts",