@semics-tech/mongolite 1.4.0 → 1.4.2

package/dist/collection.js

@@ -4,6 +4,7 @@ import { extractRawIndexColumns } from './utils/indexing.js';
 import { ChangeStream } from './changeStream.js';
 import { applyProjectionToDocument, resolveProjectionMode } from './utils/projection.js';
 import { evaluateOperators } from './utils/queryOperators.js';
+import { evaluateExpression, evaluateExprFilter, filterUsesExpr, getPath, isOperatorExpression, SUPPORTED_STAGES, SUPPORTED_ACCUMULATORS, } from './utils/aggregation.js';
 /**
  * Safely stringifies JSON data with validation and error handling.
  * Prevents storing malformed JSON that could cause parsing issues later.
@@ -135,6 +136,10 @@ function validateDocumentForStorage(doc, path = 'root') {
         }
     }
 }
+/** A $project value that is a computed expression rather than an inclusion/exclusion flag. */
+function isComputedProjection(value) {
+    return (typeof value === 'string' && value.startsWith('$')) || isOperatorExpression(value);
+}
 /**
  * MongoLiteCollection provides methods to interact with a specific SQLite table
  * as if it were a MongoDB collection.
@@ -1043,25 +1048,26 @@ export class MongoLiteCollection {
         await this.ensureTable();
         const paramsForDelete = [];
         const whereClause = new FindCursor(this.db, this.name, filter)['buildGuardedWhereClause'](filter, paramsForDelete);
-        // Get the number of documents that would be deleted
-        const countSql = `SELECT COUNT(*) as count FROM "${this.name}" WHERE ${whereClause} LIMIT 1`;
-        const countResult = await this.db.get(countSql, paramsForDelete);
         const deleteSql = `
       DELETE FROM "${this.name}"
       WHERE ROWID IN (SELECT ROWID FROM "${this.name}" WHERE ${whereClause} LIMIT 1);
     `;
+        // Use the statement's own change count rather than a separate COUNT(*):
+        // `LIMIT 1` on an aggregate does not stop the scan, so the pre-count read every
+        // matching row (a full json_extract table scan on unindexed filters).
+        let result;
         try {
-            await this.db.run(deleteSql, paramsForDelete);
+            result = await this.db.run(deleteSql, paramsForDelete);
         }
         catch (error) {
             if (error.code === 'SQLITE_BUSY') {
-                await this.retryWithBackoff(async () => this.db.run(deleteSql, paramsForDelete), `delete for filter ${JSON.stringify(filter)}`);
+                result = await this.retryWithBackoff(async () => this.db.run(deleteSql, paramsForDelete), `delete for filter ${JSON.stringify(filter)}`);
             }
             else {
                 throw error;
             }
         }
-        return { acknowledged: true, deletedCount: countResult?.count ? 1 : 0 };
+        return { acknowledged: true, deletedCount: result.changes ? 1 : 0 };
     }
     /**
      * Deletes multiple documents matching the filter.
@@ -1096,11 +1102,7 @@ export class MongoLiteCollection {
      */
     async countDocuments(filter = {}) {
         await this.ensureTable();
-        const paramsForCount = [];
-        const whereClause = new FindCursor(this.db, this.name, filter)['buildGuardedWhereClause'](filter, paramsForCount);
-        const countSql = `SELECT COUNT(*) as count FROM "${this.name}" WHERE ${whereClause}`;
-        const result = await this.db.get(countSql, paramsForCount);
-        return result?.count || 0;
+        return this.find(filter).count();
     }
     /**
      * Finds rows whose stored `data` column is not valid JSON.
@@ -1343,7 +1345,10 @@ export class MongoLiteCollection {
     }
     /**
      * Executes an aggregation pipeline on the collection.
-     * Supports: $match, $project, $sort, $limit, $skip, $count, $group, $unwind, $addFields.
+     * Supports: $match (incl. $expr), $project, $sort, $limit, $skip, $count, $group
+     * ($sum, $avg, $min, $max, $push, $addToSet, $first, $last, $count), $unwind, $addFields/$set,
+     * $replaceRoot/$replaceWith and $lookup (equality form, local collections).
+     * Unknown stages and accumulators throw.
      * @param pipeline An array of pipeline stage documents.
      * @returns An object with a toArray() method that returns the aggregation result.
      */
@@ -1364,8 +1369,20 @@ export class MongoLiteCollection {
         // Only push the initial $match down to SQL when it is the very first stage.
         // Applying a $match that appears after other stages (e.g. after $unwind/$group) too
         // early would change pipeline semantics.
+        for (const stage of pipeline) {
+            const keys = Object.keys(stage);
+            if (keys.length !== 1) {
+                throw new Error('A pipeline stage specification object must contain exactly one field.');
+            }
+            if (!SUPPORTED_STAGES.includes(keys[0])) {
+                throw new Error(`Unrecognized or unsupported aggregation pipeline stage: ${keys[0]}`);
+            }
+        }
         const firstStage = pipeline[0];
-        const firstIsMatch = firstStage !== undefined && '$match' in firstStage;
+        // $expr can't be pushed to SQL; it's evaluated in memory instead.
+        const firstIsMatch = firstStage !== undefined &&
+            '$match' in firstStage &&
+            !filterUsesExpr(firstStage.$match);
         if (firstIsMatch) {
             const matchFilter = firstStage.$match;
             const docs = await this.find(matchFilter).toArray();
@@ -1393,9 +1410,43 @@ export class MongoLiteCollection {
                     results = results.map((doc) => this.applyAggregateProjection(doc, projection));
                     break;
                 }
-                case '$addFields': {
-                    const fields = stage['$addFields'];
-                    results = results.map((doc) => ({ ...doc, ...fields }));
+                case '$addFields':
+                case '$set': {
+                    const fields = stage[stageKey];
+                    results = results.map((doc) => {
+                        const out = structuredClone(doc);
+                        for (const [path, expr] of Object.entries(fields)) {
+                            const val = evaluateExpression(doc, expr);
+                            if (val === undefined)
+                                continue;
+                            if (path.includes('.'))
+                                this.setNestedValue(out, path, val);
+                            else
+                                out[path] = val;
+                        }
+                        return out;
+                    });
+                    break;
+                }
+                case '$replaceRoot':
+                case '$replaceWith': {
+                    const expr = stageKey === '$replaceRoot'
+                        ? stage['$replaceRoot'].newRoot
+                        : stage['$replaceWith'];
+                    if (expr === undefined) {
+                        throw new Error('$replaceRoot requires a newRoot expression');
+                    }
+                    results = results.map((doc) => {
+                        const root = evaluateExpression(doc, expr);
+                        if (typeof root !== 'object' || root === null || Array.isArray(root)) {
+                            throw new Error("'newRoot' expression must evaluate to an object");
+                        }
+                        return root;
+                    });
+                    break;
+                }
+                case '$lookup': {
+                    results = await this.applyLookupStage(results, stage['$lookup']);
                     break;
                 }
                 case '$sort': {
@@ -1437,22 +1488,43 @@ export class MongoLiteCollection {
                     break;
                 }
                 case '$unwind': {
-                    const path = typeof stage['$unwind'] === 'string'
-                        ? stage['$unwind']
-                        : stage['$unwind'].path;
-                    const fieldName = path.startsWith('$') ? path.slice(1) : path;
+                    const spec = stage['$unwind'];
+                    const opts = typeof spec === 'string' ? { path: spec } : spec;
+                    const path = opts.path;
+                    if (typeof path !== 'string' || !path.startsWith('$')) {
+                        throw new Error('$unwind requires a path starting with "$"');
+                    }
+                    const fieldName = path.slice(1);
+                    const preserve = opts.preserveNullAndEmptyArrays === true;
+                    const indexField = typeof opts.includeArrayIndex === 'string' ? opts.includeArrayIndex : undefined;
                     const unwound = [];
                     for (const doc of results) {
-                        const arrayVal = this.getNestedValue(doc, fieldName);
-                        if (Array.isArray(arrayVal)) {
-                            for (const item of arrayVal) {
-                                const newDoc = { ...doc };
+                        const arrayVal = getPath(doc, fieldName);
+                        if (Array.isArray(arrayVal) && arrayVal.length > 0) {
+                            arrayVal.forEach((item, idx) => {
+                                const newDoc = fieldName.includes('.') ? structuredClone(doc) : { ...doc };
                                 this.setNestedValue(newDoc, fieldName, item);
+                                if (indexField)
+                                    newDoc[indexField] = idx;
+                                unwound.push(newDoc);
+                            });
+                        }
+                        else if (Array.isArray(arrayVal) || arrayVal === null || arrayVal === undefined) {
+                            if (preserve) {
+                                const newDoc = fieldName.includes('.') ? structuredClone(doc) : { ...doc };
+                                if (Array.isArray(arrayVal))
+                                    this.unsetNestedValue(newDoc, fieldName);
+                                if (indexField)
+                                    newDoc[indexField] = null;
                                 unwound.push(newDoc);
                             }
                         }
-                        else if (arrayVal !== undefined && arrayVal !== null) {
-                            unwound.push(doc);
+                        else {
+                            // Non-array operand is treated as a single-element array
+                            const newDoc = { ...doc };
+                            if (indexField)
+                                newDoc[indexField] = null;
+                            unwound.push(newDoc);
                         }
                     }
                     results = unwound;
@@ -1469,130 +1541,131 @@ export class MongoLiteCollection {
     applyGroupStage(docs, groupSpec) {
         const groups = new Map();
         const idSpec = groupSpec['_id'];
-        for (const doc of docs) {
-            let groupKey;
-            if (idSpec === null) {
-                groupKey = null;
-            }
-            else if (typeof idSpec === 'string' && idSpec.startsWith('$')) {
-                groupKey = this.getNestedValue(doc, idSpec.slice(1));
+        // Validate accumulators up front so an unknown one throws even for empty input.
+        const accumulators = [];
+        for (const [field, expr] of Object.entries(groupSpec)) {
+            if (field === '_id')
+                continue;
+            if (typeof expr !== 'object' || expr === null || Array.isArray(expr)) {
+                throw new Error(`The field '${field}' must be an accumulator object`);
             }
-            else if (typeof idSpec === 'object' && idSpec !== null) {
-                const keyObj = {};
-                for (const [k, expr] of Object.entries(idSpec)) {
-                    if (typeof expr === 'string' && expr.startsWith('$')) {
-                        keyObj[k] = this.getNestedValue(doc, expr.slice(1));
-                    }
-                    else {
-                        keyObj[k] = expr;
-                    }
-                }
-                groupKey = keyObj;
+            const keys = Object.keys(expr);
+            if (keys.length !== 1) {
+                throw new Error(`The field '${field}' must specify exactly one accumulator`);
             }
-            else {
-                groupKey = idSpec;
+            if (!SUPPORTED_ACCUMULATORS.includes(keys[0])) {
+                throw new Error(`Unknown or unsupported $group accumulator operator: ${keys[0]}`);
             }
+            accumulators.push([field, keys[0], expr[keys[0]]]);
+        }
+        const keyValues = new Map();
+        for (const doc of docs) {
+            let groupKey = idSpec === undefined ? null : evaluateExpression(doc, idSpec);
+            if (groupKey === undefined)
+                groupKey = null;
             const keyStr = JSON.stringify(groupKey);
             if (!groups.has(keyStr)) {
                 groups.set(keyStr, []);
+                keyValues.set(keyStr, groupKey);
             }
             groups.get(keyStr).push(doc);
         }
         const results = [];
         for (const [keyStr, groupDocs] of groups) {
-            const groupResult = { _id: JSON.parse(keyStr) };
-            for (const [field, expr] of Object.entries(groupSpec)) {
-                if (field === '_id')
-                    continue;
-                if (typeof expr === 'object' && expr !== null) {
-                    const aggOp = Object.keys(expr)[0];
-                    const aggArg = expr[aggOp];
-                    switch (aggOp) {
-                        case '$sum': {
-                            if (typeof aggArg === 'number') {
-                                groupResult[field] = groupDocs.length * aggArg;
-                            }
-                            else if (typeof aggArg === 'string' && aggArg.startsWith('$')) {
-                                const fieldName = aggArg.slice(1);
-                                groupResult[field] = groupDocs.reduce((sum, doc) => {
-                                    const val = this.getNestedValue(doc, fieldName);
-                                    return sum + (typeof val === 'number' ? val : 0);
-                                }, 0);
-                            }
-                            break;
-                        }
-                        case '$avg': {
-                            if (typeof aggArg === 'string' && aggArg.startsWith('$')) {
-                                const fieldName = aggArg.slice(1);
-                                const nums = groupDocs
-                                    .map((doc) => this.getNestedValue(doc, fieldName))
-                                    .filter((v) => typeof v === 'number');
-                                groupResult[field] =
-                                    nums.length > 0 ? nums.reduce((a, b) => a + b, 0) / nums.length : null;
-                            }
-                            break;
-                        }
-                        case '$min': {
-                            if (typeof aggArg === 'string' && aggArg.startsWith('$')) {
-                                const fieldName = aggArg.slice(1);
-                                const vals = groupDocs
-                                    .map((doc) => this.getNestedValue(doc, fieldName))
-                                    .filter((v) => v !== undefined && v !== null);
-                                groupResult[field] =
-                                    vals.length > 0 ? vals.reduce((a, b) => (a < b ? a : b)) : null;
-                            }
-                            break;
-                        }
-                        case '$max': {
-                            if (typeof aggArg === 'string' && aggArg.startsWith('$')) {
-                                const fieldName = aggArg.slice(1);
-                                const vals = groupDocs
-                                    .map((doc) => this.getNestedValue(doc, fieldName))
-                                    .filter((v) => v !== undefined && v !== null);
-                                groupResult[field] =
-                                    vals.length > 0 ? vals.reduce((a, b) => (a > b ? a : b)) : null;
-                            }
-                            break;
-                        }
-                        case '$push': {
-                            if (typeof aggArg === 'string' && aggArg.startsWith('$')) {
-                                const fieldName = aggArg.slice(1);
-                                groupResult[field] = groupDocs.map((doc) => this.getNestedValue(doc, fieldName));
-                            }
-                            else {
-                                groupResult[field] = groupDocs.map(() => aggArg);
-                            }
-                            break;
-                        }
-                        case '$first': {
-                            if (typeof aggArg === 'string' && aggArg.startsWith('$')) {
-                                const fieldName = aggArg.slice(1);
-                                groupResult[field] =
-                                    groupDocs.length > 0 ? this.getNestedValue(groupDocs[0], fieldName) : null;
-                            }
-                            break;
-                        }
-                        case '$last': {
-                            if (typeof aggArg === 'string' && aggArg.startsWith('$')) {
-                                const fieldName = aggArg.slice(1);
-                                groupResult[field] =
-                                    groupDocs.length > 0
-                                        ? this.getNestedValue(groupDocs[groupDocs.length - 1], fieldName)
-                                        : null;
-                            }
-                            break;
-                        }
-                        case '$count': {
-                            groupResult[field] = groupDocs.length;
-                            break;
+            const groupResult = { _id: keyValues.get(keyStr) };
+            for (const [field, aggOp, aggArg] of accumulators) {
+                const vals = () => groupDocs.map((doc) => evaluateExpression(doc, aggArg));
+                switch (aggOp) {
+                    case '$sum':
+                        groupResult[field] = vals().reduce((sum, v) => sum + (typeof v === 'number' ? v : 0), 0);
+                        break;
+                    case '$avg': {
+                        const nums = vals().filter((v) => typeof v === 'number');
+                        groupResult[field] =
+                            nums.length > 0 ? nums.reduce((a, b) => a + b, 0) / nums.length : null;
+                        break;
+                    }
+                    case '$min': {
+                        const v = vals().filter((x) => x !== undefined && x !== null);
+                        groupResult[field] = v.length > 0 ? v.reduce((a, b) => (a < b ? a : b)) : null;
+                        break;
+                    }
+                    case '$max': {
+                        const v = vals().filter((x) => x !== undefined && x !== null);
+                        groupResult[field] = v.length > 0 ? v.reduce((a, b) => (a > b ? a : b)) : null;
+                        break;
+                    }
+                    case '$push':
+                        groupResult[field] = vals().filter((v) => v !== undefined);
+                        break;
+                    case '$addToSet': {
+                        const seen = new Set();
+                        const out = [];
+                        for (const v of vals()) {
+                            if (v === undefined)
+                                continue;
+                            const k = v instanceof Date ? `d:${v.getTime()}` : JSON.stringify(v);
+                            if (seen.has(k))
+                                continue;
+                            seen.add(k);
+                            out.push(v);
                         }
+                        groupResult[field] = out;
+                        break;
+                    }
+                    case '$first': {
+                        const v = groupDocs.length > 0 ? evaluateExpression(groupDocs[0], aggArg) : null;
+                        groupResult[field] = v === undefined ? null : v;
+                        break;
+                    }
+                    case '$last': {
+                        const v = groupDocs.length > 0
+                            ? evaluateExpression(groupDocs[groupDocs.length - 1], aggArg)
+                            : null;
+                        groupResult[field] = v === undefined ? null : v;
+                        break;
                     }
+                    case '$count':
+                        groupResult[field] = groupDocs.length;
+                        break;
                 }
             }
             results.push(groupResult);
         }
         return results;
     }
+    /**
+     * Applies a $lookup stage (equality match against another collection in the same database).
+     * @private
+     */
+    async applyLookupStage(docs, spec) {
+        const { from, localField, foreignField, as } = spec;
+        if ('pipeline' in spec || 'let' in spec) {
+            throw new Error('$lookup with pipeline/let is not supported; use localField/foreignField');
+        }
+        if (typeof from !== 'string' ||
+            typeof localField !== 'string' ||
+            typeof foreignField !== 'string' ||
+            typeof as !== 'string') {
+            throw new Error('$lookup requires string from, localField, foreignField and as');
+        }
+        const foreign = (await new MongoLiteCollection(this.db, from, this.options)
+            .find({})
+            .toArray());
+        const key = (v) => JSON.stringify(v === undefined ? null : v);
+        return docs.map((doc) => {
+            const local = getPath(doc, localField);
+            const localKeys = new Set((Array.isArray(local) ? (local.length > 0 ? local : [null]) : [local]).map(key));
+            const matches = foreign.filter((f) => {
+                const fv = getPath(f, foreignField);
+                const fvs = Array.isArray(fv) ? [...fv, fv] : [fv];
+                return fvs.some((x) => localKeys.has(key(x)));
+            });
+            const out = { ...doc };
+            this.setNestedValue(out, as, matches);
+            return out;
+        });
+    }
     /**
      * Applies a $project stage to a document.
      * @private
@@ -1606,16 +1679,18 @@ export class MongoLiteCollection {
         // for mode-determination purposes, matching real $project behavior.
         const modeInput = {};
         for (const [key, value] of Object.entries(projection)) {
-            modeInput[key] = typeof value === 'string' && value.startsWith('$') ? 1 : value;
+            modeInput[key] = isComputedProjection(value) ? 1 : value;
         }
         const mode = resolveProjectionMode(modeInput);
-        const base = applyProjectionToDocument(doc, projection);
+        const base = applyProjectionToDocument(doc, modeInput);
         if (mode !== 'include')
             return base;
         const result = { ...base };
         for (const [field, val] of Object.entries(projection)) {
-            if (typeof val === 'string' && val.startsWith('$')) {
-                result[field] = this.getNestedValue(doc, val.slice(1));
+            if (isComputedProjection(val)) {
+                const computed = evaluateExpression(doc, val);
+                if (computed !== undefined)
+                    this.setNestedValue(result, field, computed);
             }
         }
         return result;
@@ -1643,6 +1718,11 @@ export class MongoLiteCollection {
                     return false;
                 continue;
             }
+            if (key === '$expr') {
+                if (!evaluateExprFilter(doc, value))
+                    return false;
+                continue;
+            }
             if (key.startsWith('$'))
                 continue;
             const docValue = this.getNestedValue(doc, key);