@push.rocks/smartmongo 2.0.14 → 2.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartmongo",
-  "version": "2.0.14",
+  "version": "2.1.0",
   "private": false,
   "description": "A module for creating and managing a local MongoDB instance for testing purposes.",
   "main": "dist_ts/index.js",
@@ -8,19 +8,28 @@
   "type": "module",
   "author": "Lossless GmbH",
   "license": "MIT",
+  "scripts": {
+    "test": "(tstest test/)",
+    "build": "(tsbuild --web)",
+    "buildDocs": "tsdoc"
+  },
   "devDependencies": {
-    "@git.zone/tsbuild": "^2.1.66",
-    "@git.zone/tsbundle": "^2.0.8",
-    "@git.zone/tsrun": "^1.2.44",
-    "@git.zone/tstest": "^1.0.77",
-    "@push.rocks/tapbundle": "^5.0.12",
-    "@types/node": "^22.14.0"
+    "@git.zone/tsbuild": "^4.1.2",
+    "@git.zone/tsbundle": "^2.8.3",
+    "@git.zone/tsrun": "^2.0.1",
+    "@git.zone/tstest": "^3.1.8",
+    "@types/node": "^25.1.0",
+    "mongodb": "^7.0.0"
   },
   "dependencies": {
     "@push.rocks/mongodump": "^1.0.7",
     "@push.rocks/smartdata": "^5.0.23",
+    "@push.rocks/smartfs": "^1.3.1",
     "@push.rocks/smartpath": "^5.0.11",
     "@push.rocks/smartpromise": "^4.0.3",
+    "@push.rocks/smartrx": "^3.0.0",
+    "bson": "^6.10.0",
+    "mingo": "^7.2.0",
     "mongodb-memory-server": "^10.1.4"
   },
   "browserslist": [
@@ -52,12 +61,11 @@
     "type": "git",
     "url": "https://code.foss.global/push.rocks/smartmongo.git"
   },
+  "packageManager": "pnpm@10.7.0+sha512.6b865ad4b62a1d9842b61d674a393903b871d9244954f652b8842c2b553c72176b278f64c463e52d40fff8aba385c235c8c9ecf5cc7de4fd78b8bb6d49633ab6",
   "bugs": {
     "url": "https://code.foss.global/push.rocks/smartmongo/issues"
   },
-  "scripts": {
-    "test": "(tstest test/ --web)",
-    "build": "(tsbuild --web --allowimplicitany)",
-    "buildDocs": "tsdoc"
+  "pnpm": {
+    "overrides": {}
   }
-}
+}

package/readme.hints.md

@@ -8,3 +8,82 @@
 - This works in both Node.js and Deno environments
 - **Why:** Deno wraps CommonJS exports in a `default` property, so default imports are required
 - Fixed in version 2.0.13 (changed from `import * as mongoPlugin`)
+
+## CongoDB - MongoDB Wire Protocol Server
+
+### Architecture
+CongoDB implements the MongoDB binary wire protocol (OP_MSG, OP_QUERY) allowing official MongoDB drivers to connect directly.
+
+```
+Official MongoClient  →  TCP (wire protocol)  →  CongoServer  →  Engines  →  Storage
+     (mongodb npm)           OP_MSG/BSON            (port)
+```
+
+### Module Structure
+```
+ts/congodb/
+├── server/                   # Wire protocol server
+│   ├── CongoServer.ts        # TCP server, connection handling
+│   ├── WireProtocol.ts       # OP_MSG/OP_QUERY parsing & encoding
+│   ├── CommandRouter.ts      # Route commands to handlers
+│   └── handlers/             # Command implementations
+│       ├── HelloHandler.ts   # hello/isMaster handshake
+│       ├── FindHandler.ts    # find, getMore, killCursors, count, distinct
+│       ├── InsertHandler.ts  # insert
+│       ├── UpdateHandler.ts  # update, findAndModify
+│       ├── DeleteHandler.ts  # delete
+│       ├── AggregateHandler.ts # aggregate
+│       ├── IndexHandler.ts   # createIndexes, dropIndexes, listIndexes
+│       └── AdminHandler.ts   # ping, listDatabases, listCollections, etc.
+│
+├── engine/                   # Core logic (reused)
+│   ├── QueryEngine.ts        # Query filtering with mingo
+│   ├── UpdateEngine.ts       # Update operations
+│   ├── AggregationEngine.ts  # Aggregation pipelines
+│   ├── IndexEngine.ts        # Index management
+│   └── TransactionEngine.ts  # Transaction support
+│
+├── storage/                  # Storage layer
+│   ├── IStorageAdapter.ts    # Interface
+│   ├── MemoryStorageAdapter.ts
+│   └── FileStorageAdapter.ts
+│
+└── types/interfaces.ts       # Type definitions
+```
+
+### Usage Example
+```typescript
+import { CongoServer } from '@push.rocks/smartmongo/congodb';
+import { MongoClient } from 'mongodb';
+
+// Start server
+const server = new CongoServer({ port: 27117 });
+await server.start();
+
+// Connect with official MongoDB driver
+const client = new MongoClient('mongodb://127.0.0.1:27117', {
+  directConnection: true
+});
+await client.connect();
+
+// Use like any MongoDB instance
+const db = client.db('mydb');
+await db.collection('users').insertOne({ name: 'John' });
+const user = await db.collection('users').findOne({ name: 'John' });
+
+// Cleanup
+await client.close();
+await server.stop();
+```
+
+### Supported Commands
+- **Handshake**: hello, isMaster
+- **CRUD**: find, insert, update, delete, findAndModify, getMore, killCursors
+- **Aggregation**: aggregate, count, distinct
+- **Indexes**: createIndexes, dropIndexes, listIndexes
+- **Admin**: ping, listDatabases, listCollections, drop, dropDatabase, create, serverStatus, buildInfo
+
+### Notes
+- The old CongoClient/CongoDb/CongoCollection classes have been removed
+- Use the official `mongodb` npm package's MongoClient instead
+- Server supports MongoDB wire protocol versions 0-21 (MongoDB 3.6 through 7.0 compatible)

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartmongo',
-  version: '2.0.14',
+  version: '2.1.0',
   description: 'A module for creating and managing a local MongoDB instance for testing purposes.'
 }

package/ts/congodb/congodb.plugins.ts

@@ -0,0 +1,17 @@
+// @push.rocks scope
+import * as smartfs from '@push.rocks/smartfs';
+import * as smartpath from '@push.rocks/smartpath';
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartrx from '@push.rocks/smartrx';
+
+export { smartfs, smartpath, smartpromise, smartrx };
+
+// thirdparty
+import * as bson from 'bson';
+import * as mingo from 'mingo';
+
+export { bson, mingo };
+
+// Re-export commonly used mingo classes
+export { Query } from 'mingo';
+export { Aggregator } from 'mingo';

package/ts/congodb/engine/AggregationEngine.ts

@@ -0,0 +1,283 @@
+import * as plugins from '../congodb.plugins.js';
+import type { Document, IStoredDocument, IAggregateOptions } from '../types/interfaces.js';
+
+// Import mingo Aggregator
+import { Aggregator } from 'mingo';
+
+/**
+ * Aggregation engine using mingo for MongoDB-compatible aggregation pipeline execution
+ */
+export class AggregationEngine {
+  /**
+   * Execute an aggregation pipeline on a collection of documents
+   */
+  static aggregate(
+    documents: IStoredDocument[],
+    pipeline: Document[],
+    options?: IAggregateOptions
+  ): Document[] {
+    if (!pipeline || pipeline.length === 0) {
+      return documents;
+    }
+
+    // Create mingo aggregator with the pipeline
+    const aggregator = new Aggregator(pipeline, {
+      collation: options?.collation as any,
+    });
+
+    // Run the aggregation
+    const result = aggregator.run(documents);
+
+    return Array.isArray(result) ? result : [];
+  }
+
+  /**
+   * Execute aggregation and return an iterator for lazy evaluation
+   */
+  static *aggregateIterator(
+    documents: IStoredDocument[],
+    pipeline: Document[],
+    options?: IAggregateOptions
+  ): Generator<Document> {
+    const aggregator = new Aggregator(pipeline, {
+      collation: options?.collation as any,
+    });
+
+    // Get the cursor from mingo
+    const cursor = aggregator.stream(documents);
+
+    for (const doc of cursor) {
+      yield doc;
+    }
+  }
+
+  /**
+   * Execute a $lookup stage manually (for cross-collection lookups)
+   * This is used when the lookup references another collection in the same database
+   */
+  static executeLookup(
+    documents: IStoredDocument[],
+    lookupSpec: {
+      from: string;
+      localField: string;
+      foreignField: string;
+      as: string;
+    },
+    foreignCollection: IStoredDocument[]
+  ): Document[] {
+    const { localField, foreignField, as } = lookupSpec;
+
+    return documents.map(doc => {
+      const localValue = this.getNestedValue(doc, localField);
+      const matches = foreignCollection.filter(foreignDoc => {
+        const foreignValue = this.getNestedValue(foreignDoc, foreignField);
+        return this.valuesMatch(localValue, foreignValue);
+      });
+
+      return {
+        ...doc,
+        [as]: matches,
+      };
+    });
+  }
+
+  /**
+   * Execute a $graphLookup stage manually
+   */
+  static executeGraphLookup(
+    documents: IStoredDocument[],
+    graphLookupSpec: {
+      from: string;
+      startWith: string | Document;
+      connectFromField: string;
+      connectToField: string;
+      as: string;
+      maxDepth?: number;
+      depthField?: string;
+      restrictSearchWithMatch?: Document;
+    },
+    foreignCollection: IStoredDocument[]
+  ): Document[] {
+    const {
+      startWith,
+      connectFromField,
+      connectToField,
+      as,
+      maxDepth = 10,
+      depthField,
+      restrictSearchWithMatch,
+    } = graphLookupSpec;
+
+    return documents.map(doc => {
+      const startValue = typeof startWith === 'string' && startWith.startsWith('$')
+        ? this.getNestedValue(doc, startWith.slice(1))
+        : startWith;
+
+      const results: Document[] = [];
+      const visited = new Set<string>();
+      const queue: Array<{ value: any; depth: number }> = [];
+
+      // Initialize with start value(s)
+      const startValues = Array.isArray(startValue) ? startValue : [startValue];
+      for (const val of startValues) {
+        queue.push({ value: val, depth: 0 });
+      }
+
+      while (queue.length > 0) {
+        const { value, depth } = queue.shift()!;
+        if (depth > maxDepth) continue;
+
+        const valueKey = JSON.stringify(value);
+        if (visited.has(valueKey)) continue;
+        visited.add(valueKey);
+
+        // Find matching documents
+        for (const foreignDoc of foreignCollection) {
+          const foreignValue = this.getNestedValue(foreignDoc, connectToField);
+
+          if (this.valuesMatch(value, foreignValue)) {
+            // Check restrictSearchWithMatch
+            if (restrictSearchWithMatch) {
+              const matchQuery = new plugins.mingo.Query(restrictSearchWithMatch);
+              if (!matchQuery.test(foreignDoc)) continue;
+            }
+
+            const resultDoc = depthField
+              ? { ...foreignDoc, [depthField]: depth }
+              : { ...foreignDoc };
+
+            // Avoid duplicates in results
+            const docKey = foreignDoc._id.toHexString();
+            if (!results.some(r => r._id?.toHexString?.() === docKey)) {
+              results.push(resultDoc);
+
+              // Add connected values to queue
+              const nextValue = this.getNestedValue(foreignDoc, connectFromField);
+              if (nextValue !== undefined) {
+                const nextValues = Array.isArray(nextValue) ? nextValue : [nextValue];
+                for (const nv of nextValues) {
+                  queue.push({ value: nv, depth: depth + 1 });
+                }
+              }
+            }
+          }
+        }
+      }
+
+      return {
+        ...doc,
+        [as]: results,
+      };
+    });
+  }
+
+  /**
+   * Execute a $facet stage manually
+   */
+  static executeFacet(
+    documents: IStoredDocument[],
+    facetSpec: Record<string, Document[]>
+  ): Document {
+    const result: Document = {};
+
+    for (const [facetName, pipeline] of Object.entries(facetSpec)) {
+      result[facetName] = this.aggregate(documents, pipeline);
+    }
+
+    return result;
+  }
+
+  /**
+   * Execute a $unionWith stage
+   */
+  static executeUnionWith(
+    documents: IStoredDocument[],
+    otherDocuments: IStoredDocument[],
+    pipeline?: Document[]
+  ): Document[] {
+    let unionDocs: Document[] = otherDocuments;
+    if (pipeline && pipeline.length > 0) {
+      unionDocs = this.aggregate(otherDocuments, pipeline);
+    }
+    return [...documents, ...unionDocs];
+  }
+
+  /**
+   * Execute a $merge stage (output to another collection)
+   * Returns the documents that would be inserted/updated
+   */
+  static prepareMerge(
+    documents: Document[],
+    mergeSpec: {
+      into: string;
+      on?: string | string[];
+      whenMatched?: 'replace' | 'keepExisting' | 'merge' | 'fail' | Document[];
+      whenNotMatched?: 'insert' | 'discard' | 'fail';
+    }
+  ): {
+    toInsert: Document[];
+    toUpdate: Array<{ filter: Document; update: Document }>;
+    onField: string | string[];
+    whenMatched: string | Document[];
+    whenNotMatched: string;
+  } {
+    const onField = mergeSpec.on || '_id';
+    const whenMatched = mergeSpec.whenMatched || 'merge';
+    const whenNotMatched = mergeSpec.whenNotMatched || 'insert';
+
+    return {
+      toInsert: [],
+      toUpdate: [],
+      onField,
+      whenMatched,
+      whenNotMatched,
+    };
+  }
+
+  // ============================================================================
+  // Helper Methods
+  // ============================================================================
+
+  private static getNestedValue(obj: any, path: string): any {
+    const parts = path.split('.');
+    let current = obj;
+
+    for (const part of parts) {
+      if (current === null || current === undefined) {
+        return undefined;
+      }
+      current = current[part];
+    }
+
+    return current;
+  }
+
+  private static valuesMatch(a: any, b: any): boolean {
+    if (a === b) return true;
+
+    // Handle ObjectId comparison
+    if (a instanceof plugins.bson.ObjectId && b instanceof plugins.bson.ObjectId) {
+      return a.equals(b);
+    }
+
+    // Handle array contains check
+    if (Array.isArray(a)) {
+      return a.some(item => this.valuesMatch(item, b));
+    }
+    if (Array.isArray(b)) {
+      return b.some(item => this.valuesMatch(a, item));
+    }
+
+    // Handle Date comparison
+    if (a instanceof Date && b instanceof Date) {
+      return a.getTime() === b.getTime();
+    }
+
+    // Handle object comparison
+    if (typeof a === 'object' && typeof b === 'object' && a !== null && b !== null) {
+      return JSON.stringify(a) === JSON.stringify(b);
+    }
+
+    return false;
+  }
+}