@push.rocks/smartdb 1.0.1

package/ts/ts_smartdb/index.ts

@@ -0,0 +1,46 @@
+// SmartDB - MongoDB Wire Protocol compatible in-memory database server
+// Use the official MongoDB driver to connect to SmartdbServer
+
+// Re-export plugins for external use
+import * as plugins from './plugins.js';
+export { plugins };
+
+// Export BSON types for convenience
+export { ObjectId, Binary, Timestamp, Long, Decimal128, UUID } from 'bson';
+
+// Export all types
+export * from './types/interfaces.js';
+
+// Export errors
+export * from './errors/SmartdbErrors.js';
+
+// Export storage adapters
+export type { IStorageAdapter } from './storage/IStorageAdapter.js';
+export { MemoryStorageAdapter } from './storage/MemoryStorageAdapter.js';
+export { FileStorageAdapter } from './storage/FileStorageAdapter.js';
+export { OpLog } from './storage/OpLog.js';
+export { WAL } from './storage/WAL.js';
+export type { IWalEntry, TWalOperation } from './storage/WAL.js';
+
+// Export engines
+export { QueryEngine } from './engine/QueryEngine.js';
+export { UpdateEngine } from './engine/UpdateEngine.js';
+export { AggregationEngine } from './engine/AggregationEngine.js';
+export { IndexEngine } from './engine/IndexEngine.js';
+export { TransactionEngine } from './engine/TransactionEngine.js';
+export { QueryPlanner } from './engine/QueryPlanner.js';
+export type { IQueryPlan, TQueryPlanType } from './engine/QueryPlanner.js';
+export { SessionEngine } from './engine/SessionEngine.js';
+export type { ISession, ISessionEngineOptions } from './engine/SessionEngine.js';
+
+// Export server (the main entry point for using SmartDB)
+export { SmartdbServer } from './server/SmartdbServer.js';
+export type { ISmartdbServerOptions } from './server/SmartdbServer.js';
+
+// Export wire protocol utilities (for advanced usage)
+export { WireProtocol } from './server/WireProtocol.js';
+export { CommandRouter } from './server/CommandRouter.js';
+export type { ICommandHandler, IHandlerContext, ICursorState } from './server/CommandRouter.js';
+
+// Export utilities
+export * from './utils/checksum.js';

package/ts/ts_smartdb/plugins.ts

@@ -0,0 +1,17 @@
+// @push.rocks scope
+import * as smartfs from '@push.rocks/smartfs';
+import * as smartpath from '@push.rocks/smartpath';
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartrx from '@push.rocks/smartrx';
+
+export { smartfs, smartpath, smartpromise, smartrx };
+
+// thirdparty
+import * as bson from 'bson';
+import * as mingo from 'mingo';
+
+export { bson, mingo };
+
+// Re-export commonly used mingo classes
+export { Query } from 'mingo';
+export { Aggregator } from 'mingo';

package/ts/ts_smartdb/server/CommandRouter.ts

@@ -0,0 +1,289 @@
+import * as plugins from '../plugins.js';
+import type { IStorageAdapter } from '../storage/IStorageAdapter.js';
+import type { IParsedCommand } from './WireProtocol.js';
+import type { SmartdbServer } from './SmartdbServer.js';
+import { IndexEngine } from '../engine/IndexEngine.js';
+import { TransactionEngine } from '../engine/TransactionEngine.js';
+import { SessionEngine } from '../engine/SessionEngine.js';
+
+// Import handlers
+import { HelloHandler } from './handlers/HelloHandler.js';
+import { InsertHandler } from './handlers/InsertHandler.js';
+import { FindHandler } from './handlers/FindHandler.js';
+import { UpdateHandler } from './handlers/UpdateHandler.js';
+import { DeleteHandler } from './handlers/DeleteHandler.js';
+import { AggregateHandler } from './handlers/AggregateHandler.js';
+import { IndexHandler } from './handlers/IndexHandler.js';
+import { AdminHandler } from './handlers/AdminHandler.js';
+
+/**
+ * Handler context passed to command handlers
+ */
+export interface IHandlerContext {
+  storage: IStorageAdapter;
+  server: SmartdbServer;
+  database: string;
+  command: plugins.bson.Document;
+  documentSequences?: Map<string, plugins.bson.Document[]>;
+  /** Get or create an IndexEngine for a collection */
+  getIndexEngine: (collName: string) => IndexEngine;
+  /** Transaction engine instance */
+  transactionEngine: TransactionEngine;
+  /** Current transaction ID (if in a transaction) */
+  txnId?: string;
+  /** Session ID (from lsid) */
+  sessionId?: string;
+  /** Session engine instance */
+  sessionEngine: SessionEngine;
+}
+
+/**
+ * Command handler interface
+ */
+export interface ICommandHandler {
+  handle(context: IHandlerContext): Promise<plugins.bson.Document>;
+}
+
+/**
+ * CommandRouter - Routes incoming commands to appropriate handlers
+ */
+export class CommandRouter {
+  private storage: IStorageAdapter;
+  private server: SmartdbServer;
+  private handlers: Map<string, ICommandHandler> = new Map();
+
+  // Cursor state for getMore operations
+  private cursors: Map<bigint, ICursorState> = new Map();
+  private cursorIdCounter: bigint = BigInt(1);
+
+  // Index engine cache: db.collection -> IndexEngine
+  private indexEngines: Map<string, IndexEngine> = new Map();
+
+  // Transaction engine (shared across all handlers)
+  private transactionEngine: TransactionEngine;
+
+  // Session engine (shared across all handlers)
+  private sessionEngine: SessionEngine;
+
+  constructor(storage: IStorageAdapter, server: SmartdbServer) {
+    this.storage = storage;
+    this.server = server;
+    this.transactionEngine = new TransactionEngine(storage);
+    this.sessionEngine = new SessionEngine();
+    // Link session engine to transaction engine for auto-abort on session expiry
+    this.sessionEngine.setTransactionEngine(this.transactionEngine);
+    this.registerHandlers();
+  }
+
+  /**
+   * Get or create an IndexEngine for a database.collection
+   */
+  getIndexEngine(dbName: string, collName: string): IndexEngine {
+    const key = `${dbName}.${collName}`;
+    let engine = this.indexEngines.get(key);
+    if (!engine) {
+      engine = new IndexEngine(dbName, collName, this.storage);
+      this.indexEngines.set(key, engine);
+    }
+    return engine;
+  }
+
+  /**
+   * Clear index engine cache for a collection (used when collection is dropped)
+   */
+  clearIndexEngineCache(dbName: string, collName?: string): void {
+    if (collName) {
+      this.indexEngines.delete(`${dbName}.${collName}`);
+    } else {
+      // Clear all engines for the database
+      for (const key of this.indexEngines.keys()) {
+        if (key.startsWith(`${dbName}.`)) {
+          this.indexEngines.delete(key);
+        }
+      }
+    }
+  }
+
+  /**
+   * Register all command handlers
+   */
+  private registerHandlers(): void {
+    // Create handler instances with shared state
+    const helloHandler = new HelloHandler();
+    const findHandler = new FindHandler(this.cursors, () => this.cursorIdCounter++);
+    const insertHandler = new InsertHandler();
+    const updateHandler = new UpdateHandler();
+    const deleteHandler = new DeleteHandler();
+    const aggregateHandler = new AggregateHandler(this.cursors, () => this.cursorIdCounter++);
+    const indexHandler = new IndexHandler();
+    const adminHandler = new AdminHandler();
+
+    // Handshake commands
+    this.handlers.set('hello', helloHandler);
+    this.handlers.set('ismaster', helloHandler);
+    this.handlers.set('isMaster', helloHandler);
+
+    // CRUD commands
+    this.handlers.set('find', findHandler);
+    this.handlers.set('insert', insertHandler);
+    this.handlers.set('update', updateHandler);
+    this.handlers.set('delete', deleteHandler);
+    this.handlers.set('findAndModify', updateHandler);
+    this.handlers.set('getMore', findHandler);
+    this.handlers.set('killCursors', findHandler);
+
+    // Aggregation
+    this.handlers.set('aggregate', aggregateHandler);
+    this.handlers.set('count', findHandler);
+    this.handlers.set('distinct', findHandler);
+
+    // Index operations
+    this.handlers.set('createIndexes', indexHandler);
+    this.handlers.set('dropIndexes', indexHandler);
+    this.handlers.set('listIndexes', indexHandler);
+
+    // Admin/Database operations
+    this.handlers.set('ping', adminHandler);
+    this.handlers.set('listDatabases', adminHandler);
+    this.handlers.set('listCollections', adminHandler);
+    this.handlers.set('drop', adminHandler);
+    this.handlers.set('dropDatabase', adminHandler);
+    this.handlers.set('create', adminHandler);
+    this.handlers.set('serverStatus', adminHandler);
+    this.handlers.set('buildInfo', adminHandler);
+    this.handlers.set('whatsmyuri', adminHandler);
+    this.handlers.set('getLog', adminHandler);
+    this.handlers.set('hostInfo', adminHandler);
+    this.handlers.set('replSetGetStatus', adminHandler);
+    this.handlers.set('isMaster', helloHandler);
+    this.handlers.set('saslStart', adminHandler);
+    this.handlers.set('saslContinue', adminHandler);
+    this.handlers.set('endSessions', adminHandler);
+    this.handlers.set('abortTransaction', adminHandler);
+    this.handlers.set('commitTransaction', adminHandler);
+    this.handlers.set('collStats', adminHandler);
+    this.handlers.set('dbStats', adminHandler);
+    this.handlers.set('connectionStatus', adminHandler);
+    this.handlers.set('currentOp', adminHandler);
+    this.handlers.set('collMod', adminHandler);
+    this.handlers.set('renameCollection', adminHandler);
+  }
+
+  /**
+   * Route a command to its handler
+   */
+  async route(parsedCommand: IParsedCommand): Promise<plugins.bson.Document> {
+    const { commandName, command, database, documentSequences } = parsedCommand;
+
+    // Extract session ID from lsid using SessionEngine helper
+    let sessionId = SessionEngine.extractSessionId(command.lsid);
+    let txnId: string | undefined;
+
+    // If we have a session ID, register/touch the session
+    if (sessionId) {
+      this.sessionEngine.getOrCreateSession(sessionId);
+    }
+
+    // Check if this starts a new transaction
+    if (command.startTransaction && sessionId) {
+      txnId = this.transactionEngine.startTransaction(sessionId);
+      this.sessionEngine.startTransaction(sessionId, txnId, command.txnNumber);
+    } else if (sessionId && this.sessionEngine.isInTransaction(sessionId)) {
+      // Continue existing transaction
+      txnId = this.sessionEngine.getTransactionId(sessionId);
+      // Verify transaction is still active
+      if (txnId && !this.transactionEngine.isActive(txnId)) {
+        this.sessionEngine.endTransaction(sessionId);
+        txnId = undefined;
+      }
+    }
+
+    // Create handler context
+    const context: IHandlerContext = {
+      storage: this.storage,
+      server: this.server,
+      database,
+      command,
+      documentSequences,
+      getIndexEngine: (collName: string) => this.getIndexEngine(database, collName),
+      transactionEngine: this.transactionEngine,
+      sessionEngine: this.sessionEngine,
+      txnId,
+      sessionId,
+    };
+
+    // Find handler
+    const handler = this.handlers.get(commandName);
+
+    if (!handler) {
+      // Unknown command
+      return {
+        ok: 0,
+        errmsg: `no such command: '${commandName}'`,
+        code: 59,
+        codeName: 'CommandNotFound',
+      };
+    }
+
+    try {
+      return await handler.handle(context);
+    } catch (error: any) {
+      // Handle known error types
+      if (error.code) {
+        return {
+          ok: 0,
+          errmsg: error.message,
+          code: error.code,
+          codeName: error.codeName || 'UnknownError',
+        };
+      }
+
+      // Generic error
+      return {
+        ok: 0,
+        errmsg: error.message || 'Internal error',
+        code: 1,
+        codeName: 'InternalError',
+      };
+    }
+  }
+
+  /**
+   * Close the command router and cleanup resources
+   */
+  close(): void {
+    // Close session engine (stops cleanup interval, clears sessions)
+    this.sessionEngine.close();
+    // Clear cursors
+    this.cursors.clear();
+    // Clear index engine cache
+    this.indexEngines.clear();
+  }
+
+  /**
+   * Get session engine (for administrative purposes)
+   */
+  getSessionEngine(): SessionEngine {
+    return this.sessionEngine;
+  }
+
+  /**
+   * Get transaction engine (for administrative purposes)
+   */
+  getTransactionEngine(): TransactionEngine {
+    return this.transactionEngine;
+  }
+}
+
+/**
+ * Cursor state for multi-batch queries
+ */
+export interface ICursorState {
+  id: bigint;
+  database: string;
+  collection: string;
+  documents: plugins.bson.Document[];
+  position: number;
+  batchSize: number;
+  createdAt: Date;
+}

package/ts/ts_smartdb/server/SmartdbServer.ts

@@ -0,0 +1,354 @@
+import * as net from 'net';
+import * as fs from 'fs/promises';
+import * as plugins from '../plugins.js';
+import { WireProtocol, OP_QUERY } from './WireProtocol.js';
+import { CommandRouter } from './CommandRouter.js';
+import { MemoryStorageAdapter } from '../storage/MemoryStorageAdapter.js';
+import { FileStorageAdapter } from '../storage/FileStorageAdapter.js';
+import type { IStorageAdapter } from '../storage/IStorageAdapter.js';
+
+/**
+ * Server configuration options
+ */
+export interface ISmartdbServerOptions {
+  /** Port to listen on (default: 27017) - ignored if socketPath is set */
+  port?: number;
+  /** Host to bind to (default: 127.0.0.1) - ignored if socketPath is set */
+  host?: string;
+  /** Unix socket path - if set, server listens on socket instead of TCP */
+  socketPath?: string;
+  /** Storage type: 'memory' or 'file' (default: 'memory') */
+  storage?: 'memory' | 'file';
+  /** Path for file storage (required if storage is 'file') */
+  storagePath?: string;
+  /** Enable persistence for memory storage */
+  persistPath?: string;
+  /** Persistence interval in ms (default: 60000) */
+  persistIntervalMs?: number;
+}
+
+/**
+ * Connection state for each client
+ */
+interface IConnectionState {
+  id: number;
+  socket: net.Socket;
+  buffer: Buffer;
+  authenticated: boolean;
+  database: string;
+}
+
+/**
+ * SmartdbServer - MongoDB Wire Protocol compatible server
+ *
+ * This server implements the MongoDB wire protocol (OP_MSG) to allow
+ * official MongoDB drivers to connect and perform operations.
+ *
+ * @example
+ * ```typescript
+ * import { SmartdbServer } from '@push.rocks/smartmongo/smartdb';
+ * import { MongoClient } from 'mongodb';
+ *
+ * const server = new SmartdbServer({ port: 27017 });
+ * await server.start();
+ *
+ * const client = new MongoClient('mongodb://127.0.0.1:27017');
+ * await client.connect();
+ * ```
+ */
+export class SmartdbServer {
+  private options: Required<Omit<ISmartdbServerOptions, 'socketPath'>> & { socketPath: string };
+  private server: net.Server | null = null;
+  private storage: IStorageAdapter;
+  private commandRouter: CommandRouter;
+  private connections: Map<number, IConnectionState> = new Map();
+  private connectionIdCounter = 0;
+  private isRunning = false;
+  private startTime: Date = new Date();
+  private useSocket: boolean;
+
+  constructor(options: ISmartdbServerOptions = {}) {
+    this.useSocket = !!options.socketPath;
+    this.options = {
+      port: options.port ?? 27017,
+      host: options.host ?? '127.0.0.1',
+      socketPath: options.socketPath ?? '',
+      storage: options.storage ?? 'memory',
+      storagePath: options.storagePath ?? './data',
+      persistPath: options.persistPath ?? '',
+      persistIntervalMs: options.persistIntervalMs ?? 60000,
+    };
+
+    // Create storage adapter
+    if (this.options.storage === 'file') {
+      this.storage = new FileStorageAdapter(this.options.storagePath);
+    } else {
+      this.storage = new MemoryStorageAdapter({
+        persistPath: this.options.persistPath || undefined,
+        persistIntervalMs: this.options.persistPath ? this.options.persistIntervalMs : undefined,
+      });
+    }
+
+    // Create command router
+    this.commandRouter = new CommandRouter(this.storage, this);
+  }
+
+  /**
+   * Get the storage adapter (for testing/debugging)
+   */
+  getStorage(): IStorageAdapter {
+    return this.storage;
+  }
+
+  /**
+   * Get server uptime in seconds
+   */
+  getUptime(): number {
+    return Math.floor((Date.now() - this.startTime.getTime()) / 1000);
+  }
+
+  /**
+   * Get current connection count
+   */
+  getConnectionCount(): number {
+    return this.connections.size;
+  }
+
+  /**
+   * Start the server
+   */
+  async start(): Promise<void> {
+    if (this.isRunning) {
+      throw new Error('Server is already running');
+    }
+
+    // Initialize storage
+    await this.storage.initialize();
+
+    // Clean up stale socket file if using Unix socket
+    if (this.useSocket && this.options.socketPath) {
+      try {
+        await fs.unlink(this.options.socketPath);
+      } catch (err: any) {
+        // Ignore ENOENT (file doesn't exist)
+        if (err.code !== 'ENOENT') {
+          throw err;
+        }
+      }
+    }
+
+    return new Promise((resolve, reject) => {
+      this.server = net.createServer((socket) => {
+        this.handleConnection(socket);
+      });
+
+      this.server.on('error', (err) => {
+        if (!this.isRunning) {
+          reject(err);
+        } else {
+          console.error('Server error:', err);
+        }
+      });
+
+      if (this.useSocket && this.options.socketPath) {
+        // Listen on Unix socket
+        this.server.listen(this.options.socketPath, () => {
+          this.isRunning = true;
+          this.startTime = new Date();
+          resolve();
+        });
+      } else {
+        // Listen on TCP
+        this.server.listen(this.options.port, this.options.host, () => {
+          this.isRunning = true;
+          this.startTime = new Date();
+          resolve();
+        });
+      }
+    });
+  }
+
+  /**
+   * Stop the server
+   */
+  async stop(): Promise<void> {
+    if (!this.isRunning || !this.server) {
+      return;
+    }
+
+    // Close all connections
+    for (const conn of this.connections.values()) {
+      conn.socket.destroy();
+    }
+    this.connections.clear();
+
+    // Close command router (cleans up session engine, cursors, etc.)
+    this.commandRouter.close();
+
+    // Close storage
+    await this.storage.close();
+
+    return new Promise((resolve) => {
+      this.server!.close(async () => {
+        this.isRunning = false;
+        this.server = null;
+
+        // Clean up socket file if using Unix socket
+        if (this.useSocket && this.options.socketPath) {
+          try {
+            await fs.unlink(this.options.socketPath);
+          } catch (err: any) {
+            // Ignore ENOENT (file doesn't exist)
+            if (err.code !== 'ENOENT') {
+              console.error('Failed to remove socket file:', err);
+            }
+          }
+        }
+
+        resolve();
+      });
+    });
+  }
+
+  /**
+   * Handle a new client connection
+   */
+  private handleConnection(socket: net.Socket): void {
+    const connectionId = ++this.connectionIdCounter;
+
+    const state: IConnectionState = {
+      id: connectionId,
+      socket,
+      buffer: Buffer.alloc(0),
+      authenticated: true, // No auth required for now
+      database: 'test',
+    };
+
+    this.connections.set(connectionId, state);
+
+    socket.on('data', (data) => {
+      this.handleData(state, Buffer.isBuffer(data) ? data : Buffer.from(data));
+    });
+
+    socket.on('close', () => {
+      this.connections.delete(connectionId);
+    });
+
+    socket.on('error', (err) => {
+      // Connection errors are expected when clients disconnect
+      this.connections.delete(connectionId);
+    });
+  }
+
+  /**
+   * Handle incoming data from a client
+   */
+  private handleData(state: IConnectionState, data: Buffer): void {
+    // Append new data to buffer
+    state.buffer = Buffer.concat([state.buffer, data]);
+
+    // Process messages from buffer
+    this.processMessages(state);
+  }
+
+  /**
+   * Process complete messages from the buffer
+   */
+  private async processMessages(state: IConnectionState): Promise<void> {
+    while (state.buffer.length >= 16) {
+      try {
+        const result = WireProtocol.parseMessage(state.buffer);
+
+        if (!result) {
+          // Not enough data for a complete message
+          break;
+        }
+
+        const { command, bytesConsumed } = result;
+
+        // Remove processed bytes from buffer
+        state.buffer = state.buffer.subarray(bytesConsumed);
+
+        // Process the command
+        const response = await this.commandRouter.route(command);
+
+        // Encode and send response
+        let responseBuffer: Buffer;
+        if (command.opCode === OP_QUERY) {
+          // Legacy OP_QUERY gets OP_REPLY response
+          responseBuffer = WireProtocol.encodeOpReplyResponse(
+            command.requestID,
+            [response]
+          );
+        } else {
+          // OP_MSG gets OP_MSG response
+          responseBuffer = WireProtocol.encodeOpMsgResponse(
+            command.requestID,
+            response
+          );
+        }
+
+        if (!state.socket.destroyed) {
+          state.socket.write(responseBuffer);
+        }
+      } catch (error: any) {
+        // Send error response
+        const errorResponse = WireProtocol.encodeErrorResponse(
+          0, // We don't have the requestID at this point
+          1,
+          error.message || 'Internal error'
+        );
+
+        if (!state.socket.destroyed) {
+          state.socket.write(errorResponse);
+        }
+
+        // Clear buffer on parse errors to avoid infinite loops
+        if (error.message?.includes('opCode') || error.message?.includes('section')) {
+          state.buffer = Buffer.alloc(0);
+        }
+        break;
+      }
+    }
+  }
+
+  /**
+   * Get the connection URI for this server
+   */
+  getConnectionUri(): string {
+    if (this.useSocket && this.options.socketPath) {
+      // URL-encode the socket path (replace / with %2F)
+      const encodedPath = encodeURIComponent(this.options.socketPath);
+      return `mongodb://${encodedPath}`;
+    }
+    return `mongodb://${this.options.host}:${this.options.port}`;
+  }
+
+  /**
+   * Get the socket path (if using Unix socket mode)
+   */
+  get socketPath(): string | undefined {
+    return this.useSocket ? this.options.socketPath : undefined;
+  }
+
+  /**
+   * Check if the server is running
+   */
+  get running(): boolean {
+    return this.isRunning;
+  }
+
+  /**
+   * Get the port the server is listening on
+   */
+  get port(): number {
+    return this.options.port;
+  }
+
+  /**
+   * Get the host the server is bound to
+   */
+  get host(): string {
+    return this.options.host;
+  }
+}