@push.rocks/smartmongo 2.0.12 → 2.1.0

package/ts/congodb/storage/OpLog.ts

@@ -0,0 +1,282 @@
+import * as plugins from '../congodb.plugins.js';
+import type { IStorageAdapter } from './IStorageAdapter.js';
+import type { IOpLogEntry, Document, IResumeToken, ChangeStreamOperationType } from '../types/interfaces.js';
+
+/**
+ * Operation Log for tracking all mutations
+ * Used primarily for change stream support
+ */
+export class OpLog {
+  private storage: IStorageAdapter;
+  private counter = 0;
+  private listeners: Array<(entry: IOpLogEntry) => void> = [];
+
+  constructor(storage: IStorageAdapter) {
+    this.storage = storage;
+  }
+
+  /**
+   * Generate a new timestamp for oplog entries
+   */
+  generateTimestamp(): plugins.bson.Timestamp {
+    this.counter++;
+    return new plugins.bson.Timestamp({ t: Math.floor(Date.now() / 1000), i: this.counter });
+  }
+
+  /**
+   * Generate a resume token from a timestamp
+   */
+  generateResumeToken(ts: plugins.bson.Timestamp): IResumeToken {
+    // Create a resume token similar to MongoDB's format
+    // It's a base64-encoded BSON document containing the timestamp
+    const tokenData = {
+      _data: Buffer.from(JSON.stringify({
+        ts: { t: ts.high, i: ts.low },
+        version: 1,
+      })).toString('base64'),
+    };
+    return tokenData;
+  }
+
+  /**
+   * Parse a resume token to get the timestamp
+   */
+  parseResumeToken(token: IResumeToken): plugins.bson.Timestamp {
+    try {
+      const data = JSON.parse(Buffer.from(token._data, 'base64').toString('utf-8'));
+      return new plugins.bson.Timestamp({ t: data.ts.t, i: data.ts.i });
+    } catch {
+      throw new Error('Invalid resume token');
+    }
+  }
+
+  /**
+   * Record an insert operation
+   */
+  async recordInsert(
+    dbName: string,
+    collName: string,
+    document: Document,
+    txnInfo?: { txnNumber?: number; lsid?: { id: plugins.bson.Binary } }
+  ): Promise<IOpLogEntry> {
+    const entry: IOpLogEntry = {
+      ts: this.generateTimestamp(),
+      op: 'i',
+      ns: `${dbName}.${collName}`,
+      o: document,
+      ...txnInfo,
+    };
+
+    await this.storage.appendOpLog(entry);
+    this.notifyListeners(entry);
+    return entry;
+  }
+
+  /**
+   * Record an update operation
+   */
+  async recordUpdate(
+    dbName: string,
+    collName: string,
+    filter: Document,
+    update: Document,
+    txnInfo?: { txnNumber?: number; lsid?: { id: plugins.bson.Binary } }
+  ): Promise<IOpLogEntry> {
+    const entry: IOpLogEntry = {
+      ts: this.generateTimestamp(),
+      op: 'u',
+      ns: `${dbName}.${collName}`,
+      o: update,
+      o2: filter,
+      ...txnInfo,
+    };
+
+    await this.storage.appendOpLog(entry);
+    this.notifyListeners(entry);
+    return entry;
+  }
+
+  /**
+   * Record a delete operation
+   */
+  async recordDelete(
+    dbName: string,
+    collName: string,
+    filter: Document,
+    txnInfo?: { txnNumber?: number; lsid?: { id: plugins.bson.Binary } }
+  ): Promise<IOpLogEntry> {
+    const entry: IOpLogEntry = {
+      ts: this.generateTimestamp(),
+      op: 'd',
+      ns: `${dbName}.${collName}`,
+      o: filter,
+      ...txnInfo,
+    };
+
+    await this.storage.appendOpLog(entry);
+    this.notifyListeners(entry);
+    return entry;
+  }
+
+  /**
+   * Record a command (drop, rename, etc.)
+   */
+  async recordCommand(
+    dbName: string,
+    command: Document
+  ): Promise<IOpLogEntry> {
+    const entry: IOpLogEntry = {
+      ts: this.generateTimestamp(),
+      op: 'c',
+      ns: `${dbName}.$cmd`,
+      o: command,
+    };
+
+    await this.storage.appendOpLog(entry);
+    this.notifyListeners(entry);
+    return entry;
+  }
+
+  /**
+   * Get oplog entries after a timestamp
+   */
+  async getEntriesAfter(ts: plugins.bson.Timestamp, limit?: number): Promise<IOpLogEntry[]> {
+    return this.storage.getOpLogAfter(ts, limit);
+  }
+
+  /**
+   * Get the latest timestamp
+   */
+  async getLatestTimestamp(): Promise<plugins.bson.Timestamp | null> {
+    return this.storage.getLatestOpLogTimestamp();
+  }
+
+  /**
+   * Subscribe to oplog changes (for change streams)
+   */
+  subscribe(listener: (entry: IOpLogEntry) => void): () => void {
+    this.listeners.push(listener);
+    return () => {
+      const idx = this.listeners.indexOf(listener);
+      if (idx >= 0) {
+        this.listeners.splice(idx, 1);
+      }
+    };
+  }
+
+  /**
+   * Notify all listeners of a new entry
+   */
+  private notifyListeners(entry: IOpLogEntry): void {
+    for (const listener of this.listeners) {
+      try {
+        listener(entry);
+      } catch (error) {
+        console.error('Error in oplog listener:', error);
+      }
+    }
+  }
+
+  /**
+   * Convert an oplog entry to a change stream document
+   */
+  opLogEntryToChangeEvent(
+    entry: IOpLogEntry,
+    fullDocument?: Document,
+    fullDocumentBeforeChange?: Document
+  ): {
+    _id: IResumeToken;
+    operationType: ChangeStreamOperationType;
+    fullDocument?: Document;
+    fullDocumentBeforeChange?: Document;
+    ns: { db: string; coll?: string };
+    documentKey?: { _id: plugins.bson.ObjectId };
+    updateDescription?: {
+      updatedFields?: Document;
+      removedFields?: string[];
+    };
+    clusterTime: plugins.bson.Timestamp;
+  } {
+    const [db, coll] = entry.ns.split('.');
+    const resumeToken = this.generateResumeToken(entry.ts);
+
+    const baseEvent = {
+      _id: resumeToken,
+      ns: { db, coll: coll === '$cmd' ? undefined : coll },
+      clusterTime: entry.ts,
+    };
+
+    switch (entry.op) {
+      case 'i':
+        return {
+          ...baseEvent,
+          operationType: 'insert' as ChangeStreamOperationType,
+          fullDocument: fullDocument || entry.o,
+          documentKey: entry.o._id ? { _id: entry.o._id } : undefined,
+        };
+
+      case 'u':
+        const updateEvent: any = {
+          ...baseEvent,
+          operationType: 'update' as ChangeStreamOperationType,
+          documentKey: entry.o2?._id ? { _id: entry.o2._id } : undefined,
+        };
+
+        if (fullDocument) {
+          updateEvent.fullDocument = fullDocument;
+        }
+        if (fullDocumentBeforeChange) {
+          updateEvent.fullDocumentBeforeChange = fullDocumentBeforeChange;
+        }
+
+        // Parse update description
+        if (entry.o.$set || entry.o.$unset) {
+          updateEvent.updateDescription = {
+            updatedFields: entry.o.$set || {},
+            removedFields: entry.o.$unset ? Object.keys(entry.o.$unset) : [],
+          };
+        }
+
+        return updateEvent;
+
+      case 'd':
+        return {
+          ...baseEvent,
+          operationType: 'delete' as ChangeStreamOperationType,
+          documentKey: entry.o._id ? { _id: entry.o._id } : undefined,
+          fullDocumentBeforeChange,
+        };
+
+      case 'c':
+        if (entry.o.drop) {
+          return {
+            ...baseEvent,
+            operationType: 'drop' as ChangeStreamOperationType,
+            ns: { db, coll: entry.o.drop },
+          };
+        }
+        if (entry.o.dropDatabase) {
+          return {
+            ...baseEvent,
+            operationType: 'dropDatabase' as ChangeStreamOperationType,
+          };
+        }
+        if (entry.o.renameCollection) {
+          return {
+            ...baseEvent,
+            operationType: 'rename' as ChangeStreamOperationType,
+          };
+        }
+        return {
+          ...baseEvent,
+          operationType: 'invalidate' as ChangeStreamOperationType,
+        };
+
+      default:
+        return {
+          ...baseEvent,
+          operationType: 'invalidate' as ChangeStreamOperationType,
+        };
+    }
+  }
+}

package/ts/congodb/types/interfaces.ts

@@ -0,0 +1,433 @@
+import type * as plugins from '../congodb.plugins.js';
+
+// ============================================================================
+// Document Types
+// ============================================================================
+
+export type Document = Record<string, any>;
+
+export interface WithId<TSchema> {
+  _id: plugins.bson.ObjectId;
+}
+
+// ============================================================================
+// Client Options
+// ============================================================================
+
+export interface ICongoClientOptions {
+  /** Storage adapter type: 'memory' or 'file' */
+  storageType?: 'memory' | 'file';
+  /** Path for file-based storage */
+  storagePath?: string;
+  /** Enable persistence for memory adapter */
+  persist?: boolean;
+  /** Path for persistence file when using memory adapter */
+  persistPath?: string;
+}
+
+// ============================================================================
+// Connection String Parsing
+// ============================================================================
+
+export interface IParsedConnectionString {
+  protocol: 'congo';
+  storageType: 'memory' | 'file';
+  options: {
+    persist?: string;
+    path?: string;
+  };
+}
+
+// ============================================================================
+// CRUD Operation Options
+// ============================================================================
+
+export interface IInsertOneOptions {
+  /** Session for transaction support */
+  session?: IClientSession;
+  /** Custom write concern */
+  writeConcern?: IWriteConcern;
+}
+
+export interface IInsertManyOptions extends IInsertOneOptions {
+  /** If true, inserts are ordered and stop on first error */
+  ordered?: boolean;
+}
+
+export interface IFindOptions<TSchema = Document> {
+  /** Projection to apply */
+  projection?: Partial<Record<keyof TSchema | string, 0 | 1 | boolean>>;
+  /** Sort specification */
+  sort?: ISortSpecification;
+  /** Number of documents to skip */
+  skip?: number;
+  /** Maximum number of documents to return */
+  limit?: number;
+  /** Session for transaction support */
+  session?: IClientSession;
+  /** Hint for index usage */
+  hint?: string | Document;
+}
+
+export interface IUpdateOptions {
+  /** Create document if it doesn't exist */
+  upsert?: boolean;
+  /** Session for transaction support */
+  session?: IClientSession;
+  /** Array filters for positional updates */
+  arrayFilters?: Document[];
+  /** Custom write concern */
+  writeConcern?: IWriteConcern;
+  /** Hint for index usage */
+  hint?: string | Document;
+}
+
+export interface IReplaceOptions extends IUpdateOptions {}
+
+export interface IDeleteOptions {
+  /** Session for transaction support */
+  session?: IClientSession;
+  /** Custom write concern */
+  writeConcern?: IWriteConcern;
+  /** Hint for index usage */
+  hint?: string | Document;
+}
+
+export interface IFindOneAndUpdateOptions extends IUpdateOptions {
+  /** Return the document before or after the update */
+  returnDocument?: 'before' | 'after';
+  /** Projection to apply */
+  projection?: Document;
+  /** Sort specification to determine which document to modify */
+  sort?: ISortSpecification;
+}
+
+export interface IFindOneAndReplaceOptions extends IFindOneAndUpdateOptions {}
+
+export interface IFindOneAndDeleteOptions {
+  /** Projection to apply */
+  projection?: Document;
+  /** Sort specification to determine which document to delete */
+  sort?: ISortSpecification;
+  /** Session for transaction support */
+  session?: IClientSession;
+}
+
+// ============================================================================
+// CRUD Results
+// ============================================================================
+
+export interface IInsertOneResult {
+  acknowledged: boolean;
+  insertedId: plugins.bson.ObjectId;
+}
+
+export interface IInsertManyResult {
+  acknowledged: boolean;
+  insertedCount: number;
+  insertedIds: Record<number, plugins.bson.ObjectId>;
+}
+
+export interface IUpdateResult {
+  acknowledged: boolean;
+  matchedCount: number;
+  modifiedCount: number;
+  upsertedCount: number;
+  upsertedId: plugins.bson.ObjectId | null;
+}
+
+export interface IDeleteResult {
+  acknowledged: boolean;
+  deletedCount: number;
+}
+
+export interface IModifyResult<TSchema> {
+  value: TSchema | null;
+  ok: 1 | 0;
+  lastErrorObject?: {
+    n: number;
+    updatedExisting?: boolean;
+    upserted?: plugins.bson.ObjectId;
+  };
+}
+
+// ============================================================================
+// Sort and Index Types
+// ============================================================================
+
+export type ISortDirection = 1 | -1 | 'asc' | 'desc' | 'ascending' | 'descending';
+
+export type ISortSpecification = Record<string, ISortDirection> | [string, ISortDirection][];
+
+export interface IIndexSpecification {
+  key: Record<string, 1 | -1 | 'text' | '2dsphere'>;
+  name?: string;
+  unique?: boolean;
+  sparse?: boolean;
+  expireAfterSeconds?: number;
+  background?: boolean;
+  partialFilterExpression?: Document;
+}
+
+export interface IIndexInfo {
+  v: number;
+  key: Record<string, 1 | -1 | string>;
+  name: string;
+  unique?: boolean;
+  sparse?: boolean;
+  expireAfterSeconds?: number;
+}
+
+export interface ICreateIndexOptions {
+  unique?: boolean;
+  sparse?: boolean;
+  expireAfterSeconds?: number;
+  name?: string;
+  background?: boolean;
+  partialFilterExpression?: Document;
+}
+
+// ============================================================================
+// Write Concern
+// ============================================================================
+
+export interface IWriteConcern {
+  w?: number | 'majority';
+  j?: boolean;
+  wtimeout?: number;
+}
+
+// ============================================================================
+// Aggregation Types
+// ============================================================================
+
+export interface IAggregateOptions {
+  /** Allow disk use for large aggregations */
+  allowDiskUse?: boolean;
+  /** Maximum time in ms */
+  maxTimeMS?: number;
+  /** Session for transaction support */
+  session?: IClientSession;
+  /** Batch size for cursor */
+  batchSize?: number;
+  /** Collation settings */
+  collation?: ICollation;
+  /** Hint for index usage */
+  hint?: string | Document;
+  /** Comment for profiling */
+  comment?: string;
+}
+
+export interface ICollation {
+  locale: string;
+  caseLevel?: boolean;
+  caseFirst?: string;
+  strength?: number;
+  numericOrdering?: boolean;
+  alternate?: string;
+  maxVariable?: string;
+  backwards?: boolean;
+}
+
+// ============================================================================
+// Change Stream Types
+// ============================================================================
+
+export interface IChangeStreamOptions {
+  /** Resume after this token */
+  resumeAfter?: IResumeToken;
+  /** Start at this operation time */
+  startAtOperationTime?: plugins.bson.Timestamp;
+  /** Start after this token */
+  startAfter?: IResumeToken;
+  /** Full document lookup mode */
+  fullDocument?: 'default' | 'updateLookup' | 'whenAvailable' | 'required';
+  /** Full document before change */
+  fullDocumentBeforeChange?: 'off' | 'whenAvailable' | 'required';
+  /** Batch size */
+  batchSize?: number;
+  /** Maximum await time in ms */
+  maxAwaitTimeMS?: number;
+}
+
+export interface IResumeToken {
+  _data: string;
+}
+
+export type ChangeStreamOperationType =
+  | 'insert'
+  | 'update'
+  | 'replace'
+  | 'delete'
+  | 'drop'
+  | 'rename'
+  | 'dropDatabase'
+  | 'invalidate';
+
+export interface IChangeStreamDocument<TSchema = Document> {
+  _id: IResumeToken;
+  operationType: ChangeStreamOperationType;
+  fullDocument?: TSchema;
+  fullDocumentBeforeChange?: TSchema;
+  ns: {
+    db: string;
+    coll?: string;
+  };
+  documentKey?: { _id: plugins.bson.ObjectId };
+  updateDescription?: {
+    updatedFields?: Document;
+    removedFields?: string[];
+    truncatedArrays?: Array<{ field: string; newSize: number }>;
+  };
+  clusterTime?: plugins.bson.Timestamp;
+  txnNumber?: number;
+  lsid?: { id: plugins.bson.Binary; uid: plugins.bson.Binary };
+}
+
+// ============================================================================
+// Transaction Types
+// ============================================================================
+
+export interface IClientSession {
+  id: { id: plugins.bson.Binary };
+  inTransaction(): boolean;
+  startTransaction(options?: ITransactionOptions): void;
+  commitTransaction(): Promise<void>;
+  abortTransaction(): Promise<void>;
+  withTransaction<T>(fn: () => Promise<T>, options?: ITransactionOptions): Promise<T>;
+  endSession(): Promise<void>;
+}
+
+export interface ITransactionOptions {
+  readConcern?: IReadConcern;
+  writeConcern?: IWriteConcern;
+  readPreference?: string;
+  maxCommitTimeMS?: number;
+}
+
+export interface IReadConcern {
+  level: 'local' | 'available' | 'majority' | 'linearizable' | 'snapshot';
+}
+
+// ============================================================================
+// Bulk Operation Types
+// ============================================================================
+
+export interface IBulkWriteOptions {
+  ordered?: boolean;
+  session?: IClientSession;
+  writeConcern?: IWriteConcern;
+}
+
+export interface IBulkWriteOperation<TSchema = Document> {
+  insertOne?: { document: TSchema };
+  updateOne?: { filter: Document; update: Document; upsert?: boolean; arrayFilters?: Document[]; hint?: Document | string };
+  updateMany?: { filter: Document; update: Document; upsert?: boolean; arrayFilters?: Document[]; hint?: Document | string };
+  replaceOne?: { filter: Document; replacement: TSchema; upsert?: boolean; hint?: Document | string };
+  deleteOne?: { filter: Document; hint?: Document | string };
+  deleteMany?: { filter: Document; hint?: Document | string };
+}
+
+export interface IBulkWriteResult {
+  acknowledged: boolean;
+  insertedCount: number;
+  matchedCount: number;
+  modifiedCount: number;
+  deletedCount: number;
+  upsertedCount: number;
+  insertedIds: Record<number, plugins.bson.ObjectId>;
+  upsertedIds: Record<number, plugins.bson.ObjectId>;
+}
+
+// ============================================================================
+// Storage Types
+// ============================================================================
+
+export interface IStoredDocument extends Document {
+  _id: plugins.bson.ObjectId;
+}
+
+export interface IOpLogEntry {
+  ts: plugins.bson.Timestamp;
+  op: 'i' | 'u' | 'd' | 'c' | 'n';
+  ns: string;
+  o: Document;
+  o2?: Document;
+  txnNumber?: number;
+  lsid?: { id: plugins.bson.Binary };
+}
+
+// ============================================================================
+// Admin Types
+// ============================================================================
+
+export interface IDatabaseInfo {
+  name: string;
+  sizeOnDisk: number;
+  empty: boolean;
+}
+
+export interface ICollectionInfo {
+  name: string;
+  type: 'collection' | 'view';
+  options: Document;
+  info: {
+    readOnly: boolean;
+    uuid?: plugins.bson.Binary;
+  };
+  idIndex?: IIndexInfo;
+}
+
+export interface IServerStatus {
+  host: string;
+  version: string;
+  process: string;
+  pid: number;
+  uptime: number;
+  uptimeMillis: number;
+  uptimeEstimate: number;
+  localTime: Date;
+  mem: {
+    resident: number;
+    virtual: number;
+  };
+  connections: {
+    current: number;
+    available: number;
+    totalCreated: number;
+  };
+  ok: 1;
+}
+
+export interface ICollectionStats {
+  ns: string;
+  count: number;
+  size: number;
+  avgObjSize: number;
+  storageSize: number;
+  totalIndexSize: number;
+  indexSizes: Record<string, number>;
+  nindexes: number;
+  ok: 1;
+}
+
+// ============================================================================
+// Count Types
+// ============================================================================
+
+export interface ICountDocumentsOptions {
+  skip?: number;
+  limit?: number;
+  session?: IClientSession;
+  hint?: string | Document;
+  maxTimeMS?: number;
+}
+
+export interface IEstimatedDocumentCountOptions {
+  maxTimeMS?: number;
+}
+
+export interface IDistinctOptions {
+  session?: IClientSession;
+  maxTimeMS?: number;
+}

package/ts/index.ts

@@ -1,6 +1,9 @@
 import { commitinfo } from './00_commitinfo_data.js';
 import * as plugins from './smartmongo.plugins.js';
 
+// Export CongoDB module
+export * as congodb from './congodb/index.js';
+
 export class SmartMongo {
   // STATIC
   public static async createAndStart(replCountArg: number = 1) {