@push.rocks/smartdb 1.0.1 → 2.1.1

package/ts/ts_smartdb/storage/OpLog.ts

@@ -1,282 +0,0 @@
-import * as plugins from '../plugins.js';
-import type { IStorageAdapter } from './IStorageAdapter.js';
-import type { IOpLogEntry, Document, IResumeToken, ChangeStreamOperationType } from '../types/interfaces.js';
-
-/**
- * Operation Log for tracking all mutations
- * Used primarily for change stream support
- */
-export class OpLog {
-  private storage: IStorageAdapter;
-  private counter = 0;
-  private listeners: Array<(entry: IOpLogEntry) => void> = [];
-
-  constructor(storage: IStorageAdapter) {
-    this.storage = storage;
-  }
-
-  /**
-   * Generate a new timestamp for oplog entries
-   */
-  generateTimestamp(): plugins.bson.Timestamp {
-    this.counter++;
-    return new plugins.bson.Timestamp({ t: Math.floor(Date.now() / 1000), i: this.counter });
-  }
-
-  /**
-   * Generate a resume token from a timestamp
-   */
-  generateResumeToken(ts: plugins.bson.Timestamp): IResumeToken {
-    // Create a resume token similar to MongoDB's format
-    // It's a base64-encoded BSON document containing the timestamp
-    const tokenData = {
-      _data: Buffer.from(JSON.stringify({
-        ts: { t: ts.high, i: ts.low },
-        version: 1,
-      })).toString('base64'),
-    };
-    return tokenData;
-  }
-
-  /**
-   * Parse a resume token to get the timestamp
-   */
-  parseResumeToken(token: IResumeToken): plugins.bson.Timestamp {
-    try {
-      const data = JSON.parse(Buffer.from(token._data, 'base64').toString('utf-8'));
-      return new plugins.bson.Timestamp({ t: data.ts.t, i: data.ts.i });
-    } catch {
-      throw new Error('Invalid resume token');
-    }
-  }
-
-  /**
-   * Record an insert operation
-   */
-  async recordInsert(
-    dbName: string,
-    collName: string,
-    document: Document,
-    txnInfo?: { txnNumber?: number; lsid?: { id: plugins.bson.Binary } }
-  ): Promise<IOpLogEntry> {
-    const entry: IOpLogEntry = {
-      ts: this.generateTimestamp(),
-      op: 'i',
-      ns: `${dbName}.${collName}`,
-      o: document,
-      ...txnInfo,
-    };
-
-    await this.storage.appendOpLog(entry);
-    this.notifyListeners(entry);
-    return entry;
-  }
-
-  /**
-   * Record an update operation
-   */
-  async recordUpdate(
-    dbName: string,
-    collName: string,
-    filter: Document,
-    update: Document,
-    txnInfo?: { txnNumber?: number; lsid?: { id: plugins.bson.Binary } }
-  ): Promise<IOpLogEntry> {
-    const entry: IOpLogEntry = {
-      ts: this.generateTimestamp(),
-      op: 'u',
-      ns: `${dbName}.${collName}`,
-      o: update,
-      o2: filter,
-      ...txnInfo,
-    };
-
-    await this.storage.appendOpLog(entry);
-    this.notifyListeners(entry);
-    return entry;
-  }
-
-  /**
-   * Record a delete operation
-   */
-  async recordDelete(
-    dbName: string,
-    collName: string,
-    filter: Document,
-    txnInfo?: { txnNumber?: number; lsid?: { id: plugins.bson.Binary } }
-  ): Promise<IOpLogEntry> {
-    const entry: IOpLogEntry = {
-      ts: this.generateTimestamp(),
-      op: 'd',
-      ns: `${dbName}.${collName}`,
-      o: filter,
-      ...txnInfo,
-    };
-
-    await this.storage.appendOpLog(entry);
-    this.notifyListeners(entry);
-    return entry;
-  }
-
-  /**
-   * Record a command (drop, rename, etc.)
-   */
-  async recordCommand(
-    dbName: string,
-    command: Document
-  ): Promise<IOpLogEntry> {
-    const entry: IOpLogEntry = {
-      ts: this.generateTimestamp(),
-      op: 'c',
-      ns: `${dbName}.$cmd`,
-      o: command,
-    };
-
-    await this.storage.appendOpLog(entry);
-    this.notifyListeners(entry);
-    return entry;
-  }
-
-  /**
-   * Get oplog entries after a timestamp
-   */
-  async getEntriesAfter(ts: plugins.bson.Timestamp, limit?: number): Promise<IOpLogEntry[]> {
-    return this.storage.getOpLogAfter(ts, limit);
-  }
-
-  /**
-   * Get the latest timestamp
-   */
-  async getLatestTimestamp(): Promise<plugins.bson.Timestamp | null> {
-    return this.storage.getLatestOpLogTimestamp();
-  }
-
-  /**
-   * Subscribe to oplog changes (for change streams)
-   */
-  subscribe(listener: (entry: IOpLogEntry) => void): () => void {
-    this.listeners.push(listener);
-    return () => {
-      const idx = this.listeners.indexOf(listener);
-      if (idx >= 0) {
-        this.listeners.splice(idx, 1);
-      }
-    };
-  }
-
-  /**
-   * Notify all listeners of a new entry
-   */
-  private notifyListeners(entry: IOpLogEntry): void {
-    for (const listener of this.listeners) {
-      try {
-        listener(entry);
-      } catch (error) {
-        console.error('Error in oplog listener:', error);
-      }
-    }
-  }
-
-  /**
-   * Convert an oplog entry to a change stream document
-   */
-  opLogEntryToChangeEvent(
-    entry: IOpLogEntry,
-    fullDocument?: Document,
-    fullDocumentBeforeChange?: Document
-  ): {
-    _id: IResumeToken;
-    operationType: ChangeStreamOperationType;
-    fullDocument?: Document;
-    fullDocumentBeforeChange?: Document;
-    ns: { db: string; coll?: string };
-    documentKey?: { _id: plugins.bson.ObjectId };
-    updateDescription?: {
-      updatedFields?: Document;
-      removedFields?: string[];
-    };
-    clusterTime: plugins.bson.Timestamp;
-  } {
-    const [db, coll] = entry.ns.split('.');
-    const resumeToken = this.generateResumeToken(entry.ts);
-
-    const baseEvent = {
-      _id: resumeToken,
-      ns: { db, coll: coll === '$cmd' ? undefined : coll },
-      clusterTime: entry.ts,
-    };
-
-    switch (entry.op) {
-      case 'i':
-        return {
-          ...baseEvent,
-          operationType: 'insert' as ChangeStreamOperationType,
-          fullDocument: fullDocument || entry.o,
-          documentKey: entry.o._id ? { _id: entry.o._id } : undefined,
-        };
-
-      case 'u':
-        const updateEvent: any = {
-          ...baseEvent,
-          operationType: 'update' as ChangeStreamOperationType,
-          documentKey: entry.o2?._id ? { _id: entry.o2._id } : undefined,
-        };
-
-        if (fullDocument) {
-          updateEvent.fullDocument = fullDocument;
-        }
-        if (fullDocumentBeforeChange) {
-          updateEvent.fullDocumentBeforeChange = fullDocumentBeforeChange;
-        }
-
-        // Parse update description
-        if (entry.o.$set || entry.o.$unset) {
-          updateEvent.updateDescription = {
-            updatedFields: entry.o.$set || {},
-            removedFields: entry.o.$unset ? Object.keys(entry.o.$unset) : [],
-          };
-        }
-
-        return updateEvent;
-
-      case 'd':
-        return {
-          ...baseEvent,
-          operationType: 'delete' as ChangeStreamOperationType,
-          documentKey: entry.o._id ? { _id: entry.o._id } : undefined,
-          fullDocumentBeforeChange,
-        };
-
-      case 'c':
-        if (entry.o.drop) {
-          return {
-            ...baseEvent,
-            operationType: 'drop' as ChangeStreamOperationType,
-            ns: { db, coll: entry.o.drop },
-          };
-        }
-        if (entry.o.dropDatabase) {
-          return {
-            ...baseEvent,
-            operationType: 'dropDatabase' as ChangeStreamOperationType,
-          };
-        }
-        if (entry.o.renameCollection) {
-          return {
-            ...baseEvent,
-            operationType: 'rename' as ChangeStreamOperationType,
-          };
-        }
-        return {
-          ...baseEvent,
-          operationType: 'invalidate' as ChangeStreamOperationType,
-        };
-
-      default:
-        return {
-          ...baseEvent,
-          operationType: 'invalidate' as ChangeStreamOperationType,
-        };
-    }
-  }
-}

package/ts/ts_smartdb/storage/WAL.ts

@@ -1,375 +0,0 @@
-import * as plugins from '../plugins.js';
-import type { Document, IStoredDocument } from '../types/interfaces.js';
-
-/**
- * WAL entry operation types
- */
-export type TWalOperation = 'insert' | 'update' | 'delete' | 'checkpoint' | 'begin' | 'commit' | 'abort';
-
-/**
- * WAL entry structure
- */
-export interface IWalEntry {
-  /** Log Sequence Number - monotonically increasing */
-  lsn: number;
-  /** Timestamp of the operation */
-  timestamp: number;
-  /** Operation type */
-  operation: TWalOperation;
-  /** Database name */
-  dbName: string;
-  /** Collection name */
-  collName: string;
-  /** Document ID (hex string) */
-  documentId: string;
-  /** Document data (BSON serialized, base64 encoded) */
-  data?: string;
-  /** Previous document data for updates (for rollback) */
-  previousData?: string;
-  /** Transaction ID if part of a transaction */
-  txnId?: string;
-  /** CRC32 checksum of the entry (excluding this field) */
-  checksum: number;
-}
-
-/**
- * Checkpoint record
- */
-interface ICheckpointRecord {
-  lsn: number;
-  timestamp: number;
-  lastCommittedLsn: number;
-}
-
-/**
- * Write-Ahead Log (WAL) for durability and crash recovery
- *
- * The WAL ensures durability by writing operations to a log file before
- * they are applied to the main storage. On crash recovery, uncommitted
- * operations can be replayed to restore the database to a consistent state.
- */
-export class WAL {
-  private walPath: string;
-  private currentLsn: number = 0;
-  private lastCheckpointLsn: number = 0;
-  private entries: IWalEntry[] = [];
-  private isInitialized: boolean = false;
-  private fs = new plugins.smartfs.SmartFs(new plugins.smartfs.SmartFsProviderNode());
-
-  // In-memory uncommitted entries per transaction
-  private uncommittedTxns: Map<string, IWalEntry[]> = new Map();
-
-  // Checkpoint interval (number of entries between checkpoints)
-  private checkpointInterval: number = 1000;
-
-  constructor(walPath: string, options?: { checkpointInterval?: number }) {
-    this.walPath = walPath;
-    if (options?.checkpointInterval) {
-      this.checkpointInterval = options.checkpointInterval;
-    }
-  }
-
-  /**
-   * Initialize the WAL, loading existing entries and recovering if needed
-   */
-  async initialize(): Promise<{ recoveredEntries: IWalEntry[] }> {
-    if (this.isInitialized) {
-      return { recoveredEntries: [] };
-    }
-
-    // Ensure WAL directory exists
-    const walDir = this.walPath.substring(0, this.walPath.lastIndexOf('/'));
-    if (walDir) {
-      await this.fs.directory(walDir).recursive().create();
-    }
-
-    // Try to load existing WAL
-    const exists = await this.fs.file(this.walPath).exists();
-    if (exists) {
-      const content = await this.fs.file(this.walPath).encoding('utf8').read();
-      const lines = (content as string).split('\n').filter(line => line.trim());
-
-      for (const line of lines) {
-        try {
-          const entry = JSON.parse(line) as IWalEntry;
-          // Verify checksum
-          if (this.verifyChecksum(entry)) {
-            this.entries.push(entry);
-            if (entry.lsn > this.currentLsn) {
-              this.currentLsn = entry.lsn;
-            }
-            if (entry.operation === 'checkpoint') {
-              this.lastCheckpointLsn = entry.lsn;
-            }
-          }
-        } catch {
-          // Skip corrupted entries
-          console.warn('Skipping corrupted WAL entry');
-        }
-      }
-    }
-
-    this.isInitialized = true;
-
-    // Return entries after last checkpoint that need recovery
-    const recoveredEntries = this.entries.filter(
-      e => e.lsn > this.lastCheckpointLsn &&
-           (e.operation === 'insert' || e.operation === 'update' || e.operation === 'delete')
-    );
-
-    return { recoveredEntries };
-  }
-
-  /**
-   * Log an insert operation
-   */
-  async logInsert(dbName: string, collName: string, doc: IStoredDocument, txnId?: string): Promise<number> {
-    return this.appendEntry({
-      operation: 'insert',
-      dbName,
-      collName,
-      documentId: doc._id.toHexString(),
-      data: this.serializeDocument(doc),
-      txnId,
-    });
-  }
-
-  /**
-   * Log an update operation
-   */
-  async logUpdate(
-    dbName: string,
-    collName: string,
-    oldDoc: IStoredDocument,
-    newDoc: IStoredDocument,
-    txnId?: string
-  ): Promise<number> {
-    return this.appendEntry({
-      operation: 'update',
-      dbName,
-      collName,
-      documentId: oldDoc._id.toHexString(),
-      data: this.serializeDocument(newDoc),
-      previousData: this.serializeDocument(oldDoc),
-      txnId,
-    });
-  }
-
-  /**
-   * Log a delete operation
-   */
-  async logDelete(dbName: string, collName: string, doc: IStoredDocument, txnId?: string): Promise<number> {
-    return this.appendEntry({
-      operation: 'delete',
-      dbName,
-      collName,
-      documentId: doc._id.toHexString(),
-      previousData: this.serializeDocument(doc),
-      txnId,
-    });
-  }
-
-  /**
-   * Log transaction begin
-   */
-  async logBeginTransaction(txnId: string): Promise<number> {
-    this.uncommittedTxns.set(txnId, []);
-    return this.appendEntry({
-      operation: 'begin',
-      dbName: '',
-      collName: '',
-      documentId: '',
-      txnId,
-    });
-  }
-
-  /**
-   * Log transaction commit
-   */
-  async logCommitTransaction(txnId: string): Promise<number> {
-    this.uncommittedTxns.delete(txnId);
-    return this.appendEntry({
-      operation: 'commit',
-      dbName: '',
-      collName: '',
-      documentId: '',
-      txnId,
-    });
-  }
-
-  /**
-   * Log transaction abort
-   */
-  async logAbortTransaction(txnId: string): Promise<number> {
-    this.uncommittedTxns.delete(txnId);
-    return this.appendEntry({
-      operation: 'abort',
-      dbName: '',
-      collName: '',
-      documentId: '',
-      txnId,
-    });
-  }
-
-  /**
-   * Get entries to roll back for an aborted transaction
-   */
-  getTransactionEntries(txnId: string): IWalEntry[] {
-    return this.entries.filter(e => e.txnId === txnId);
-  }
-
-  /**
-   * Create a checkpoint - marks a consistent point in the log
-   */
-  async checkpoint(): Promise<number> {
-    const lsn = await this.appendEntry({
-      operation: 'checkpoint',
-      dbName: '',
-      collName: '',
-      documentId: '',
-    });
-    this.lastCheckpointLsn = lsn;
-
-    // Truncate old entries (keep only entries after checkpoint)
-    await this.truncate();
-
-    return lsn;
-  }
-
-  /**
-   * Truncate the WAL file, removing entries before the last checkpoint
-   */
-  private async truncate(): Promise<void> {
-    // Keep entries after last checkpoint
-    const newEntries = this.entries.filter(e => e.lsn >= this.lastCheckpointLsn);
-    this.entries = newEntries;
-
-    // Rewrite the WAL file
-    const lines = this.entries.map(e => JSON.stringify(e)).join('\n');
-    await this.fs.file(this.walPath).encoding('utf8').write(lines);
-  }
-
-  /**
-   * Get current LSN
-   */
-  getCurrentLsn(): number {
-    return this.currentLsn;
-  }
-
-  /**
-   * Get entries after a specific LSN (for recovery)
-   */
-  getEntriesAfter(lsn: number): IWalEntry[] {
-    return this.entries.filter(e => e.lsn > lsn);
-  }
-
-  /**
-   * Close the WAL
-   */
-  async close(): Promise<void> {
-    if (this.isInitialized) {
-      // Final checkpoint before close
-      await this.checkpoint();
-    }
-    this.isInitialized = false;
-  }
-
-  // ============================================================================
-  // Private Methods
-  // ============================================================================
-
-  private async appendEntry(
-    partial: Omit<IWalEntry, 'lsn' | 'timestamp' | 'checksum'>
-  ): Promise<number> {
-    await this.initialize();
-
-    this.currentLsn++;
-    const entry: IWalEntry = {
-      ...partial,
-      lsn: this.currentLsn,
-      timestamp: Date.now(),
-      checksum: 0, // Will be calculated
-    };
-
-    // Calculate checksum
-    entry.checksum = this.calculateChecksum(entry);
-
-    // Track in transaction if applicable
-    if (partial.txnId && this.uncommittedTxns.has(partial.txnId)) {
-      this.uncommittedTxns.get(partial.txnId)!.push(entry);
-    }
-
-    // Add to in-memory log
-    this.entries.push(entry);
-
-    // Append to file (append mode for durability)
-    await this.fs.file(this.walPath).encoding('utf8').append(JSON.stringify(entry) + '\n');
-
-    // Check if we need a checkpoint
-    if (this.entries.length - this.lastCheckpointLsn >= this.checkpointInterval) {
-      await this.checkpoint();
-    }
-
-    return entry.lsn;
-  }
-
-  private serializeDocument(doc: Document): string {
-    // Serialize document to BSON and encode as base64
-    const bsonData = plugins.bson.serialize(doc);
-    return Buffer.from(bsonData).toString('base64');
-  }
-
-  private deserializeDocument(data: string): Document {
-    // Decode base64 and deserialize from BSON
-    const buffer = Buffer.from(data, 'base64');
-    return plugins.bson.deserialize(buffer);
-  }
-
-  private calculateChecksum(entry: IWalEntry): number {
-    // Simple CRC32-like checksum
-    const str = JSON.stringify({
-      lsn: entry.lsn,
-      timestamp: entry.timestamp,
-      operation: entry.operation,
-      dbName: entry.dbName,
-      collName: entry.collName,
-      documentId: entry.documentId,
-      data: entry.data,
-      previousData: entry.previousData,
-      txnId: entry.txnId,
-    });
-
-    let crc = 0xFFFFFFFF;
-    for (let i = 0; i < str.length; i++) {
-      crc ^= str.charCodeAt(i);
-      for (let j = 0; j < 8; j++) {
-        crc = (crc >>> 1) ^ (crc & 1 ? 0xEDB88320 : 0);
-      }
-    }
-    return (~crc) >>> 0;
-  }
-
-  private verifyChecksum(entry: IWalEntry): boolean {
-    const savedChecksum = entry.checksum;
-    entry.checksum = 0;
-    const calculatedChecksum = this.calculateChecksum(entry);
-    entry.checksum = savedChecksum;
-    return calculatedChecksum === savedChecksum;
-  }
-
-  /**
-   * Recover document from WAL entry
-   */
-  recoverDocument(entry: IWalEntry): IStoredDocument | null {
-    if (!entry.data) return null;
-    return this.deserializeDocument(entry.data) as IStoredDocument;
-  }
-
-  /**
-   * Recover previous document state from WAL entry (for rollback)
-   */
-  recoverPreviousDocument(entry: IWalEntry): IStoredDocument | null {
-    if (!entry.previousData) return null;
-    return this.deserializeDocument(entry.previousData) as IStoredDocument;
-  }
-}