@push.rocks/smartdb 2.0.0 → 2.3.1

package/ts/ts_migration/migrators/v0_to_v1.ts

@@ -0,0 +1,253 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+import { BSON } from 'bson';
+
+// ---------------------------------------------------------------------------
+// Binary format constants (must match Rust: record.rs)
+// ---------------------------------------------------------------------------
+
+/** File-level magic: "SMARTDB\0" */
+const FILE_MAGIC = Buffer.from('SMARTDB\0', 'ascii');
+/** Current format version */
+const FORMAT_VERSION = 1;
+/** File type tags */
+const FILE_TYPE_DATA = 1;
+const FILE_TYPE_HINT = 3;
+/** File header total size */
+const FILE_HEADER_SIZE = 64;
+/** Per-record magic */
+const RECORD_MAGIC = 0xDB01;
+/** Per-record header size */
+const RECORD_HEADER_SIZE = 22; // 2 + 8 + 4 + 4 + 4
+
+// ---------------------------------------------------------------------------
+// Binary encoding helpers
+// ---------------------------------------------------------------------------
+
+function writeFileHeader(fileType: number): Buffer {
+  const buf = Buffer.alloc(FILE_HEADER_SIZE, 0);
+  FILE_MAGIC.copy(buf, 0);
+  buf.writeUInt16LE(FORMAT_VERSION, 8);
+  buf.writeUInt8(fileType, 10);
+  buf.writeUInt32LE(0, 11); // flags
+  const now = BigInt(Date.now());
+  buf.writeBigUInt64LE(now, 15);
+  // bytes 23..64 are reserved (zeros)
+  return buf;
+}
+
+function encodeDataRecord(timestamp: bigint, key: Buffer, value: Buffer): Buffer {
+  const keyLen = key.length;
+  const valLen = value.length;
+  const totalSize = RECORD_HEADER_SIZE + keyLen + valLen;
+  const buf = Buffer.alloc(totalSize);
+
+  // Write header fields (without CRC)
+  buf.writeUInt16LE(RECORD_MAGIC, 0);
+  buf.writeBigUInt64LE(timestamp, 2);
+  buf.writeUInt32LE(keyLen, 10);
+  buf.writeUInt32LE(valLen, 14);
+  // CRC placeholder at offset 18..22 (will fill below)
+  key.copy(buf, RECORD_HEADER_SIZE);
+  value.copy(buf, RECORD_HEADER_SIZE + keyLen);
+
+  // CRC32 covers everything except the CRC field itself:
+  // bytes [0..18] + bytes [22..]
+  const crc = crc32(Buffer.concat([
+    buf.subarray(0, 18),
+    buf.subarray(22),
+  ]));
+  buf.writeUInt32LE(crc, 18);
+
+  return buf;
+}
+
+function encodeHintEntry(key: string, offset: bigint, recordLen: number, valueLen: number, timestamp: bigint): Buffer {
+  const keyBuf = Buffer.from(key, 'utf-8');
+  const buf = Buffer.alloc(4 + keyBuf.length + 8 + 4 + 4 + 8);
+  let pos = 0;
+  buf.writeUInt32LE(keyBuf.length, pos); pos += 4;
+  keyBuf.copy(buf, pos); pos += keyBuf.length;
+  buf.writeBigUInt64LE(offset, pos); pos += 8;
+  buf.writeUInt32LE(recordLen, pos); pos += 4;
+  buf.writeUInt32LE(valueLen, pos); pos += 4;
+  buf.writeBigUInt64LE(timestamp, pos);
+  return buf;
+}
+
+// ---------------------------------------------------------------------------
+// CRC32 (matching crc32fast in Rust)
+// ---------------------------------------------------------------------------
+
+const CRC32_TABLE = (() => {
+  const table = new Uint32Array(256);
+  for (let i = 0; i < 256; i++) {
+    let crc = i;
+    for (let j = 0; j < 8; j++) {
+      crc = (crc & 1) ? (0xEDB88320 ^ (crc >>> 1)) : (crc >>> 1);
+    }
+    table[i] = crc;
+  }
+  return table;
+})();
+
+function crc32(data: Buffer): number {
+  let crc = 0xFFFFFFFF;
+  for (let i = 0; i < data.length; i++) {
+    crc = CRC32_TABLE[(crc ^ data[i]) & 0xFF] ^ (crc >>> 8);
+  }
+  return (crc ^ 0xFFFFFFFF) >>> 0;
+}
+
+// ---------------------------------------------------------------------------
+// Migration: v0 (JSON) → v1 (Bitcask binary)
+// ---------------------------------------------------------------------------
+
+interface IKeyDirEntry {
+  offset: bigint;
+  recordLen: number;
+  valueLen: number;
+  timestamp: bigint;
+}
+
+/**
+ * Migrate a storage directory from v0 (JSON-per-collection) to v1 (Bitcask binary).
+ *
+ * - Original .json files are NOT modified or deleted.
+ * - New v1 files are written into {db}/{coll}/ subdirectories.
+ * - Returns a list of old files that can be safely deleted.
+ * - On failure, cleans up any partial new files and throws.
+ */
+export async function migrateV0ToV1(storagePath: string): Promise<string[]> {
+  const deletableFiles: string[] = [];
+  const createdDirs: string[] = [];
+
+  try {
+    const dbEntries = fs.readdirSync(storagePath, { withFileTypes: true });
+
+    for (const dbEntry of dbEntries) {
+      if (!dbEntry.isDirectory()) continue;
+
+      const dbDir = path.join(storagePath, dbEntry.name);
+      const collFiles = fs.readdirSync(dbDir, { withFileTypes: true });
+
+      for (const collFile of collFiles) {
+        if (!collFile.isFile()) continue;
+        if (!collFile.name.endsWith('.json')) continue;
+        if (collFile.name.endsWith('.indexes.json')) continue;
+
+        const collName = collFile.name.replace(/\.json$/, '');
+        const jsonPath = path.join(dbDir, collFile.name);
+        const indexJsonPath = path.join(dbDir, `${collName}.indexes.json`);
+
+        // Target directory
+        const collDir = path.join(dbDir, collName);
+        if (fs.existsSync(collDir)) {
+          // Already migrated
+          continue;
+        }
+
+        console.log(`[smartdb]   Migrating ${dbEntry.name}.${collName}...`);
+
+        // Read the JSON collection
+        const jsonData = fs.readFileSync(jsonPath, 'utf-8');
+        const docs: any[] = JSON.parse(jsonData);
+
+        // Create collection directory
+        fs.mkdirSync(collDir, { recursive: true });
+        createdDirs.push(collDir);
+
+        // Write data.rdb
+        const dataPath = path.join(collDir, 'data.rdb');
+        const fd = fs.openSync(dataPath, 'w');
+
+        try {
+          // File header
+          const headerBuf = writeFileHeader(FILE_TYPE_DATA);
+          fs.writeSync(fd, headerBuf);
+
+          let currentOffset = BigInt(FILE_HEADER_SIZE);
+          const keydir: Map<string, IKeyDirEntry> = new Map();
+          const ts = BigInt(Date.now());
+
+          for (const doc of docs) {
+            // Extract _id
+            let idHex: string;
+            if (doc._id && doc._id.$oid) {
+              idHex = doc._id.$oid;
+            } else if (typeof doc._id === 'string') {
+              idHex = doc._id;
+            } else if (doc._id) {
+              idHex = String(doc._id);
+            } else {
+              // Generate a new ObjectId
+              idHex = crypto.randomBytes(12).toString('hex');
+              doc._id = { $oid: idHex };
+            }
+
+            // Serialize to BSON
+            const bsonBytes = BSON.serialize(doc);
+            const keyBuf = Buffer.from(idHex, 'utf-8');
+            const valueBuf = Buffer.from(bsonBytes);
+
+            const record = encodeDataRecord(ts, keyBuf, valueBuf);
+            fs.writeSync(fd, record);
+
+            keydir.set(idHex, {
+              offset: currentOffset,
+              recordLen: record.length,
+              valueLen: valueBuf.length,
+              timestamp: ts,
+            });
+
+            currentOffset += BigInt(record.length);
+          }
+
+          fs.fsyncSync(fd);
+          fs.closeSync(fd);
+
+          // Write keydir.hint
+          const hintPath = path.join(collDir, 'keydir.hint');
+          const hintFd = fs.openSync(hintPath, 'w');
+          fs.writeSync(hintFd, writeFileHeader(FILE_TYPE_HINT));
+          for (const [key, entry] of keydir) {
+            fs.writeSync(hintFd, encodeHintEntry(key, entry.offset, entry.recordLen, entry.valueLen, entry.timestamp));
+          }
+          fs.fsyncSync(hintFd);
+          fs.closeSync(hintFd);
+
+        } catch (writeErr) {
+          // Clean up on write failure
+          try { fs.closeSync(fd); } catch {}
+          throw writeErr;
+        }
+
+        // Copy indexes.json if it exists
+        if (fs.existsSync(indexJsonPath)) {
+          const destIndexPath = path.join(collDir, 'indexes.json');
+          fs.copyFileSync(indexJsonPath, destIndexPath);
+          deletableFiles.push(indexJsonPath);
+        } else {
+          // Write default _id index
+          const destIndexPath = path.join(collDir, 'indexes.json');
+          fs.writeFileSync(destIndexPath, JSON.stringify([{ name: '_id_', key: { _id: 1 } }], null, 2));
+        }
+
+        deletableFiles.push(jsonPath);
+
+        console.log(`[smartdb]   Migrated ${dbEntry.name}.${collName}: ${docs.length} documents`);
+      }
+    }
+  } catch (err) {
+    // Clean up any partially created directories
+    for (const dir of createdDirs) {
+      try {
+        fs.rmSync(dir, { recursive: true, force: true });
+      } catch {}
+    }
+    throw err;
+  }
+
+  return deletableFiles;
+}

package/ts/ts_smartdb/index.ts

@@ -6,3 +6,14 @@ export type { ISmartdbServerOptions } from './server/SmartdbServer.js';
 
 // Export bridge for advanced usage
 export { RustDbBridge } from './rust-db-bridge.js';
+
+// Export oplog / debug types
+export type {
+  IOpLogEntry,
+  IOpLogResult,
+  IOpLogStats,
+  IRevertResult,
+  ICollectionInfo,
+  IDocumentsResult,
+  ISmartDbMetrics,
+} from './rust-db-bridge.js';

package/ts/ts_smartdb/rust-db-bridge.ts

@@ -3,6 +3,82 @@ import * as path from 'path';
 import * as url from 'url';
 import { EventEmitter } from 'events';
 
+/**
+ * A single oplog entry returned from the Rust engine.
+ */
+export interface IOpLogEntry {
+  seq: number;
+  timestampMs: number;
+  op: 'insert' | 'update' | 'delete';
+  db: string;
+  collection: string;
+  documentId: string;
+  document: Record<string, any> | null;
+  previousDocument: Record<string, any> | null;
+}
+
+/**
+ * Aggregate oplog statistics.
+ */
+export interface IOpLogStats {
+  currentSeq: number;
+  totalEntries: number;
+  oldestSeq: number;
+  entriesByOp: {
+    insert: number;
+    update: number;
+    delete: number;
+  };
+}
+
+/**
+ * Result of a getOpLog query.
+ */
+export interface IOpLogResult {
+  entries: IOpLogEntry[];
+  currentSeq: number;
+  totalEntries: number;
+}
+
+/**
+ * Result of a revertToSeq command.
+ */
+export interface IRevertResult {
+  dryRun: boolean;
+  reverted: number;
+  targetSeq?: number;
+  entries?: IOpLogEntry[];
+  errors?: string[];
+}
+
+/**
+ * A collection info entry.
+ */
+export interface ICollectionInfo {
+  db: string;
+  name: string;
+  count: number;
+}
+
+/**
+ * Result of a getDocuments query.
+ */
+export interface IDocumentsResult {
+  documents: Record<string, any>[];
+  total: number;
+}
+
+/**
+ * Server metrics.
+ */
+export interface ISmartDbMetrics {
+  databases: number;
+  collections: number;
+  oplogEntries: number;
+  oplogCurrentSeq: number;
+  uptimeSeconds: number;
+}
+
 /**
  * Type-safe command definitions for the RustDb IPC protocol.
  */
@@ -10,7 +86,24 @@ type TSmartDbCommands = {
   start: { params: { config: ISmartDbRustConfig }; result: { connectionUri: string } };
   stop: { params: Record<string, never>; result: void };
   getStatus: { params: Record<string, never>; result: { running: boolean } };
-  getMetrics: { params: Record<string, never>; result: any };
+  getMetrics: { params: Record<string, never>; result: ISmartDbMetrics };
+  getOpLog: {
+    params: { sinceSeq?: number; limit?: number; db?: string; collection?: string };
+    result: IOpLogResult;
+  };
+  getOpLogStats: { params: Record<string, never>; result: IOpLogStats };
+  revertToSeq: {
+    params: { seq: number; dryRun?: boolean };
+    result: IRevertResult;
+  };
+  getCollections: {
+    params: { db?: string };
+    result: { collections: ICollectionInfo[] };
+  };
+  getDocuments: {
+    params: { db: string; collection: string; limit?: number; skip?: number };
+    result: IDocumentsResult;
+  };
 };
 
 /**
@@ -132,7 +225,38 @@ export class RustDbBridge extends EventEmitter {
     return await this.bridge.sendCommand('getStatus', {} as Record<string, never>) as { running: boolean };
   }
 
-  public async getMetrics(): Promise<any> {
-    return this.bridge.sendCommand('getMetrics', {} as Record<string, never>);
+  public async getMetrics(): Promise<ISmartDbMetrics> {
+    return this.bridge.sendCommand('getMetrics', {} as Record<string, never>) as Promise<ISmartDbMetrics>;
+  }
+
+  public async getOpLog(params: {
+    sinceSeq?: number;
+    limit?: number;
+    db?: string;
+    collection?: string;
+  } = {}): Promise<IOpLogResult> {
+    return this.bridge.sendCommand('getOpLog', params) as Promise<IOpLogResult>;
+  }
+
+  public async getOpLogStats(): Promise<IOpLogStats> {
+    return this.bridge.sendCommand('getOpLogStats', {} as Record<string, never>) as Promise<IOpLogStats>;
+  }
+
+  public async revertToSeq(seq: number, dryRun = false): Promise<IRevertResult> {
+    return this.bridge.sendCommand('revertToSeq', { seq, dryRun }) as Promise<IRevertResult>;
+  }
+
+  public async getCollections(db?: string): Promise<ICollectionInfo[]> {
+    const result = await this.bridge.sendCommand('getCollections', db ? { db } : {}) as { collections: ICollectionInfo[] };
+    return result.collections;
+  }
+
+  public async getDocuments(
+    db: string,
+    collection: string,
+    limit = 50,
+    skip = 0,
+  ): Promise<IDocumentsResult> {
+    return this.bridge.sendCommand('getDocuments', { db, collection, limit, skip }) as Promise<IDocumentsResult>;
   }
 }

package/ts/ts_smartdb/server/SmartdbServer.ts

@@ -1,4 +1,14 @@
 import { RustDbBridge } from '../rust-db-bridge.js';
+import { StorageMigrator } from '../../ts_migration/index.js';
+import type {
+  IOpLogEntry,
+  IOpLogResult,
+  IOpLogStats,
+  IRevertResult,
+  ICollectionInfo,
+  IDocumentsResult,
+  ISmartDbMetrics,
+} from '../rust-db-bridge.js';
 
 /**
  * Server configuration options
@@ -66,6 +76,12 @@ export class SmartdbServer {
       throw new Error('Server is already running');
     }
 
+    // Run storage migration for file-based storage before starting Rust engine
+    if (this.options.storage === 'file' && this.options.storagePath) {
+      const migrator = new StorageMigrator(this.options.storagePath);
+      await migrator.run();
+    }
+
     const spawned = await this.bridge.spawn();
     if (!spawned) {
       throw new Error(
@@ -156,4 +172,59 @@ export class SmartdbServer {
   get host(): string {
     return this.options.host ?? '127.0.0.1';
   }
+
+  // --- OpLog / Debug API ---
+
+  /**
+   * Get oplog entries, optionally filtered.
+   */
+  async getOpLog(params: {
+    sinceSeq?: number;
+    limit?: number;
+    db?: string;
+    collection?: string;
+  } = {}): Promise<IOpLogResult> {
+    return this.bridge.getOpLog(params);
+  }
+
+  /**
+   * Get aggregate oplog statistics.
+   */
+  async getOpLogStats(): Promise<IOpLogStats> {
+    return this.bridge.getOpLogStats();
+  }
+
+  /**
+   * Revert database state to a specific oplog sequence number.
+   * Use dryRun=true to preview which entries would be reverted.
+   */
+  async revertToSeq(seq: number, dryRun = false): Promise<IRevertResult> {
+    return this.bridge.revertToSeq(seq, dryRun);
+  }
+
+  /**
+   * List all collections across all databases, with document counts.
+   */
+  async getCollections(db?: string): Promise<ICollectionInfo[]> {
+    return this.bridge.getCollections(db);
+  }
+
+  /**
+   * Get documents from a collection with pagination.
+   */
+  async getDocuments(
+    db: string,
+    collection: string,
+    limit = 50,
+    skip = 0,
+  ): Promise<IDocumentsResult> {
+    return this.bridge.getDocuments(db, collection, limit, skip);
+  }
+
+  /**
+   * Get server metrics including database/collection counts and oplog info.
+   */
+  async getMetrics(): Promise<ISmartDbMetrics> {
+    return this.bridge.getMetrics();
+  }
 }