@push.rocks/smartdb 2.0.0 → 2.3.1

package/package.json

@@ -1,10 +1,12 @@
 {
   "name": "@push.rocks/smartdb",
-  "version": "2.0.0",
+  "version": "2.3.1",
   "private": false,
   "description": "A MongoDB-compatible embedded database server with wire protocol support, backed by a high-performance Rust engine.",
   "exports": {
-    ".": "./dist_ts/index.js"
+    ".": "./dist_ts/index.js",
+    "./debugui": "./dist_ts_debugui/index.js",
+    "./debugserver": "./dist_ts_debugserver/index.js"
   },
   "type": "module",
   "author": "Task Venture Capital GmbH",
@@ -12,7 +14,7 @@
   "scripts": {
     "test:before": "(tsrust)",
     "test": "(tstest test/. --verbose --logfile --timeout 60)",
-    "build": "(tsbuild tsfolders) && (tsrust)",
+    "build": "(tsbundle) && (tsbuild tsfolders) && (tsrust)",
     "buildDocs": "tsdoc"
   },
   "devDependencies": {
@@ -25,7 +27,10 @@
     "mongodb": "^7.1.1"
   },
   "dependencies": {
-    "@push.rocks/smartrust": "^1.3.2"
+    "@api.global/typedserver": "^8.0.0",
+    "@design.estate/dees-element": "^2.0.0",
+    "@push.rocks/smartrust": "^1.3.2",
+    "bson": "^7.2.0"
   },
   "browserslist": [
     "last 1 chrome versions"

package/readme.md

@@ -1,6 +1,6 @@
 # @push.rocks/smartdb
 
-A MongoDB-compatible embedded database server powered by Rust 🦀⚡ — use the official `mongodb` driver and it just works. No binary downloads, instant startup, zero config.
+A MongoDB-compatible embedded database server powered by Rust 🦀⚡ — use the official `mongodb` driver and it just works. No binary downloads, instant startup, zero config. Features a built-in **operation log** with **point-in-time revert** and a web-based **debug dashboard**.
 
 ## Install
 
@@ -14,6 +14,8 @@ npm install @push.rocks/smartdb
 
 For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
 
+---
+
 ## What It Does
 
 `@push.rocks/smartdb` is a **real database server** that speaks the wire protocol used by MongoDB drivers. The core engine is written in Rust for high performance, with a thin TypeScript orchestration layer. Connect with the standard `mongodb` Node.js driver — no mocks, no stubs, no external binaries required.
@@ -26,52 +28,55 @@ For reporting bugs, issues, or security vulnerabilities, please visit [community
 | **Binary download** | Bundled (~7MB) | ~200MB+ |
 | **Install** | `pnpm add` | System package / Docker |
 | **Persistence** | Memory or file-based | Full disk engine |
+| **Debug UI** | Built-in 🖥️ | External tooling |
+| **Point-in-time revert** | Built-in ⏪ | Requires oplog tailing |
 | **Perfect for** | Unit tests, CI/CD, prototyping, local dev, embedded | Production at scale |
 
-### Two Ways to Use It
+### Three Ways to Use It
 
-- 🏗️ **`SmartdbServer`** — Full control. Configure port, host, storage backend, Unix sockets. Great for test fixtures or custom setups.
 - 🎯 **`LocalSmartDb`** — Zero-config convenience. Give it a folder path, get a persistent database over a Unix socket. Done.
+- 🏗️ **`SmartdbServer`** — Full control. Configure port, host, storage backend, Unix sockets. Great for test fixtures or custom setups.
+- 🖥️ **`SmartdbDebugServer`** — Launch a web dashboard to visually browse collections, inspect the operation log, and revert to any point in time.
 
 ### Architecture: TypeScript + Rust 🦀
 
-SmartDB uses the same **sidecar binary** pattern as [@push.rocks/smartproxy](https://code.foss.global/push.rocks/smartproxy):
+SmartDB uses a **sidecar binary** pattern — TypeScript handles lifecycle, Rust handles all database operations:
 
 ```
-┌──────────────────────────────────────────────────────────┐
-│                   Your Application                        │
-│                  (TypeScript / Node.js)                    │
-│  ┌─────────────────┐     ┌───────────────────────────┐   │
-│  │  SmartdbServer   │────▶│  RustDbBridge (IPC)       │   │
-│  │  or LocalSmartDb │     │  @push.rocks/smartrust    │   │
-│  └─────────────────┘     └───────────┬───────────────┘   │
-└──────────────────────────────────────┼───────────────────┘
+┌──────────────────────────────────────────────────────────────┐
+│                   Your Application                            │
+│                  (TypeScript / Node.js)                        │
+│  ┌─────────────────┐     ┌───────────────────────────┐       │
+│  │  SmartdbServer   │────▶│  RustDbBridge (IPC)       │       │
+│  │  or LocalSmartDb │     │  @push.rocks/smartrust    │       │
+│  └─────────────────┘     └───────────┬───────────────┘       │
+└──────────────────────────────────────┼───────────────────────┘
                                        │ spawn + JSON IPC
                                        ▼
-┌──────────────────────────────────────────────────────────┐
-│                    rustdb binary 🦀                       │
-│                                                          │
-│  ┌──────────────┐  ┌──────────────┐  ┌───────────────┐  │
-│  │ Wire Protocol│→ │Command Router│→ │   Handlers    │  │
-│  │  (OP_MSG)    │  │  (40 cmds)   │  │ Find,Insert.. │  │
-│  └──────────────┘  └──────────────┘  └───────┬───────┘  │
-│                                               │          │
-│  ┌─────────┐ ┌────────┐ ┌───────────┐ ┌──────┴──────┐  │
-│  │  Query  │ │ Update │ │Aggregation│ │   Index     │  │
-│  │ Matcher │ │ Engine │ │  Engine   │ │   Engine    │  │
-│  └─────────┘ └────────┘ └───────────┘ └─────────────┘  │
-│                                                          │
-│  ┌──────────────────┐  ┌──────────────────┐             │
-│  │  MemoryStorage   │  │   FileStorage    │             │
-│  └──────────────────┘  └──────────────────┘             │
-└──────────────────────────────────────────────────────────┘
+┌──────────────────────────────────────────────────────────────┐
+│                    rustdb binary 🦀                           │
+│                                                              │
+│  ┌──────────────┐  ┌──────────────┐  ┌───────────────┐      │
+│  │ Wire Protocol│→ │Command Router│→ │   Handlers    │      │
+│  │  (OP_MSG)    │  │  (40+ cmds)  │  │ Find,Insert.. │      │
+│  └──────────────┘  └──────────────┘  └───────┬───────┘      │
+│                                               │              │
+│  ┌─────────┐ ┌────────┐ ┌───────────┐ ┌──────┴──────┐      │
+│  │  Query  │ │ Update │ │Aggregation│ │   Index     │      │
+│  │ Matcher │ │ Engine │ │  Engine   │ │   Engine    │      │
+│  └─────────┘ └────────┘ └───────────┘ └─────────────┘      │
+│                                                              │
+│  ┌──────────────────┐  ┌──────────────────┐  ┌──────────┐   │
+│  │  MemoryStorage   │  │   FileStorage    │  │  OpLog   │   │
+│  └──────────────────┘  └──────────────────┘  └──────────┘   │
+└──────────────────────────────────────────────────────────────┘
               ▲
               │ TCP / Unix Socket (wire protocol)
               │
-┌─────────────┴────────────────────────────────────────────┐
-│              MongoClient (mongodb npm driver)             │
-│              Connects directly to Rust binary             │
-└──────────────────────────────────────────────────────────┘
+┌─────────────┴────────────────────────────────────────────────┐
+│              MongoClient (mongodb npm driver)                  │
+│              Connects directly to Rust binary                 │
+└──────────────────────────────────────────────────────────────┘
 ```
 
 The TypeScript layer handles **lifecycle only** (start/stop/configure via IPC). All database operations flow directly from the `MongoClient` to the Rust binary over TCP or Unix sockets — **zero per-query IPC overhead**.
@@ -128,6 +133,87 @@ await client.close();
 await server.stop();
 ```
 
+### Option 3: Debug Server (Visual Dashboard) 🖥️
+
+Launch a web-based dashboard to inspect your database in real time:
+
+```typescript
+import { SmartdbServer } from '@push.rocks/smartdb';
+import { SmartdbDebugServer } from '@push.rocks/smartdb/debugserver';
+
+const server = new SmartdbServer({ storage: 'memory' });
+await server.start();
+
+const debugServer = new SmartdbDebugServer(server, { port: 4000 });
+await debugServer.start();
+// Open http://localhost:4000 in your browser 🚀
+```
+
+The debug dashboard gives you:
+- 📊 **Dashboard** — server status, uptime, database/collection counts, operation breakdown
+- 📁 **Collection Browser** — browse databases, collections, and documents interactively
+- 📝 **OpLog Timeline** — every insert, update, and delete with expandable field-level diffs
+- ⏪ **Point-in-Time Revert** — select any oplog sequence, preview what will be undone, and execute
+
+---
+
+## 📝 Operation Log & Point-in-Time Revert
+
+Every write operation (insert, update, delete) is automatically recorded in an in-memory **operation log (OpLog)** with full before/after document snapshots. This enables:
+
+- **Change tracking** — see exactly what changed, when, and in which collection
+- **Field-level diffs** — compare previous and new document states
+- **Point-in-time revert** — undo operations back to any sequence number
+- **Dry-run preview** — see what would be reverted before executing
+
+### Programmatic OpLog API
+
+```typescript
+import { SmartdbServer } from '@push.rocks/smartdb';
+
+const server = new SmartdbServer({ port: 27017 });
+await server.start();
+
+// ... perform some CRUD operations via MongoClient ...
+
+// Get oplog entries
+const oplog = await server.getOpLog({ limit: 50 });
+console.log(oplog.entries);
+// [{ seq: 1, op: 'insert', db: 'myapp', collection: 'users', document: {...}, previousDocument: null }, ...]
+
+// Get aggregate stats
+const stats = await server.getOpLogStats();
+console.log(stats);
+// { currentSeq: 42, totalEntries: 42, entriesByOp: { insert: 20, update: 15, delete: 7 } }
+
+// Preview a revert (dry run)
+const preview = await server.revertToSeq(30, true);
+console.log(`Would undo ${preview.reverted} operations`);
+
+// Execute the revert — undoes all operations after seq 30
+const result = await server.revertToSeq(30, false);
+console.log(`Reverted ${result.reverted} operations`);
+
+// Browse collections programmatically
+const collections = await server.getCollections();
+const docs = await server.getDocuments('myapp', 'users', 50, 0);
+```
+
+### OpLog Entry Structure
+
+Each entry contains:
+
+| Field | Type | Description |
+|---|---|---|
+| `seq` | `number` | Monotonically increasing sequence number |
+| `timestampMs` | `number` | Unix timestamp in milliseconds |
+| `op` | `'insert' \| 'update' \| 'delete'` | Operation type |
+| `db` | `string` | Database name |
+| `collection` | `string` | Collection name |
+| `documentId` | `string` | Document `_id` as hex string |
+| `document` | `object \| null` | New document state (null for deletes) |
+| `previousDocument` | `object \| null` | Previous document state (null for inserts) |
+
 ---
 
 ## API Reference
@@ -175,6 +261,12 @@ const server = new SmartdbServer({
 | `port` | `number` | Configured port (TCP mode) |
 | `host` | `string` | Configured host (TCP mode) |
 | `socketPath` | `string \| undefined` | Socket path (socket mode) |
+| `getMetrics()` | `Promise<ISmartDbMetrics>` | Server metrics (db/collection counts, uptime) |
+| `getOpLog(params?)` | `Promise<IOpLogResult>` | Query oplog entries with optional filters |
+| `getOpLogStats()` | `Promise<IOpLogStats>` | Aggregate oplog statistics |
+| `revertToSeq(seq, dryRun?)` | `Promise<IRevertResult>` | Revert to a specific oplog sequence |
+| `getCollections(db?)` | `Promise<ICollectionInfo[]>` | List all collections with counts |
+| `getDocuments(db, coll, limit?, skip?)` | `Promise<IDocumentsResult>` | Browse documents with pagination |
 
 ### LocalSmartDb
 
@@ -202,13 +294,34 @@ const db = new LocalSmartDb({
 | `getServer()` | `SmartdbServer` | Access the underlying server |
 | `running` | `boolean` | Whether the server is running |
 
-#### Connection Info (`ILocalSmartDbConnectionInfo`)
+### SmartdbDebugServer
+
+Web-based debug dashboard served via `@api.global/typedserver`. Import from the `debugserver` subpath:
+
+```typescript
+import { SmartdbDebugServer } from '@push.rocks/smartdb/debugserver';
+
+const debugServer = new SmartdbDebugServer(server, { port: 4000 });
+await debugServer.start();
+// Dashboard at http://localhost:4000
+
+await debugServer.stop();
+```
+
+The UI is bundled as base64-encoded content (via `@git.zone/tsbundle`) and served from memory — no static file directory needed.
+
+### SmartdbDebugUi (Web Component)
+
+For embedding the debug UI directly into your own web application, import the `<smartdb-debugui>` web component:
 
 ```typescript
-interface ILocalSmartDbConnectionInfo {
-  socketPath: string;      // e.g., /tmp/smartdb-abc123.sock
-  connectionUri: string;   // e.g., mongodb://%2Ftmp%2Fsmartdb-abc123.sock
-}
+import { SmartdbDebugUi } from '@push.rocks/smartdb/debugui';
+
+// In your HTML/lit template:
+// <smartdb-debugui .server=${mySmartdbServer}></smartdb-debugui>
+//
+// Or in HTTP mode (when served by SmartdbDebugServer):
+// <smartdb-debugui apiBaseUrl=""></smartdb-debugui>
 ```
 
 ---
@@ -377,10 +490,10 @@ The Rust engine is organized as a Cargo workspace with 8 focused crates:
 | `rustdb-config` | Server configuration types (serde, camelCase JSON) |
 | `rustdb-wire` | Wire protocol parser/encoder (OP_MSG, OP_QUERY, OP_REPLY) |
 | `rustdb-query` | Query matcher, update engine, aggregation, sort, projection |
-| `rustdb-storage` | Storage backends (memory, file) + WAL + OpLog |
+| `rustdb-storage` | Storage backends (memory, file), OpLog with point-in-time replay |
 | `rustdb-index` | B-tree/hash indexes, query planner (IXSCAN/COLLSCAN) |
 | `rustdb-txn` | Transaction + session management with snapshot isolation |
-| `rustdb-commands` | 40 command handlers wiring everything together |
+| `rustdb-commands` | 40+ command handlers wiring everything together |
 
 Cross-compiled for `linux_amd64` and `linux_arm64` via [@git.zone/tsrust](https://www.npmjs.com/package/@git.zone/tsrust).
 
@@ -410,6 +523,12 @@ tap.test('should insert and find', async () => {
   expect(item?.price).toEqual(9.99);
 });
 
+tap.test('should track changes in oplog', async () => {
+  const oplog = await server.getOpLog();
+  expect(oplog.entries.length).toBeGreaterThan(0);
+  expect(oplog.entries[0].op).toEqual('insert');
+});
+
 tap.test('teardown', async () => {
   await client.close();
   await server.stop();
@@ -422,7 +541,7 @@ export default tap.start();
 
 ## License and Legal Information
 
-This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [license](./license) file.
+This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [LICENSE](./LICENSE) file.
 
 **Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
 
@@ -434,7 +553,7 @@ Use of these trademarks must comply with Task Venture Capital GmbH's Trademark G
 
 ### Company Information
 
-Task Venture Capital GmbH
+Task Venture Capital GmbH  
 Registered at District Court Bremen HRB 35230 HB, Germany
 
 For any legal inquiries or further information, please contact us via email at hello@task.vc.

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartdb',
-  version: '2.0.0',
+  version: '2.3.1',
   description: 'A MongoDB-compatible embedded database server with wire protocol support, backed by a high-performance Rust engine.'
 }

package/ts/index.ts

@@ -7,5 +7,19 @@ export * from './ts_smartdb/index.js';
 export { LocalSmartDb } from './ts_local/index.js';
 export type { ILocalSmartDbOptions, ILocalSmartDbConnectionInfo } from './ts_local/index.js';
 
+// Export migration
+export { StorageMigrator } from './ts_migration/index.js';
+
 // Export commitinfo
 export { commitinfo };
+
+// Re-export oplog / debug types for convenience
+export type {
+  IOpLogEntry,
+  IOpLogResult,
+  IOpLogStats,
+  IRevertResult,
+  ICollectionInfo,
+  IDocumentsResult,
+  ISmartDbMetrics,
+} from './ts_smartdb/index.js';

package/ts/ts_local/classes.localsmartdb.ts

@@ -2,6 +2,7 @@ import * as crypto from 'crypto';
 import * as path from 'path';
 import * as os from 'os';
 import { SmartdbServer } from '../ts_smartdb/index.js';
+import { StorageMigrator } from '../ts_migration/index.js';
 
 /**
  * Connection information returned by LocalSmartDb.start()
@@ -73,6 +74,10 @@ export class LocalSmartDb {
       throw new Error('LocalSmartDb is already running');
     }
 
+    // Run storage migration before starting the Rust engine
+    const migrator = new StorageMigrator(this.options.folderPath);
+    await migrator.run();
+
     // Use provided socket path or generate one
     this.generatedSocketPath = this.options.socketPath ?? this.generateSocketPath();
 

package/ts/ts_migration/classes.storagemigrator.ts

@@ -0,0 +1,93 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { migrateV0ToV1 } from './migrators/v0_to_v1.js';
+
+/**
+ * Detected storage format version.
+ * - v0: Legacy JSON format ({db}/{coll}.json files)
+ * - v1: Bitcask binary format ({db}/{coll}/data.rdb directories)
+ */
+type TStorageVersion = 0 | 1;
+
+/**
+ * StorageMigrator — runs before the Rust engine starts.
+ *
+ * Detects the current storage format version and runs the appropriate
+ * migration chain. The Rust engine only knows the current format (v1).
+ *
+ * Migration is safe: original files are never modified or deleted.
+ * On success, a console hint is printed about which old files can be removed.
+ */
+export class StorageMigrator {
+  private storagePath: string;
+
+  constructor(storagePath: string) {
+    this.storagePath = storagePath;
+  }
+
+  /**
+   * Run any needed migrations. Safe to call even if storage is already current.
+   */
+  async run(): Promise<void> {
+    if (!fs.existsSync(this.storagePath)) {
+      return; // No data yet — nothing to migrate
+    }
+
+    const version = this.detectVersion();
+
+    if (version === 1) {
+      return; // Already current
+    }
+
+    if (version === 0) {
+      console.log(`[smartdb] Detected v0 (JSON) storage format at ${this.storagePath}`);
+      console.log(`[smartdb] Running migration v0 → v1 (Bitcask binary format)...`);
+
+      const deletableFiles = await migrateV0ToV1(this.storagePath);
+
+      if (deletableFiles.length > 0) {
+        console.log(`[smartdb] Migration v0 → v1 complete.`);
+        console.log(`[smartdb] The following old files can be safely deleted:`);
+        for (const f of deletableFiles) {
+          console.log(`[smartdb]   ${f}`);
+        }
+      } else {
+        console.log(`[smartdb] Migration v0 → v1 complete. No old files to clean up.`);
+      }
+    }
+  }
+
+  /**
+   * Detect the storage format version by inspecting the directory structure.
+   *
+   * v0: {db}/{coll}.json files exist
+   * v1: {db}/{coll}/data.rdb directories exist
+   */
+  private detectVersion(): TStorageVersion {
+    const entries = fs.readdirSync(this.storagePath, { withFileTypes: true });
+
+    for (const entry of entries) {
+      if (!entry.isDirectory()) continue;
+
+      const dbDir = path.join(this.storagePath, entry.name);
+      const dbEntries = fs.readdirSync(dbDir, { withFileTypes: true });
+
+      for (const dbEntry of dbEntries) {
+        // v1: subdirectory with data.rdb
+        if (dbEntry.isDirectory()) {
+          const dataRdb = path.join(dbDir, dbEntry.name, 'data.rdb');
+          if (fs.existsSync(dataRdb)) {
+            return 1;
+          }
+        }
+        // v0: .json file (not .indexes.json)
+        if (dbEntry.isFile() && dbEntry.name.endsWith('.json') && !dbEntry.name.endsWith('.indexes.json')) {
+          return 0;
+        }
+      }
+    }
+
+    // Empty or unrecognized — treat as v1 (fresh start)
+    return 1;
+  }
+}

package/ts/ts_migration/index.ts

@@ -0,0 +1 @@
+export { StorageMigrator } from './classes.storagemigrator.js';