@sqd-pipes/delta-db 0.0.1-alpha.0

package/README.md

@@ -0,0 +1,83 @@
+# @subsquid/delta-db
+
+Embedded rollback-aware computation engine for blockchain data. Routes raw events through **reducers** and **materialized views**, producing delta records (insert/update/delete) for downstream targets.
+
+## Build
+
+Requires **Rust** (stable) and **Node.js** >= 18.
+
+```bash
+cd packages/delta-db-node
+npm install
+
+# Debug build
+npx napi build --cargo-cwd ../.. --features napi
+
+# Release build
+npx napi build --cargo-cwd ../.. --features napi --release
+```
+
+This produces `delta-db.node` (native binary) and regenerates `index.d.ts`.
+
+If `index.js` is missing after build, create it:
+
+```js
+const { DeltaDb } = require('./delta-db.node')
+module.exports.DeltaDb = DeltaDb
+```
+
+## Test
+
+```bash
+npm test
+```
+
+## Usage
+
+### Direct API
+
+```typescript
+import { DeltaDb } from '@subsquid/delta-db'
+
+const db = DeltaDb.open({
+  schema: `
+    CREATE TABLE swaps (
+      block_number UInt64,
+      pool         String,
+      amount       Float64
+    );
+    CREATE MATERIALIZED VIEW volume AS
+      SELECT pool, sum(amount) AS total, count() AS cnt
+      FROM swaps GROUP BY pool;
+  `,
+  dataDir: './data', // optional, enables persistence
+})
+
+const batch = db.ingest({
+  data: {
+    swaps: [
+      { block_number: 1000, pool: 'ETH/USDC', amount: 100 },
+    ],
+  },
+  rollbackChain: [{ number: 1000, hash: '0x...' }],
+  finalizedHead: { number: 999, hash: '0x...' },
+})
+// batch.records → [{ table: 'swaps', op: 'insert', ... }, { table: 'volume', op: 'insert', ... }]
+```
+
+### Pipes SDK
+
+```typescript
+import { deltaDbTarget } from '@subsquid/delta-db/pipes'
+
+await source
+  .pipe(decoder)
+  .pipeTo(deltaDbTarget({
+    schema: '...',
+    dataDir: './data',
+    transform: (data) => ({ transfers: data.transfers.map(formatRow) }),
+    onDelta: async ({ batch }) => {
+      await clickhouse.insert(batch.records)
+    },
+  }))
+```

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@sqd-pipes/delta-db",
+  "version": "0.0.1-alpha.0",
+  "description": "Embedded rollback-aware computation engine for blockchain data",
+  "main": "src/index.js",
+  "types": "src/index.d.ts",
+  "napi": {
+    "name": "delta-db",
+    "triples": {
+      "defaults": true
+    }
+  },
+  "scripts": {
+    "build": "napi build --cargo-cwd ../.. --features napi --release",
+    "build:debug": "napi build --cargo-cwd ../.. --features napi",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  },
+  "dependencies": {
+    "@subsquid/pipes": "^0.1.0-beta.16",
+    "@msgpack/msgpack": "^3.0.0"
+  },
+  "devDependencies": {
+    "@napi-rs/cli": "^2.18.0",
+    "vitest": "^3.0.0"
+  },
+  "files": [
+    "src/index.js",
+    "src/index.d.ts",
+    "src/delta-db.node"
+  ],
+  "license": "MIT"
+}

package/src/index.d.ts

@@ -0,0 +1,80 @@
+/** Configuration for opening a DeltaDb instance. */
+export interface DeltaDbConfig {
+  /** SQL schema definition string. */
+  schema: string
+  /**
+   * Path to RocksDB data directory for persistence.
+   * When omitted, uses in-memory storage (data lost on restart).
+   */
+  dataDir?: string
+  /** Maximum buffer size before backpressure (default: 10000). */
+  maxBufferSize?: number
+}
+
+/** Block cursor: number + hash. */
+export interface DeltaDbCursor {
+  number: number
+  hash: string
+}
+
+/** A single delta record. */
+export interface DeltaRecord {
+  table: string
+  operation: string
+  key: Record<string, any>
+  values: Record<string, any>
+  prevValues?: Record<string, any> | null
+}
+
+/** A batch of delta records. */
+export interface DeltaBatch {
+  sequence: number
+  finalizedHead?: DeltaDbCursor | null
+  latestHead?: DeltaDbCursor | null
+  records: Array<DeltaRecord>
+}
+
+/** Input for the atomic `ingest()` method. */
+export interface IngestInput {
+  /** Table name → rows. Rows must contain `block_number`. */
+  data: Record<string, Array<Record<string, any>>>
+  /** Unfinalized blocks with hashes for fork resolution. */
+  rollbackChain?: Array<DeltaDbCursor>
+  /** Finalized head cursor — both number and hash stored. */
+  finalizedHead: DeltaDbCursor
+}
+
+/** Delta DB — embedded rollback-aware computation engine. */
+export declare class DeltaDb {
+  /** Open a new DeltaDb instance. */
+  static open(config: DeltaDbConfig): DeltaDb
+  /**
+   * Process a batch of rows for a raw table.
+   * Returns true if backpressure should be applied.
+   */
+  processBatch(table: string, block: number, rows: Array<Record<string, any>>): boolean
+  /** Roll back all state after fork_point. */
+  rollback(forkPoint: number): void
+  /** Finalize all state up to and including the given block. */
+  finalize(block: number): void
+  /**
+   * Atomic ingest: process all tables, store rollback chain, finalize, flush.
+   * Returns the delta batch, or null if no records produced.
+   */
+  ingest(input: IngestInput): DeltaBatch | null
+  /**
+   * Find the common ancestor between our state and the Portal's chain.
+   * Returns the matching block cursor, or null if no common ancestor found.
+   */
+  resolveForkCursor(previousBlocks: Array<DeltaDbCursor>): DeltaDbCursor | null
+  /** Flush buffered deltas into a batch. Returns null if no pending records. */
+  flush(): DeltaBatch | null
+  /** Acknowledge a flushed batch by sequence number. */
+  ack(sequence: number): void
+  /** Number of pending (unflushed) delta records. */
+  get pendingCount(): number
+  /** Whether backpressure should be applied. */
+  get isBackpressured(): boolean
+  /** Current cursor: latest processed block + hash. Null if no blocks processed. */
+  get cursor(): DeltaDbCursor | null
+}

package/src/index.js

@@ -0,0 +1,85 @@
+// @ts-ignore
+const { DeltaDb: NativeDeltaDb } = require('./delta-db.node')
+const { encode, decode } = require('@msgpack/msgpack')
+
+class DeltaDb {
+  /** @type {InstanceType<typeof NativeDeltaDb>} */
+  #native
+
+  /** @param {InstanceType<typeof NativeDeltaDb>} native */
+  constructor(native) {
+    this.#native = native
+  }
+
+  /** Open a new DeltaDb instance. */
+  static open(config) {
+    return new DeltaDb(NativeDeltaDb.open(config))
+  }
+
+  /**
+   * Process a batch of rows for a raw table.
+   * Returns true if backpressure should be applied.
+   */
+  processBatch(table, block, rows) {
+    return this.#native.processBatch(table, block, Buffer.from(encode(rows)))
+  }
+
+  /** Roll back all state after fork_point. */
+  rollback(forkPoint) {
+    this.#native.rollback(forkPoint)
+  }
+
+  /** Finalize all state up to and including the given block. */
+  finalize(block) {
+    this.#native.finalize(block)
+  }
+
+  /**
+   * Atomic ingest: process all tables, store rollback chain, finalize, flush.
+   * Returns the delta batch, or null if no records produced.
+   */
+  ingest(input) {
+    const buf = this.#native.ingest({
+      data: Buffer.from(encode(input.data)),
+      rollbackChain: input.rollbackChain,
+      finalizedHead: input.finalizedHead,
+    })
+    return buf ? decode(buf) : null
+  }
+
+  /**
+   * Find the common ancestor between our state and the Portal's chain.
+   * Returns the matching block cursor, or null if no common ancestor found.
+   */
+  resolveForkCursor(previousBlocks) {
+    return this.#native.resolveForkCursor(previousBlocks)
+  }
+
+  /** Flush buffered deltas into a batch. Returns null if no pending records. */
+  flush() {
+    const buf = this.#native.flush()
+    return buf ? decode(buf) : null
+  }
+
+  /** Acknowledge a flushed batch by sequence number. */
+  ack(sequence) {
+    this.#native.ack(sequence)
+  }
+
+  /** Number of pending (unflushed) delta records. */
+  get pendingCount() {
+    return this.#native.pendingCount
+  }
+
+  /** Whether backpressure should be applied. */
+  get isBackpressured() {
+    return this.#native.isBackpressured
+  }
+
+  /** Current cursor: latest processed block + hash. Null if no blocks processed. */
+  get cursor() {
+    return this.#native.cursor
+  }
+}
+
+module.exports.DeltaDb = DeltaDb