@sqd-pipes/delta-db 0.0.1-alpha.6 → 0.0.1-alpha.8

package/README.md

@@ -1,83 +1,173 @@
-# @subsquid/delta-db
+# @sqd-pipes/delta-db
 
-Embedded rollback-aware computation engine for blockchain data. Routes raw events through **reducers** and **materialized views**, producing delta records (insert/update/delete) for downstream targets.
+Embedded rollback-aware computation engine for blockchain data. Routes raw events through **reducers** (TypeScript functions) and **materialized views**, producing delta records (insert/update/delete) for downstream targets.
 
-## Build
-
-Requires **Rust** (stable) and **Node.js** >= 18.
+## Install
 
 ```bash
-cd packages/delta-db-node
-npm install
+pnpm add @sqd-pipes/delta-db
+```
 
-# Debug build
-npx napi build --cargo-cwd ../.. --features napi
+## Build from source
+
+Requires **Rust** (stable) and **Node.js** >= 18.
 
-# Release build
-npx napi build --cargo-cwd ../.. --features napi --release
+```bash
+pnpm install
+pnpm run build        # release
+pnpm run build:debug  # debug
 ```
 
-This produces `delta-db.node` (native binary) and regenerates `index.d.ts`.
+## Quick start
 
-If `index.js` is missing after build, create it:
+```typescript
+import { Pipeline, uint64, string, float64, interval } from '@sqd-pipes/delta-db'
 
-```js
-const { DeltaDb } = require('./delta-db.node')
-module.exports.DeltaDb = DeltaDb
-```
+const p = new Pipeline()
 
-## Test
+const swaps = p.table('swaps', {
+  block_number: uint64(),
+  pool:         string(),
+  amount:       float64(),
+})
 
-```bash
-npm test
+swaps
+  .createReducer('totals', {
+    groupBy: 'pool',
+    initialState: { volume: 0 },
+    reduce(state, row) {
+      const volume = state.volume + row.amount
+      return [{ volume }, { pool: row.pool, volume }]
+    },
+  })
+  .createView('pool_volume', {
+    groupBy: ['pool'],
+    select: (agg) => ({
+      pool:       agg.key.pool,
+      totalVolume: agg.sum('volume'),
+      tradeCount: agg.count(),
+    }),
+  })
+
+const db = p.build()  // in-memory
+// const db = p.build({ dataDir: './data' })  // persistent (RocksDB)
+
+db.processBatch('swaps', 1000, [
+  { pool: 'ETH/USDC', amount: 100 },
+  { pool: 'ETH/USDC', amount: 200 },
+])
+
+const batch = db.flush()!
+// batch.tables.pool_volume → [{ key: { pool: 'ETH/USDC' }, values: { totalVolume: 300, tradeCount: 2 } }]
 ```
 
-## Usage
+## API
 
-### Direct API
+### Pipeline builder
 
 ```typescript
-import { DeltaDb } from '@subsquid/delta-db'
-
-const db = DeltaDb.open({
-  schema: `
-    CREATE TABLE swaps (
-      block_number UInt64,
-      pool         String,
-      amount       Float64
-    );
-    CREATE MATERIALIZED VIEW volume AS
-      SELECT pool, sum(amount) AS total, count() AS cnt
-      FROM swaps GROUP BY pool;
-  `,
-  dataDir: './data', // optional, enables persistence
+const p = new Pipeline()
+
+// Define a table — column types are inferred into the row type
+const table = p.table('name', {
+  block_number: uint64(),
+  user:         string(),
+  amount:       float64(),
+  timestamp:    datetime(),
+  metadata:     json<MyType>(),  // json() is generic
 })
 
-const batch = db.ingest({
-  data: {
-    swaps: [
-      { block_number: 1000, pool: 'ETH/USDC', amount: 100 },
-    ],
+// Create a reducer from a table (or another reducer)
+const reducer = table.createReducer('reducer_name', {
+  groupBy: 'user',                          // validated against row keys
+  initialState: { total: 0 },              // state type inferred
+  reduce(state, row) {                      // state & row fully typed
+    return [
+      { total: state.total + row.amount },  // new state
+      { user: row.user, total: ... },       // emit (or null to skip)
+    ]
   },
-  rollbackChain: [{ number: 1000, hash: '0x...' }],
-  finalizedHead: { number: 999, hash: '0x...' },
 })
-// batch.records → [{ table: 'swaps', op: 'insert', ... }, { table: 'volume', op: 'insert', ... }]
+
+// Chain a reducer from another reducer's output
+const chained = reducer.createReducer('downstream', { ... })
+
+// Create a materialized view
+reducer.createView('summary', {
+  groupBy: ['user'],
+  select: (agg) => ({
+    user:   agg.key.user,
+    total:  agg.sum('total'),         // validated against emit keys
+    count:  agg.count(),
+    first:  agg.first('total'),
+    last:   agg.last('total'),
+  }),
+})
+
+// Time-window grouping
+reducer.createView('candles_5m', {
+  groupBy: ['pool', interval('timestamp', '5 minutes').as('window_start')],
+  select: (agg) => ({
+    pool:        agg.key.pool,
+    windowStart: agg.key.window_start,
+    open:        agg.first('price'),
+    high:        agg.max('price'),
+    low:         agg.min('price'),
+    close:       agg.last('price'),
+    volume:      agg.sum('volume'),
+  }),
+})
+
+// Build the database
+const db = p.build()                          // in-memory
+const db = p.build({ dataDir: './data' })     // RocksDB persistence
+const db = p.build({ dataDir: ':memory:' })   // explicit in-memory (SQLite convention)
 ```
 
-### Pipes SDK
+### Column types
+
+| Function     | SQL type   | TypeScript type |
+|-------------|------------|-----------------|
+| `uint64()`  | `UInt64`   | `number`        |
+| `int64()`   | `Int64`    | `number`        |
+| `float64()` | `Float64`  | `number`        |
+| `uint256()` | `Uint256`  | `bigint`        |
+| `string()`  | `String`   | `string`        |
+| `datetime()` | `DateTime` | `number`        |
+| `boolean()` | `Boolean`  | `boolean`       |
+| `bytes()`   | `Bytes`    | `Uint8Array`    |
+| `base58()`  | `Base58`   | `string`        |
+| `json<T>()` | `Json`     | `T` (default `any`) |
+
+### DeltaDb (low-level)
 
 ```typescript
-import { deltaDbTarget } from '@subsquid/delta-db/pipes'
-
-await source
-  .pipe(decoder)
-  .pipeTo(deltaDbTarget({
-    schema: '...',
-    dataDir: './data',
-    transform: (data) => ({ transfers: data.transfers.map(formatRow) }),
-    onDelta: async ({ batch }) => {
-      await clickhouse.insert(batch.records)
-    },
-  }))
+import { DeltaDb } from '@sqd-pipes/delta-db'
+
+const db = DeltaDb.open({ schema: 'CREATE TABLE ...', dataDir: './data' })
+
+db.processBatch('table', blockNumber, rows)
+db.rollback(forkPoint)
+db.finalize(blockNumber)
+db.flush()        // → DeltaBatch | null
+db.ingest(input)  // atomic: process + finalize + flush
+```
+
+### Aggregation functions
+
+`sum`, `count`, `min`, `max`, `avg`, `first`, `last`
+
+### Virtual tables
+
+Tables declared with `{ virtual: true }` are processed by reducers but don't emit raw row deltas:
+
+```typescript
+const orders = p.table('orders', { ... }, { virtual: true })
+```
+
+## Test
+
+```bash
+pnpm test
+pnpm run lint
 ```

package/dist/column.d.ts

@@ -0,0 +1,20 @@
+export interface ColumnType<T = any> {
+    readonly _sql: string;
+    /** Phantom field for type inference — not present at runtime. */
+    readonly _type?: T;
+}
+export declare const uint64: () => ColumnType<number>;
+export declare const int64: () => ColumnType<number>;
+export declare const float64: () => ColumnType<number>;
+export declare const uint256: () => ColumnType<bigint>;
+export declare const string: () => ColumnType<string>;
+export declare const datetime: () => ColumnType<number>;
+export declare const boolean: () => ColumnType<boolean>;
+export declare const bytes: () => ColumnType<Uint8Array<ArrayBufferLike>>;
+export declare const base58: () => ColumnType<string>;
+export declare function json<T = any>(): ColumnType<T>;
+/** Infer the row type from a column definition record. */
+export type InferRow<T extends Record<string, ColumnType>> = {
+    [K in keyof T]: T[K] extends ColumnType<infer V> ? V : unknown;
+};
+//# sourceMappingURL=column.d.ts.map

package/dist/column.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.base58 = exports.bytes = exports.boolean = exports.datetime = exports.string = exports.uint256 = exports.float64 = exports.int64 = exports.uint64 = void 0;
+exports.json = json;
+function col(sql) {
+    return () => ({ _sql: sql });
+}
+exports.uint64 = col('UInt64');
+exports.int64 = col('Int64');
+exports.float64 = col('Float64');
+exports.uint256 = col('Uint256');
+exports.string = col('String');
+exports.datetime = col('DateTime');
+exports.boolean = col('Boolean');
+exports.bytes = col('Bytes');
+exports.base58 = col('Base58');
+function json() {
+    return { _sql: 'Json' };
+}
+//# sourceMappingURL=column.js.map

package/dist/column.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"column.js","sourceRoot":"","sources":["../src/column.ts"],"names":[],"mappings":";;;AAmBA,oBAEC;AAfD,SAAS,GAAG,CAAI,GAAW;IACzB,OAAO,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAA;AAC9B,CAAC;AAEY,QAAA,MAAM,GAAG,GAAG,CAAS,QAAQ,CAAC,CAAA;AAC9B,QAAA,KAAK,GAAG,GAAG,CAAS,OAAO,CAAC,CAAA;AAC5B,QAAA,OAAO,GAAG,GAAG,CAAS,SAAS,CAAC,CAAA;AAChC,QAAA,OAAO,GAAG,GAAG,CAAS,SAAS,CAAC,CAAA;AAChC,QAAA,MAAM,GAAG,GAAG,CAAS,QAAQ,CAAC,CAAA;AAC9B,QAAA,QAAQ,GAAG,GAAG,CAAS,UAAU,CAAC,CAAA;AAClC,QAAA,OAAO,GAAG,GAAG,CAAU,SAAS,CAAC,CAAA;AACjC,QAAA,KAAK,GAAG,GAAG,CAAa,OAAO,CAAC,CAAA;AAChC,QAAA,MAAM,GAAG,GAAG,CAAS,QAAQ,CAAC,CAAA;AAC3C,SAAgB,IAAI;IAClB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;AACzB,CAAC"}

package/dist/ddl.d.ts

@@ -0,0 +1,45 @@
+import type { ColumnType } from './column';
+import type { StateFieldDef } from './delta-db';
+export declare function parseDuration(s: string): number;
+export interface IntervalExpr {
+    _type: 'interval';
+    column: string;
+    seconds: number;
+    alias?: string;
+    as(alias: string): IntervalExpr;
+}
+export declare function interval(column: string, duration: string): IntervalExpr;
+export interface AggExpr {
+    _type: 'agg';
+    func: string;
+    column: string | null;
+}
+export interface KeyRef {
+    _type: 'key';
+    column: string;
+}
+export interface AggProxy<TSource = any> {
+    key: Record<string, KeyRef>;
+    sum(column: string & keyof TSource): AggExpr;
+    count(): AggExpr;
+    first(column: string & keyof TSource): AggExpr;
+    last(column: string & keyof TSource): AggExpr;
+    min(column: string & keyof TSource): AggExpr;
+    max(column: string & keyof TSource): AggExpr;
+    avg(column: string & keyof TSource): AggExpr;
+}
+export type GroupByItem = string | IntervalExpr;
+export interface ReducerOptions<TState, TRow, TEmit> {
+    groupBy: (string & keyof TRow) | (string & keyof TRow)[];
+    initialState: TState;
+    reduce: (state: TState, row: TRow) => [TState, TEmit | TEmit[] | null];
+}
+export interface ViewOptions<TSource = any> {
+    groupBy: GroupByItem | GroupByItem[];
+    select: (agg: AggProxy<TSource>) => Record<string, AggExpr | KeyRef>;
+}
+export declare function inferStateFields(initialState: Record<string, unknown>): StateFieldDef[];
+export declare function tableToSql(name: string, columns: Record<string, ColumnType>, virtual: boolean): string;
+export declare function reducerToSql(name: string, source: string, groupBy: string[], stateFields: StateFieldDef[]): string;
+export declare function viewToSql(name: string, source: string, groupByItems: GroupByItem[], selectFn: (agg: AggProxy<any>) => Record<string, AggExpr | KeyRef>): string;
+//# sourceMappingURL=ddl.d.ts.map

package/dist/ddl.js

@@ -0,0 +1,165 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseDuration = parseDuration;
+exports.interval = interval;
+exports.inferStateFields = inferStateFields;
+exports.tableToSql = tableToSql;
+exports.reducerToSql = reducerToSql;
+exports.viewToSql = viewToSql;
+// ─── Duration parsing ────────────────────────────────────────────
+const DURATION_UNITS = {
+    second: 1,
+    seconds: 1,
+    sec: 1,
+    s: 1,
+    minute: 60,
+    minutes: 60,
+    min: 60,
+    m: 60,
+    hour: 3600,
+    hours: 3600,
+    hr: 3600,
+    h: 3600,
+    day: 86400,
+    days: 86400,
+    d: 86400,
+};
+function parseDuration(s) {
+    const match = s.trim().match(/^(\d+)\s*(\w+)$/);
+    if (!match) {
+        throw new Error(`invalid duration: '${s}'`);
+    }
+    const n = parseInt(match[1], 10);
+    const unit = match[2].toLowerCase();
+    const mult = DURATION_UNITS[unit];
+    if (!mult) {
+        throw new Error(`unknown duration unit: '${unit}'`);
+    }
+    return n * mult;
+}
+function makeInterval(column, seconds, alias) {
+    return {
+        _type: 'interval',
+        column,
+        seconds,
+        alias,
+        as(a) {
+            return makeInterval(column, seconds, a);
+        },
+    };
+}
+function interval(column, duration) {
+    return makeInterval(column, parseDuration(duration));
+}
+// ─── Type inference from initialState ────────────────────────────
+function inferStateFields(initialState) {
+    const fields = [];
+    for (const [name, value] of Object.entries(initialState)) {
+        let columnType;
+        let defaultValue;
+        switch (typeof value) {
+            case 'number':
+                columnType = 'Float64';
+                defaultValue = String(value);
+                break;
+            case 'bigint':
+                columnType = 'UInt64';
+                defaultValue = String(value);
+                break;
+            case 'string':
+                columnType = 'String';
+                defaultValue = `'${value}'`;
+                break;
+            case 'boolean':
+                columnType = 'Boolean';
+                defaultValue = value ? 'true' : 'false';
+                break;
+            case 'object':
+                columnType = 'Json';
+                defaultValue = `'${JSON.stringify(value)}'`;
+                break;
+            default:
+                columnType = 'String';
+                defaultValue = `'${String(value)}'`;
+        }
+        fields.push({ name, columnType, defaultValue });
+    }
+    return fields;
+}
+// ─── DDL generators ──────────────────────────────────────────────
+function tableToSql(name, columns, virtual) {
+    const prefix = virtual ? 'CREATE VIRTUAL TABLE' : 'CREATE TABLE';
+    const cols = Object.entries(columns)
+        .map(([name, ct]) => `${name} ${ct._sql}`)
+        .join(', ');
+    return `${prefix} ${name} (${cols});`;
+}
+function reducerToSql(name, source, groupBy, stateFields) {
+    const gb = groupBy.join(', ');
+    const state = stateFields
+        .map((f) => `${f.name} ${f.columnType} DEFAULT ${f.defaultValue}`)
+        .join(', ');
+    return `CREATE REDUCER ${name} SOURCE ${source} GROUP BY ${gb} STATE (${state}) LANGUAGE EXTERNAL;`;
+}
+function viewToSql(name, source, groupByItems, selectFn) {
+    const groupByCols = [];
+    const intervalDefs = [];
+    for (const item of groupByItems) {
+        if (typeof item === 'string') {
+            groupByCols.push(item);
+        }
+        else if (item._type === 'interval') {
+            const alias = item.alias || `${item.column}_interval`;
+            intervalDefs.push({ column: item.column, seconds: item.seconds, alias });
+            groupByCols.push(alias);
+        }
+    }
+    const keyProxy = new Proxy({}, {
+        get(_target, prop) {
+            return { _type: 'key', column: prop };
+        },
+    });
+    const aggProxy = {
+        key: keyProxy,
+        sum(col) {
+            return { _type: 'agg', func: 'sum', column: col };
+        },
+        count() {
+            return { _type: 'agg', func: 'count', column: null };
+        },
+        first(col) {
+            return { _type: 'agg', func: 'first', column: col };
+        },
+        last(col) {
+            return { _type: 'agg', func: 'last', column: col };
+        },
+        min(col) {
+            return { _type: 'agg', func: 'min', column: col };
+        },
+        max(col) {
+            return { _type: 'agg', func: 'max', column: col };
+        },
+        avg(col) {
+            return { _type: 'agg', func: 'avg', column: col };
+        },
+    };
+    const selectResult = selectFn(aggProxy);
+    const selectItems = [];
+    for (const [alias, expr] of Object.entries(selectResult)) {
+        if (expr._type === 'key') {
+            const intv = intervalDefs.find((d) => d.alias === expr.column);
+            if (intv) {
+                selectItems.push(`toStartOfInterval(${intv.column}, INTERVAL ${intv.seconds} SECOND) AS ${alias}`);
+            }
+            else {
+                selectItems.push(alias === expr.column ? alias : `${expr.column} AS ${alias}`);
+            }
+        }
+        else if (expr._type === 'agg') {
+            const arg = expr.column ? `(${expr.column})` : '()';
+            selectItems.push(`${expr.func}${arg} AS ${alias}`);
+        }
+    }
+    return `CREATE MATERIALIZED VIEW ${name} AS SELECT ${selectItems.join(', ')} FROM ${source} GROUP BY ${groupByCols.join(', ')};`;
+}
+//# sourceMappingURL=ddl.js.map

package/dist/ddl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ddl.js","sourceRoot":"","sources":["../src/ddl.ts"],"names":[],"mappings":";;AAuBA,sCAcC;AAwBD,4BAEC;AA2CD,4CAiCC;AAID,gCAUC;AAED,oCAWC;AAED,8BAsEC;AA3OD,oEAAoE;AAEpE,MAAM,cAAc,GAA2B;IAC7C,MAAM,EAAE,CAAC;IACT,OAAO,EAAE,CAAC;IACV,GAAG,EAAE,CAAC;IACN,CAAC,EAAE,CAAC;IACJ,MAAM,EAAE,EAAE;IACV,OAAO,EAAE,EAAE;IACX,GAAG,EAAE,EAAE;IACP,CAAC,EAAE,EAAE;IACL,IAAI,EAAE,IAAI;IACV,KAAK,EAAE,IAAI;IACX,EAAE,EAAE,IAAI;IACR,CAAC,EAAE,IAAI;IACP,GAAG,EAAE,KAAK;IACV,IAAI,EAAE,KAAK;IACX,CAAC,EAAE,KAAK;CACT,CAAA;AAED,SAAgB,aAAa,CAAC,CAAS;IACrC,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAA;IAC/C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAA;IAC7C,CAAC;IAED,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAChC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAA;IACnC,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAA;IACjC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,GAAG,CAAC,CAAA;IACrD,CAAC;IAED,OAAO,CAAC,GAAG,IAAI,CAAA;AACjB,CAAC;AAYD,SAAS,YAAY,CAAC,MAAc,EAAE,OAAe,EAAE,KAAc;IACnE,OAAO;QACL,KAAK,EAAE,UAAU;QACjB,MAAM;QACN,OAAO;QACP,KAAK;QACL,EAAE,CAAC,CAAS;YACV,OAAO,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAA;QACzC,CAAC;KACF,CAAA;AACH,CAAC;AAED,SAAgB,QAAQ,CAAC,MAAc,EAAE,QAAgB;IACvD,OAAO,YAAY,CAAC,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAA;AACtD,CAAC;AAyCD,oEAAoE;AAEpE,SAAgB,gBAAgB,CAAC,YAAqC;IACpE,MAAM,MAAM,GAAoB,EAAE,CAAA;IAClC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;QACzD,IAAI,UAAkB,CAAA;QACtB,IAAI,YAAoB,CAAA;QACxB,QAAQ,OAAO,KAAK,EAAE,CAAC;YACrB,KAAK,QAAQ;gBACX,UAAU,GAAG,SAAS,CAAA;gBACtB,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;gBAC5B,MAAK;YACP,KAAK,QAAQ;gBACX,UAAU,GAAG,QAAQ,CAAA;gBACrB,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;gBAC5B,MAAK;YACP,KAAK,QAAQ;gBACX,UAAU,GAAG,QAAQ,CAAA;gBACrB,YAAY,GAAG,IAAI,KAAK,GAAG,CAAA;gBAC3B,MAAK;YACP,KAAK,SAAS;gBACZ,UAAU,GAAG,SAAS,CAAA;gBACtB,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAA;gBACvC,MAAK;YACP,KAAK,QAAQ;gBACX,UAAU,GAAG,MAAM,CAAA;gBACnB,YAAY,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAA;gBAC3C,MAAK;YACP;gBACE,UAAU,GAAG,QAAQ,CAAA;gBACrB,YAAY,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAA;QACvC,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC,CAAA;IACjD,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,oEAAoE;AAEpE,SAAgB,UAAU,CACxB,IAAY,EACZ,OAAmC,EACnC,OAAgB;IAEhB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,cAAc,CAAA;IAChE,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SACjC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;SACzC,IAAI,CAAC,IAAI,CAAC,CAAA;IACb,OAAO,GAAG,MAAM,IAAI,IAAI,KAAK,IAAI,IAAI,CAAA;AACvC,CAAC;AAED,SAAgB,YAAY,CAC1B,IAAY,EACZ,MAAc,EACd,OAAiB,EACjB,WAA4B;IAE5B,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC7B,MAAM,KAAK,GAAG,WAAW;SACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,UAAU,YAAY,CAAC,CAAC,YAAY,EAAE,CAAC;SACjE,IAAI,CAAC,IAAI,CAAC,CAAA;IACb,OAAO,kBAAkB,IAAI,WAAW,MAAM,aAAa,EAAE,WAAW,KAAK,sBAAsB,CAAA;AACrG,CAAC;AAED,SAAgB,SAAS,CACvB,IAAY,EACZ,MAAc,EACd,YAA2B,EAC3B,QAAkE;IAElE,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,YAAY,GAAyD,EAAE,CAAA;IAE7E,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxB,CAAC;aAAM,IAAI,IAAI,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YACrC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC,MAAM,WAAW,CAAA;YACrD,YAAY,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;YACxE,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACzB,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,KAAK,CAAC,EAA4B,EAAE;QACvD,GAAG,CAAC,OAAO,EAAE,IAAY;YACvB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACvC,CAAC;KACF,CAAC,CAAA;IAEF,MAAM,QAAQ,GAAkB;QAC9B,GAAG,EAAE,QAAQ;QACb,GAAG,CAAC,GAAW;YACb,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAA;QACnD,CAAC;QACD,KAAK;YACH,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACtD,CAAC;QACD,KAAK,CAAC,GAAW;YACf,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAA;QACrD,CAAC;QACD,IAAI,CAAC,GAAW;YACd,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,CAAA;QACpD,CAAC;QACD,GAAG,CAAC,GAAW;YACb,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAA;QACnD,CAAC;QACD,GAAG,CAAC,GAAW;YACb,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAA;QACnD,CAAC;QACD,GAAG,CAAC,GAAW;YACb,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAA;QACnD,CAAC;KACF,CAAA;IAED,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACvC,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;QACzD,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,CAAA;YAC9D,IAAI,IAAI,EAAE,CAAC;gBACT,WAAW,CAAC,IAAI,CACd,qBAAqB,IAAI,CAAC,MAAM,cAAc,IAAI,CAAC,OAAO,eAAe,KAAK,EAAE,CACjF,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,WAAW,CAAC,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,OAAO,KAAK,EAAE,CAAC,CAAA;YAChF,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAA;YACnD,WAAW,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,GAAG,GAAG,OAAO,KAAK,EAAE,CAAC,CAAA;QACpD,CAAC;IACH,CAAC;IAED,OAAO,4BAA4B,IAAI,cAAc,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,MAAM,aAAa,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAA;AAClI,CAAC"}

package/dist/delta-db.d.ts

@@ -0,0 +1,58 @@
+export interface DeltaDbConfig {
+    schema: string;
+    dataDir?: string;
+    maxBufferSize?: number;
+}
+export interface DeltaDbCursor {
+    number: number;
+    hash: string;
+}
+export type DeltaOperation = 'insert' | 'update' | 'delete';
+export interface DeltaRecord {
+    table: string;
+    operation: DeltaOperation;
+    key: Record<string, any>;
+    values: Record<string, any>;
+    prevValues: Record<string, any> | null;
+}
+export interface DeltaBatch {
+    sequence: number;
+    finalizedHead: DeltaDbCursor | null;
+    latestHead: DeltaDbCursor | null;
+    tables: Record<string, DeltaRecord[]>;
+}
+export interface IngestInput {
+    data: Record<string, Record<string, any>[]>;
+    rollbackChain?: DeltaDbCursor[];
+    finalizedHead: DeltaDbCursor;
+    onDelta?: (batch: DeltaBatch) => void;
+}
+export interface StateFieldDef {
+    name: string;
+    columnType: string;
+    defaultValue: string;
+}
+export interface ExternalReducerOptions<TState = any, TRow = any, TEmit = any> {
+    name: string;
+    source: string;
+    groupBy: string[];
+    state: StateFieldDef[];
+    reduce: (state: TState, row: TRow) => [TState, TEmit | TEmit[] | null];
+}
+export declare class DeltaDb {
+    #private;
+    private constructor();
+    static open(config: DeltaDbConfig): DeltaDb;
+    processBatch(table: string, block: number, rows: Record<string, any>[]): boolean;
+    rollback(forkPoint: number): void;
+    finalize(block: number): void;
+    ingest(input: IngestInput): DeltaBatch | null;
+    resolveForkCursor(previousBlocks: DeltaDbCursor[]): DeltaDbCursor | null;
+    flush(): DeltaBatch | null;
+    ack(sequence: number): void;
+    get pendingCount(): number;
+    get isBackpressured(): boolean;
+    get cursor(): DeltaDbCursor | null;
+    registerReducer<TState = any, TRow = any, TEmit = any>(options: ExternalReducerOptions<TState, TRow, TEmit>): void;
+}
+//# sourceMappingURL=delta-db.d.ts.map

package/dist/delta-db.js

@@ -0,0 +1,88 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DeltaDb = void 0;
+const msgpack_1 = require("@msgpack/msgpack");
+const native_js_1 = require("./native/native.js");
+const encoder = new msgpack_1.Encoder({ useBigInt64: true });
+// ─── DeltaDb class ───────────────────────────────────────────────
+class DeltaDb {
+    #native;
+    constructor(native) {
+        this.#native = native;
+    }
+    static open(config) {
+        return new DeltaDb(native_js_1.DeltaDb.open(config));
+    }
+    processBatch(table, block, rows) {
+        return this.#native.processBatch(table, block, Buffer.from(encoder.encode(rows)));
+    }
+    rollback(forkPoint) {
+        this.#native.rollback(forkPoint);
+    }
+    finalize(block) {
+        this.#native.finalize(block);
+    }
+    ingest(input) {
+        const buf = this.#native.ingest({
+            data: Buffer.from(encoder.encode(input.data)),
+            rollbackChain: input.rollbackChain,
+            finalizedHead: input.finalizedHead,
+        });
+        const batch = buf ? (0, msgpack_1.decode)(buf) : null;
+        if (batch && input.onDelta) {
+            input.onDelta(batch);
+            this.#native.ack(batch.sequence);
+        }
+        return batch;
+    }
+    resolveForkCursor(previousBlocks) {
+        return this.#native.resolveForkCursor(previousBlocks);
+    }
+    flush() {
+        const buf = this.#native.flush();
+        return buf ? (0, msgpack_1.decode)(buf) : null;
+    }
+    ack(sequence) {
+        this.#native.ack(sequence);
+    }
+    get pendingCount() {
+        return this.#native.pendingCount;
+    }
+    get isBackpressured() {
+        return this.#native.isBackpressured;
+    }
+    get cursor() {
+        return this.#native.cursor;
+    }
+    registerReducer(options) {
+        const { reduce } = options;
+        const batchFn = (groups) => {
+            return groups.map(({ state, rows }) => {
+                let s = state;
+                const emits = [];
+                for (const row of rows) {
+                    const [newState, emit] = reduce(s, row);
+                    s = newState;
+                    if (emit != null) {
+                        if (Array.isArray(emit)) {
+                            for (let i = 0; i < emit.length; i++)
+                                emits.push(emit[i]);
+                        }
+                        else {
+                            emits.push(emit);
+                        }
+                    }
+                }
+                return { state: s, emits };
+            });
+        };
+        this.#native.registerReducer({
+            name: options.name,
+            source: options.source,
+            groupBy: options.groupBy,
+            state: options.state,
+        }, batchFn);
+    }
+}
+exports.DeltaDb = DeltaDb;
+//# sourceMappingURL=delta-db.js.map

package/dist/delta-db.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"delta-db.js","sourceRoot":"","sources":["../src/delta-db.ts"],"names":[],"mappings":";;;AAAA,8CAAkD;AAClD,kDAA6D;AAE7D,MAAM,OAAO,GAAG,IAAI,iBAAO,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;AAqDlD,oEAAoE;AAEpE,MAAa,OAAO;IAClB,OAAO,CAAoC;IAE3C,YAAoB,MAA0C;QAC5D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;IACvB,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,MAAqB;QAC/B,OAAO,IAAI,OAAO,CAAC,mBAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;IAChD,CAAC;IAED,YAAY,CAAC,KAAa,EAAE,KAAa,EAAE,IAA2B;QACpE,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnF,CAAC;IAED,QAAQ,CAAC,SAAiB;QACxB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;IAClC,CAAC;IAED,QAAQ,CAAC,KAAa;QACpB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IAC9B,CAAC;IAED,MAAM,CAAC,KAAkB;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAC9B,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC7C,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,aAAa,EAAE,KAAK,CAAC,aAAa;SACnC,CAAC,CAAA;QACF,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAE,IAAA,gBAAM,EAAC,GAAG,CAAgB,CAAC,CAAC,CAAC,IAAI,CAAA;QACtD,IAAI,KAAK,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YAC3B,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YACpB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;QAClC,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,iBAAiB,CAAC,cAA+B;QAC/C,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,cAAc,CAAC,CAAA;IACvD,CAAC;IAED,KAAK;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAA;QAChC,OAAO,GAAG,CAAC,CAAC,CAAE,IAAA,gBAAM,EAAC,GAAG,CAAgB,CAAC,CAAC,CAAC,IAAI,CAAA;IACjD,CAAC;IAED,GAAG,CAAC,QAAgB;QAClB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IAC5B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAA;IAClC,CAAC;IAED,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,OAAO,CAAC,eAAe,CAAA;IACrC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;IAC5B,CAAC;IAED,eAAe,CACb,OAAoD;QAEpD,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;QAE1B,MAAM,OAAO,GAAG,CAAC,MAAyC,EAAE,EAAE;YAC5D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE;gBACpC,IAAI,CAAC,GAAG,KAAK,CAAA;gBACb,MAAM,KAAK,GAAU,EAAE,CAAA;gBACvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;oBACvC,CAAC,GAAG,QAAQ,CAAA;oBACZ,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;wBACjB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;4BACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE;gCAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;wBAC3D,CAAC;6BAAM,CAAC;4BACN,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;wBAClB,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAA;YAC5B,CAAC,CAAC,CAAA;QACJ,CAAC,CAAA;QAED,IAAI,CAAC,OAAO,CAAC,eAAe,CAC1B;YACE,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,EACD,OAAO,CACR,CAAA;IACH,CAAC;CACF;AAhGD,0BAgGC"}

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+export * from './column';
+export { type AggExpr, type AggProxy, type GroupByItem, type IntervalExpr, interval, type KeyRef, type ReducerOptions, type ViewOptions, } from './ddl';
+export { type DeltaBatch, DeltaDb, type DeltaDbConfig, type DeltaDbCursor, type DeltaOperation, type DeltaRecord, type ExternalReducerOptions, type IngestInput, type StateFieldDef, } from './delta-db';
+export { Pipeline, ReducerHandle, TableHandle, ViewHandle } from './pipeline';
+//# sourceMappingURL=index.d.ts.map