@visorcraft/mongreldb 0.18.0

package/README.md

@@ -0,0 +1,68 @@
+# mongreldb-node
+
+Native Node.js addon for MongrelDB via [NAPI](https://napi.rs) — the
+**better-sqlite3 model**: in-process, no HTTP latency, so the ~6 µs single-row
+write isn't dwarfed by a network round-trip. Exposes a **typed object/method
+interface** (not SQL); TypeScript types are generated at build time.
+
+This crate is built **separately** from the Rust workspace (it targets the NAPI
+ABI and needs Node.js tooling). It is excluded from `cargo {test,clippy}
+--workspace`.
+
+## Build
+
+Requires Node.js ≥ 16 and `@napi-rs/cli`:
+
+```bash
+cd crates/mongreldb-node
+npm install
+npm run build          # → mongreldb.<platform>.node + index.d.ts
+```
+
+## API sketch (generated `index.d.ts`)
+
+```ts
+class Database {
+  static withPath(path: string): Database
+  static open(path: string): Database
+  createTable(name: string, schema: SchemaSpec): bigint
+  table(name: string): TableHandle
+  begin(): Transaction
+  sql(sql: string): Promise<Buffer>
+  close(): void
+}
+
+class TableHandle {
+  put(cells: Cell[]): PutResult
+  putBatch(rows: Cell[][]): PutResult[]
+  bulkLoadTyped(columns: TypedColumn[]): bigint
+  commit(): bigint
+  flush(): bigint
+  count(): bigint                        // O(1)
+  countWhere(conditions: ConditionSpec[]): bigint
+  get(rowId: bigint): RowJs | null
+  query(conditions: ConditionSpec[]): RowJs[]   // hybrid: bitmap ∩ range ∩ FM ∩ HNSW
+  queryArrow(conditions: ConditionSpec[]): Buffer
+}
+```
+
+### Hybrid query — the differentiator
+
+`query` intersects row-id sets from any combination of the six index types in a
+single in-process call — something no HTTP vector DB or SQL FTS pipeline can do
+in one hop:
+
+```ts
+db.query([
+  { kind: ConditionKind.Ann, columnId: 5, embedding: queryVec, k: 50 },
+  { kind: ConditionKind.FmContains, columnId: 2, text: "rome" },
+  { kind: ConditionKind.BitmapIn, columnId: 3, values: ["eu", "na"] },
+])
+```
+
+## Notes
+
+- Row ids / counts / epochs cross the FFI as JS `BigInt` (`u64`), so the full
+  64-bit id space is lossless.
+- Most blocking table methods also expose `*Async` Promise variants that run on
+  the NAPI blocking pool.

package/index.d.ts

@@ -0,0 +1,31 @@
+/* Hand-written wrapper around the NAPI-generated native.d.ts binding.
+ * Adds a retryable `Database.prototype.transaction` helper and the
+ * `ConflictError` class.
+ */
+
+export * from './native';
+
+/** Retryable write-write conflict. */
+export declare class ConflictError extends Error {
+  constructor(message: string);
+}
+
+/** Callback used by `Database.prototype.transaction`. */
+export type TransactionCallback = (txn: import('./native').Transaction) => void | Promise<void>;
+
+/** Options for `Database.prototype.transaction`. */
+export interface TransactionOptions {
+  /** Maximum number of conflict retries before giving up. Default: 3. */
+  maxRetries?: number;
+  /** Base backoff delay in milliseconds. Default: 2. */
+  baseDelayMs?: number;
+}
+
+export declare class Database extends import('./native').Database {
+  /**
+   * Run `fn(txn)` inside a cross-table transaction, retrying on conflict.
+   * `fn` may be sync or async; it must stage operations on `txn` but must not
+   * call `commit`/`rollback` itself.
+   */
+  transaction(fn: TransactionCallback, opts?: TransactionOptions): Promise<bigint>;
+}

package/index.js

@@ -0,0 +1,70 @@
+/* eslint-disable */
+/* Hand-written wrapper around the NAPI-generated native.js binding.
+ * Adds a retryable `Database.prototype.transaction` helper and exports the
+ * `ConflictError` class so callers can distinguish retryable write conflicts.
+ */
+
+const native = require('./native.js');
+
+/** Retryable write-write conflict. */
+class ConflictError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'ConflictError';
+  }
+}
+
+const CONFLICT_RE = /^__CONFLICT__:/;
+
+function isConflict(err) {
+  return err != null && typeof err.message === 'string' && CONFLICT_RE.test(err.message);
+}
+
+/** Database wrapper that adds a `transaction(fn, opts?)` retry helper. */
+class Database extends native.Database {
+  /**
+   * Run `fn(txn)` inside a cross-table transaction, retrying on conflict.
+   * `fn` may be sync or async; it must stage operations on `txn` but must not
+   * call `commit`/`rollback` itself.
+   */
+  transaction(fn, opts = {}) {
+    const maxRetries = opts.maxRetries ?? 3;
+    const baseDelayMs = opts.baseDelayMs ?? 2;
+
+    const runOnce = async () => {
+      const txn = this.begin();
+      try {
+        await fn(txn);
+        return txn.commit();
+      } catch (err) {
+        try {
+          txn.rollback();
+        } catch (_) {
+          // ignore rollback errors
+        }
+        throw err;
+      }
+    };
+
+    const attempt = async (retriesLeft) => {
+      try {
+        return await runOnce();
+      } catch (err) {
+        if (retriesLeft > 0 && isConflict(err)) {
+          const delay = baseDelayMs * (maxRetries - retriesLeft + 1);
+          await new Promise((resolve) => setTimeout(resolve, delay));
+          return attempt(retriesLeft - 1);
+        }
+        throw err;
+      }
+    };
+
+    return attempt(maxRetries);
+  }
+}
+
+module.exports = {
+  ...native,
+  Database,
+  ConflictError,
+};

package/native.d.ts

@@ -0,0 +1,426 @@
+/* tslint:disable */
+/* eslint-disable */
+
+/* auto-generated by NAPI-RS */
+
+export const enum ColumnType {
+  Bool = 0,
+  Int64 = 1,
+  Float64 = 2,
+  TimestampNanos = 3,
+  Date32 = 4,
+  Bytes = 5,
+  Embedding = 6
+}
+export const enum IndexKindSpec {
+  Bitmap = 0,
+  FmIndex = 1,
+  Ann = 2,
+  Sparse = 3,
+  MinHash = 4
+}
+export interface ColumnSpec {
+  id: number
+  name: string
+  ty: ColumnType
+  primaryKey: boolean
+  nullable: boolean
+  /** Required when `ty == Embedding`. */
+  embeddingDim?: number
+  /**
+   * Optional default value for schema-evolution back-fill. Currently only
+   * validated; applying defaults to existing rows is not yet implemented.
+   */
+  defaultValue?: Cell
+  /**
+   * Engine-managed monotonic identity allocator for an `Int64` primary key.
+   * When `true`, the engine assigns the next counter value on insert if the
+   * column is omitted/null, and explicit ids advance the counter past them.
+   * Mutually exclusive with `nullable`; requires `primary_key` and `Int64`.
+   * Defaults to `false` (omitted/null).
+   */
+  autoIncrement?: boolean
+  /** Encrypt this column's page payload at rest (requires an encrypted db). */
+  encrypted?: boolean
+  /**
+   * Encrypt the column but keep it queryable via deterministic equality
+   * tokens / order-preserving encoding (requires an encrypted db).
+   */
+  encryptedIndexable?: boolean
+}
+export interface IndexSpec {
+  name: string
+  columnId: number
+  kind: IndexKindSpec
+}
+export interface SchemaSpec {
+  columns: Array<ColumnSpec>
+  indexes: Array<IndexSpec>
+}
+/**
+ * A single cell value. Only the field matching the column type is read; the
+ * rest are ignored. `bytes` is raw; `text` is UTF-8 (use for `Bytes` columns
+ * when the value is a string).
+ */
+export interface Cell {
+  columnId: number
+  int64?: bigint
+  float64?: number
+  boolean?: boolean
+  text?: string
+  bytes?: Buffer
+  embedding?: Array<number>
+  /**
+   * Sparse (SPLADE) token ids, paired with `sparse_weights`; bincoded into a
+   * Bytes value for a `Sparse` column on insert.
+   */
+  sparseTokens?: Array<number>
+  sparseWeights?: Array<number>
+}
+export const enum ConditionKind {
+  Pk = 0,
+  BitmapEq = 1,
+  RangeInt = 2,
+  RangeF64 = 3,
+  FmContains = 4,
+  Ann = 5,
+  PkInt64 = 6,
+  BitmapIn = 7,
+  IsNull = 8,
+  IsNotNull = 9,
+  FmContainsAll = 10,
+  SparseMatch = 11,
+  MinHashSimilar = 12
+}
+/**
+ * One predicate over the shared row-id space. Set the fields appropriate to
+ * `kind`; unused fields are ignored. A `query` takes a conjunction (AND) of
+ * these — the differentiator no SQL FTS pipeline can match.
+ */
+export interface ConditionSpec {
+  kind: ConditionKind
+  columnId: number
+  int64Lo?: bigint
+  int64Hi?: bigint
+  float64Lo?: number
+  float64Hi?: number
+  text?: string
+  values?: Array<string>
+  embedding?: Array<number>
+  k?: number
+  /** SparseMatch query token ids (paired with `sparse_weights`). */
+  sparseTokens?: Array<number>
+  /** SparseMatch query token weights (paired with `sparse_tokens`). */
+  sparseWeights?: Array<number>
+}
+export interface RowJs {
+  rowId: bigint
+  cells: Array<Cell>
+}
+export interface OwnedRowJs {
+  cells: Array<Cell>
+}
+export interface UpsertResultJs {
+  action: string
+  row: OwnedRowJs
+  autoInc?: bigint
+}
+export interface RowUpdateJs {
+  rowId: bigint
+  cells: Array<Cell>
+}
+export interface TxnOpResultJs {
+  kind: string
+  autoInc?: bigint
+  row?: OwnedRowJs
+  rows?: Array<OwnedRowJs>
+  upsert?: UpsertResultJs
+}
+export interface CommitResultJs {
+  epoch: bigint
+  results: Array<TxnOpResultJs>
+}
+/**
+ * Result of an insert: the physical row id plus, when the engine allocated
+ * it, the `AUTO_INCREMENT` value written into the row.
+ */
+export interface PutResult {
+  /** The storage row id (stable physical identity). */
+  rowId: bigint
+  /**
+   * The engine-assigned `AUTO_INCREMENT` value, when the column was omitted
+   * or null and the engine filled it. `null` for tables without an
+   * auto-increment column, or when the caller supplied an explicit value.
+   */
+  autoInc?: bigint
+}
+/** Phase 20.2: a typed column for bulk loading, wrapping JS typed-array data. */
+export interface TypedColumn {
+  columnId: number
+  ty: ColumnType
+  /** Raw data bytes (Int64 = N×8 LE, Float64 = N×8 LE, Bytes = N+1 offsets + values). */
+  data: Buffer
+  /** Optional validity bitmap (1 byte per row, 1=non-null). */
+  validity?: Buffer
+}
+/**
+ * An open MongrelDB multi-table database. Each table is accessible via
+ * `table(name)`; SQL queries via `sql()`.
+ */
+export declare class Database {
+  /** Create a fresh database at `path`. */
+  static withPath(path: string): Database
+  /** Open an existing database from disk. */
+  static open(path: string): Database
+  /** Create a new table with the given schema. */
+  createTable(name: string, schema: SchemaSpec): bigint
+  /** Drop a table by name. */
+  dropTable(name: string): void
+  /**
+   * Rename a live table. Fails if `name` does not exist, if `new_name` is
+   * empty, or if `new_name` is already in use. A no-op when `name == new_name`.
+   */
+  renameTable(name: string, newName: string): void
+  /** Get a handle to a table by name for typed put/get/query operations. */
+  getTable(name: string): TableHandle
+  /** Alias for [`Database::get_table`] matching the spec's `database.table(name)`. */
+  table(name: string): TableHandle
+  /** Begin a cross-table transaction (stage puts/deletes, then `commit`). */
+  begin(): Transaction
+  /** The current reader-visible epoch — the database-wide snapshot point. */
+  snapshotEpoch(): bigint
+  /** List all live table names. */
+  tableNames(): Array<string>
+  /**
+   * Return the column names of a table as it exists in the **database**
+   * (not the code-defined schema). Lets migrations check whether a column
+   * is already present before calling `add_column`.
+   */
+  tableColumns(name: string): Array<string>
+  /**
+   * Add a column to an existing table. The column must be nullable or supply
+   * a default value so existing rows can be evolved safely.
+   */
+  addColumn(table: string, column: ColumnSpec): bigint
+  /**
+   * Alter an existing column's native schema metadata. The source column is
+   * selected by `column_name`; the supplied column spec provides the target
+   * name, storage type, and flags while the stable engine column id is kept.
+   */
+  alterColumn(table: string, columnName: string, column: ColumnSpec): bigint
+  /** Verify database integrity. Returns a JSON-string summary. */
+  check(): string
+  /** Repair/quarantine corrupt tables. Returns a JSON-string summary. */
+  doctor(): string
+  /** Return the path passed to `withPath` / `open`. */
+  directory(): string
+  /** Run a cross-table SQL query. Returns Arrow IPC bytes. */
+  sql(sql: string): Promise<Buffer>
+  /** Flush + release. Optional — the `Database` also drops on GC. */
+  close(): void
+  /**
+   * Create a fresh encrypted database (page-level AES-256-GCM; the database
+   * KEK is derived from `passphrase` via Argon2id + HKDF).
+   */
+  static createEncrypted(path: string, passphrase: string): Database
+  /** Open an existing encrypted database with its passphrase. */
+  static openEncrypted(path: string, passphrase: string): Database
+}
+/** A handle to one table inside a [`Database`]. */
+export declare class TableHandle {
+  /**
+   * Upsert one row. Returns the row id and, when the engine allocated it,
+   * the `AUTO_INCREMENT` value.
+   */
+  put(cells: Array<Cell>): PutResult
+  /** Group-commit this table's pending writes. */
+  commit(): bigint
+  /** Flush: commit + drain memtable to a sorted run. */
+  flush(): bigint
+  /** Live row count (O(1)). */
+  count(): bigint
+  /** Count rows matching a native conjunctive predicate without materializing rows when possible. */
+  countWhere(conditions: Array<ConditionSpec>): bigint
+  /** Point read by row id. */
+  get(rowId: bigint): RowJs | null
+  /** Point read by a text primary key. */
+  getByPkText(text: string): RowJs | null
+  /**
+   * Reserve (without inserting) the next `AUTO_INCREMENT` value for this
+   * table, advancing the engine's in-memory counter. Returns `null` if the
+   * table has no auto-increment column. Used by callers that stage a row with
+   * an explicit id inside a transaction; the reservation becomes durable when
+   * a row carrying the id commits, and a never-used reservation just leaves a
+   * gap. Far cheaper than a Kit-style sequence row — no hot row, no extra
+   * commit.
+   */
+  reserveAutoInc(): bigint | null
+  /** Point read by an Int64 primary key. */
+  getByPkInt64(value: bigint): RowJs | null
+  /** Delete a row by storage row id. */
+  delete(rowId: bigint): void
+  /** Truncate all rows in this table. Call `commit()` to make it durable. */
+  truncate(): void
+  /** Delete the first row matching a text primary key. */
+  deleteByPkText(text: string): void
+  /** Delete the first row matching an Int64 primary key. */
+  deleteByPkInt64(value: bigint): void
+  /** Hybrid index query. */
+  query(conditions: Array<ConditionSpec>): Array<RowJs>
+  /**
+   * Read every row of this table visible at commit `epoch` — a point-in-time
+   * (MVCC time-travel) read. `epoch` must not exceed the current visible
+   * epoch.
+   */
+  rowsAtEpoch(epoch: bigint): Array<RowJs>
+  /**
+   * Reservoir-sampled approximate aggregate (`count`/`sum`/`avg`) with a
+   * `z`-score confidence interval. Returns a JSON object, or `null` when the
+   * reservoir is empty. `columnId` is required for `sum`/`avg`.
+   */
+  approxAggregate(agg: string, columnId: number | undefined | null, z: number): string | null
+  /**
+   * Incrementally-maintained aggregate (`count`/`sum`/`min`/`max`/`avg`),
+   * optionally filtered by pushed-down `conditions`. Returns a JSON object
+   * `{value, incremental, delta_rows}`; the value is always exact. The engine
+   * caches per `(table, column, agg, conditions)` and folds in only the delta
+   * of newly-committed rows once data has spilled to runs.
+   */
+  incrementalAggregate(agg: string, columnId: number | undefined | null, conditions: Array<ConditionSpec>): string
+  /**
+   * Insert a batch of rows in one call. Returns each row's id and, when the
+   * engine allocated it, its `AUTO_INCREMENT` value, in order.
+   */
+  putBatch(rows: Array<Array<Cell>>): Array<PutResult>
+  /**
+   * Fastest ingest path: bulk-load typed columns (Int64/Float64/Bool) in one
+   * shot, bypassing the per-cell `Value` enum. Returns the commit epoch.
+   * Bytes/Embedding columns are not supported here — use `putBatch`.
+   */
+  bulkLoadTyped(columns: Array<TypedColumn>): bigint
+  /**
+   * Hybrid index query returning the matching rows as Arrow IPC bytes
+   * (columnar, zero-copy on the JS side). Requires at least one condition —
+   * use `Database.sql` for full scans.
+   */
+  queryArrow(conditions: Array<ConditionSpec>): Buffer
+  putAsync(cells: Array<Cell>): Promise<PutResult>
+  commitAsync(): Promise<bigint>
+  countAsync(): Promise<bigint>
+  countWhereAsync(conditions: Array<ConditionSpec>): Promise<bigint>
+  getAsync(rowId: bigint): Promise<RowJs | null>
+  queryAsync(conditions: Array<ConditionSpec>): Promise<Array<RowJs>>
+  flushAsync(): Promise<bigint>
+  queryArrowAsync(conditions: Array<ConditionSpec>): Promise<Buffer>
+}
+/**
+ * A cross-table transaction (P5.2). Stage puts/deletes across tables; commit
+ * atomically. On conflict, `commit` throws a `ConflictError`.
+ */
+export declare class Transaction {
+  constructor(db: Database)
+  /**
+   * Stage a put on `table`. Returns the engine-assigned `AUTO_INCREMENT`
+   * value when the table has an auto-increment primary key and the column was
+   * omitted or null; returns `null` otherwise (explicit id, or no auto-inc
+   * column). The returned value is the id that will be written on commit.
+   */
+  put(table: string, cells: Array<Cell>): bigint | null
+  /** Stage a delete of `row_id` on `table`. */
+  delete(table: string, rowId: bigint): void
+  truncate(table: string): void
+  upsert(table: string, insertCells: Array<Cell>, updateCells?: Array<Cell> | undefined | null): void
+  updateMany(table: string, updates: Array<RowUpdateJs>): void
+  deleteMany(table: string, rowIds: Array<bigint>): void
+  /**
+   * Stage many puts on `table` from a compact little-endian buffer, skipping
+   * the per-cell NAPI `Cell` object marshalling that dominates a bulk load.
+   * Layout: `u32 rowCount`, then per row `u16 cellCount`, then per cell
+   * `u16 columnId`, `u8 tag`, payload — tag 0=null, 1=int64 (i64 LE),
+   * 2=float64 (f64 LE), 3=bool (u8), 4=bytes (u32 len + bytes; text is UTF-8).
+   * The decoded `(column_id, Value)` pairs are identical to what `put` builds.
+   */
+  putPacked(table: string, payload: Buffer): void
+  /**
+   * Stage many deletes on `table` from a compact buffer of row ids:
+   * `u32 count`, then `count` × `u64 LE` row id. One NAPI crossing instead
+   * of one per row.
+   */
+  deletePacked(table: string, payload: Buffer): void
+  /**
+   * Commit all staged ops atomically. Returns the commit epoch.
+   * Throws `ConflictError` on write-write conflict (retryable).
+   */
+  commit(): bigint
+  commitReturning(): CommitResultJs
+  /**
+   * Commit off the JS event loop (the durability fsync runs on the NAPI
+   * blocking pool). Throws `ConflictError` on write-write conflict.
+   */
+  commitAsync(): Promise<bigint>
+  /**
+   * Async commit that returns both the committed epoch and the affected row
+   * identities, off the JS event loop. Throws `ConflictError` on write-write
+   * conflict.
+   */
+  commitAsyncReturning(): Promise<CommitResultJs>
+  /** Discard all staged ops. */
+  rollback(): void
+  /**
+   * Scope subsequent ops to `table` via a `TxnTable`, so the table name
+   * isn't repeated on every `put`/`delete`. Pure sugar over the flat
+   * `put(table, …)` / `delete(table, …)` API — the sub-handle stages into
+   * THIS transaction, and this transaction's `commit`/`rollback` still drive
+   * durability.
+   */
+  table(name: string): TxnTable
+}
+/**
+ * A table-scoped view of a [`Transaction`] (returned by `transaction.table()`).
+ * Stages `put`/`delete` for its bound table into the parent transaction; the
+ * parent's `commit`/`rollback` still apply.
+ */
+export declare class TxnTable {
+  /**
+   * Stage a put on this table. Returns the engine-assigned `AUTO_INCREMENT`
+   * value when applicable, or `null` otherwise.
+   */
+  put(cells: Array<Cell>): bigint | null
+  /** Stage a batch of puts on this table. */
+  putBatch(rows: Array<Array<Cell>>): void
+  /** Stage a delete of `row_id` on this table. */
+  delete(rowId: bigint): void
+  truncate(): void
+  upsert(insertCells: Array<Cell>, updateCells?: Array<Cell> | undefined | null): void
+  updateMany(updates: Array<RowUpdateJs>): void
+  deleteMany(rowIds: Array<bigint>): void
+}
+/**
+ * Opt-in write micro-batching over one table. Writes are **not durable until
+ * flush** — the contract is the opposite of `put()`.
+ */
+export declare class WriteBuffer {
+  /** Create a new WriteBuffer over a table. `threshold` rows trigger an auto-flush. */
+  constructor(table: TableHandle, threshold?: number | undefined | null)
+  /**
+   * Buffer a write (not durable until `flush`). Auto-flushes when the
+   * threshold is reached, returning the new row id.
+   */
+  put(cells: Array<Cell>): bigint | null
+  /** Flush all buffered writes (durable after this call). Returns the new epoch. */
+  flush(): bigint
+}
+/**
+ * A daemon-backed database connection (Phase 19.4). Routes put/get/count/
+ * query/commit to a `mongreldb-server` instance over HTTP. Multiple Node
+ * processes can share one warm cache this way.
+ */
+export declare class RemoteDatabase {
+  /** Connect to a `mongreldb-server` at `url` (e.g. `http://127.0.0.1:8453`). */
+  constructor(url: string)
+  health(): string
+  tableNames(): Array<string>
+  count(table: string): number
+  sql(sql: string): Buffer
+  commit(table: string): bigint
+}