@stoolap/node 0.4.1 → 0.5.0-beta.1

package/README.md

@@ -97,7 +97,8 @@ const db = await Database.open('file:///absolute/path/to/db');
 | `exec(sql)` | `Promise<void>` | Execute a DDL statement |
 | `query(sql, params?)` | `Promise<Object[]>` | Query rows as objects |
 | `queryOne(sql, params?)` | `Promise<Object \| null>` | Query single row |
-| `queryRaw(sql, params?)` | `Promise<{columns, rows}>` | Query in columnar format |
+| `queryRaw(sql, params?)` | `Promise<{columns, rows}>` | Query as arrays of values |
+| `iterate(sql, params?)` | `RowCursor` | Cursor over rows, `for await` or `for...of` |
 | `begin()` | `Promise<Transaction>` | Begin a transaction |
 | `close()` | `Promise<void>` | Close the database |
 
@@ -113,7 +114,8 @@ Sync methods run on the main thread. Faster for simple operations but block the
 | `execSync(sql)` | `void` | Execute a DDL statement |
 | `querySync(sql, params?)` | `Object[]` | Query rows as objects |
 | `queryOneSync(sql, params?)` | `Object \| null` | Query single row |
-| `queryRawSync(sql, params?)` | `{columns, rows}` | Query in columnar format |
+| `queryRawSync(sql, params?)` | `{columns, rows}` | Query as arrays of values |
+| `iterate(sql, params?)` | `RowCursor` | Cursor over rows, fetched in chunks |
 | `executeBatchSync(sql, paramsArray)` | `RunResult` | Execute with multiple param sets |
 | `beginSync()` | `Transaction` | Begin a transaction |
 | `prepare(sql)` | `PreparedStatement` | Create a prepared statement |
@@ -185,7 +187,9 @@ Controls the durability vs. performance trade-off:
 | `wal_compression` | `on` | LZ4 compression for WAL entries |
 | `volume_compression` | `on` | LZ4 compression for cold volume files |
 | `compression` | `on` | Shorthand: set both `wal_compression` and `volume_compression` |
-| `keep_snapshots` | `5` | Number of backup snapshot files to retain |
+| `keep_snapshots` | `3` | Number of backup snapshot files to retain |
+| `hot_max_rows` | `262144` | Committed hot rows per table that trigger an early seal (`0` = off) |
+| `hot_max_bytes` | `0` | Hot row bytes per table above which commits wait for a seal (`0` = off) |
 
 #### Cloning
 
@@ -207,7 +211,7 @@ await db.close();
 
 #### Raw Query Format
 
-`queryRaw` / `queryRawSync` return `{ columns: string[], rows: any[][] }` instead of an array of objects. Faster when you don't need named keys.
+`queryRaw` / `queryRawSync` return `{ columns: string[], rows: any[][] }`, one array per row, instead of an array of objects. Faster when you don't need named keys.
 
 ```js
 const raw = db.queryRawSync('SELECT id, name, email FROM users ORDER BY id');
@@ -215,6 +219,25 @@ console.log(raw.columns); // ['id', 'name', 'email']
 console.log(raw.rows);    // [[1, 'Alice', 'alice@example.com'], [2, 'Bob', 'bob@example.com']]
 ```
 
+#### Iterating Rows
+
+`iterate` opens a cursor and fetches rows in chunks (32 rows, doubling to 256) as the loop advances, so a large result never has to be materialized at once. The same cursor works with `for...of` and `for await...of`, and it closes itself when the loop ends, breaks or throws. `Database`, `Transaction` and `PreparedStatement` all have it.
+
+```js
+for (const user of db.iterate('SELECT * FROM users WHERE active = $1', [true])) {
+  if (user.balance < 0) break; // the cursor is closed on break
+}
+
+for await (const row of db.iterate('SELECT * FROM events WHERE day = :day', { day: '2026-10-21' })) {
+  await ship(row);
+}
+
+const stmt = db.prepare('SELECT * FROM users WHERE age > :min');
+for (const user of stmt.iterate({ min: 30 })) { /* ... */ }
+```
+
+A cursor opened on a transaction is closed by `commit` and `rollback`; one opened on a database is closed with it. Call `cursor.close()` to end an iteration early without breaking out of a loop.
+
 #### Batch Execution
 
 Execute the same SQL with multiple parameter sets in a single call. Automatically wraps in a transaction.
@@ -254,7 +277,8 @@ All methods mirror `Database` but without the `sql` parameter (it's bound at pre
 | `execute(params?)` | `executeSync(params?)` | Execute DML statement |
 | `query(params?)` | `querySync(params?)` | Query rows as objects |
 | `queryOne(params?)` | `queryOneSync(params?)` | Query single row |
-| `queryRaw(params?)` | `queryRawSync(params?)` | Query in columnar format |
+| `queryRaw(params?)` | `queryRawSync(params?)` | Query as arrays of values |
+| `iterate(params?)` | — | Cursor over rows, `for await` or `for...of` |
 | | `executeBatchSync(paramsArray)` | Execute with multiple param sets |
 | | `finalize()` | Release the prepared statement |
 
@@ -303,7 +327,8 @@ console.log(result.changes); // 3
 | `execute(sql, params?)` | `executeSync(sql, params?)` | Execute DML statement |
 | `query(sql, params?)` | `querySync(sql, params?)` | Query rows as objects |
 | `queryOne(sql, params?)` | `queryOneSync(sql, params?)` | Query single row |
-| `queryRaw(sql, params?)` | `queryRawSync(sql, params?)` | Query in columnar format |
+| `queryRaw(sql, params?)` | `queryRawSync(sql, params?)` | Query as arrays of values |
+| `iterate(sql, params?)` | — | Cursor over rows, closed by commit/rollback |
 | `commit()` | `commitSync()` | Commit the transaction |
 | `rollback()` | `rollbackSync()` | Rollback the transaction |
 | | `executeBatchSync(sql, paramsArray)` | Execute with multiple param sets |
@@ -377,6 +402,8 @@ db.querySync(
 );
 ```
 
+Synchronous ad hoc statements pass named parameters to the engine as they are: the SQL is not rewritten, a `:name` inside a string literal is left alone, a name used twice is bound once, keys the SQL does not mention are ignored and a name the object lacks binds `NULL`. A prepared statement, and the asynchronous ad hoc methods, resolve `:name` placeholders in the driver and bind by position, which is the engine's fastest path.
+
 ### Error Handling
 
 All methods throw on errors (invalid SQL, constraint violations, etc.):

package/index.d.ts

@@ -88,7 +88,10 @@ export declare class Database {
    * Auto-wraps in a transaction: begin, execute all, commit.
    * Returns { changes: total_rows_affected }.
    */
-  executeBatchSync(sql: string, paramsArray: any[][]): RunResult;
+  executeBatchSync(
+    sql: string,
+    paramsArray: any[][] | Record<string, any>[],
+  ): RunResult;
   /** Create a prepared statement (synchronous). */
   prepare(sql: string): PreparedStatement;
   /** Begin a transaction. Returns Promise<Transaction>. */
@@ -97,11 +100,33 @@ export declare class Database {
   beginSync(): Transaction;
   /** Close the database. Returns Promise<void>. */
   close(): Promise<void>;
+  /**
+   * Open a cursor over the query's rows. Iterate it with `for...of` or
+   * `for await...of`; rows are fetched in chunks as the loop advances and
+   * the cursor closes itself when the loop ends, breaks or throws.
+   */
+  iterate(sql: string, params?: any[] | Record<string, any>): RowCursor;
   /** Close the database synchronously. */
   closeSync(): void;
 }
 export type JsDatabase = Database;
 
+/**
+ * A cursor over an open result set, from `iterate()`. Iterable both
+ * synchronously and with `for await`. Rows cross the native boundary in
+ * chunks of 32 to 256 rows.
+ */
+export declare class RowCursor
+  implements Iterable<Record<string, any>>, AsyncIterable<Record<string, any>>
+{
+  [Symbol.iterator](): Iterator<Record<string, any>>;
+  [Symbol.asyncIterator](): AsyncIterator<Record<string, any>>;
+  /** End the iteration early and release the result set. */
+  close(): void;
+  /** True once the cursor was closed or its result set was exhausted. */
+  get closed(): boolean;
+}
+
 export declare class PreparedStatement {
   /** Execute the statement (DML). Returns Promise<{ changes: number }>. */
   execute(params?: any[] | Record<string, any>): Promise<RunResult>;
@@ -132,7 +157,9 @@ export declare class PreparedStatement {
    * Execute the prepared SQL with multiple param sets in a single call.
    * Returns { changes: total_rows_affected }.
    */
-  executeBatchSync(paramsArray: any[][]): RunResult;
+  executeBatchSync(paramsArray: any[][] | Record<string, any>[]): RunResult;
+  /** Open a cursor over the statement's rows; see `Database.iterate`. */
+  iterate(params?: any[] | Record<string, any>): RowCursor;
   /** Get the SQL text of this prepared statement. */
   get sql(): string;
   /** Release the prepared statement. */
@@ -188,7 +215,15 @@ export declare class Transaction {
    * Execute the same SQL with multiple param sets in a single call.
    * Returns { changes: total_rows_affected }.
    */
-  executeBatchSync(sql: string, paramsArray: any[][]): RunResult;
+  executeBatchSync(
+    sql: string,
+    paramsArray: any[][] | Record<string, any>[],
+  ): RunResult;
+  /**
+   * Open a cursor over the query's rows inside the transaction; see
+   * `Database.iterate`. Commit and rollback close open cursors.
+   */
+  iterate(sql: string, params?: any[] | Record<string, any>): RowCursor;
   /** Rollback the transaction synchronously. */
   rollbackSync(): void;
 }

package/index.js

@@ -3,11 +3,13 @@
 const { Database } = require("./lib/database");
 const { Transaction } = require("./lib/transaction");
 const { PreparedStatement } = require("./lib/statement");
+const { RowCursor } = require("./lib/iterator");
 
-module.exports = { Database, Transaction, PreparedStatement };
+module.exports = { Database, Transaction, PreparedStatement, RowCursor };
 module.exports.Database = Database;
 module.exports.PreparedStatement = PreparedStatement;
 module.exports.Transaction = Transaction;
+module.exports.RowCursor = RowCursor;
 // Legacy aliases
 module.exports.JsDatabase = Database;
 module.exports.JsPreparedStatement = PreparedStatement;

package/lib/database.js

@@ -1,9 +1,15 @@
 "use strict";
 
-const { loadStoolap, processParams, rewriteNamedParams } = require("./ffi");
+const {
+  loadStoolap,
+  processParams,
+  rewriteNamedParams,
+  splitNamed,
+} = require("./ffi");
 const { parseRows, parseRaw } = require("./parse");
 const { Transaction } = require("./transaction");
 const { PreparedStatement, encodeBatch } = require("./statement");
+const { RowCursor } = require("./iterator");
 
 function normalizeDsn(dbPath) {
   let dsn = dbPath || ":memory:";
@@ -25,6 +31,7 @@ class Database {
   #closeWaiter = null;
   #closingPromise = null;
   #children = new Set();
+  #cursors = new Set(); /* WeakRef<RowCursor> */
 
   constructor(ptr, stoolap) {
     this.#ptr = ptr;
@@ -39,13 +46,33 @@ class Database {
     this.#children.delete(child);
   }
 
+  _trackCursor(ref) {
+    this.#cursors.add(ref);
+  }
+
+  _untrackCursor(ref) {
+    this.#cursors.delete(ref);
+  }
+
   #closeChildren() {
+    for (const ref of this.#cursors) {
+      const cursor = ref.deref();
+      if (cursor) cursor._invalidate();
+    }
+    this.#cursors.clear();
     for (const child of this.#children) {
       child._invalidate();
     }
     this.#children.clear();
   }
 
+  #childrenPending() {
+    for (const child of this.#children) {
+      if (child._hasPending && child._hasPending()) return true;
+    }
+    return false;
+  }
+
   static async open(dbPath) {
     const stoolap = loadStoolap();
     const ptr = await stoolap.dbOpenAsync(normalizeDsn(dbPath));
@@ -123,6 +150,16 @@ class Database {
     );
   }
 
+
+  _rowsChunk(rowsPtr, maxRows, buf) {
+    this.#checkOpen();
+    return this.#stoolap.rowsChunk(rowsPtr, maxRows, buf);
+  }
+
+  _rowsChunkAsync(rowsPtr, maxRows) {
+    return this.#runAsync(() => this.#stoolap.rowsChunkAsync(rowsPtr, maxRows));
+  }
+
   _executeBatchBufferSync(sql, batchBuf) {
     this.#checkOpen();
     return this.#stoolap.dbExecBatchBuf(this.#ptr, sql, batchBuf);
@@ -152,19 +189,13 @@ class Database {
 
   executeSync(sql, params) {
     this.#checkOpen();
-    if (!params) return this.#stoolap.dbExec(this.#ptr, sql);
-    if (Array.isArray(params)) {
-      if (params.length === 0) return this.#stoolap.dbExec(this.#ptr, sql);
-      return this.#stoolap.dbExec(this.#ptr, sql, params);
+    if (params && !Array.isArray(params)) {
+      const [names, values] = splitNamed(params);
+      return this.#stoolap.dbExecNamed(this.#ptr, sql, names, values);
     }
-    const p = processParams(sql, params);
-    return p.values
-      ? this.#stoolap.dbExec(this.#ptr, p.sql, p.values)
-      : this.#stoolap.dbExec(this.#ptr, sql);
+    return this.#stoolap.dbExec(this.#ptr, sql, params);
   }
 
-  // -- Exec (DDL) --
-
   async exec(sql) {
     await this._execAsync(sql);
   }
@@ -194,27 +225,21 @@ class Database {
     return parseRows(buf);
   }
 
+  /* One FFI call. mode: 0 = rows, 1 = first row, 2 = raw buffer */
+  #queryAll(sql, params, mode) {
+    if (params && !Array.isArray(params)) {
+      const [names, values] = splitNamed(params);
+      return this.#stoolap.dbQueryNamedAll(this.#ptr, null, sql, names, values, mode);
+    }
+    return this.#stoolap.dbQueryAll(this.#ptr, null, sql, params, mode);
+  }
+
   querySync(sql, params) {
     this.#checkOpen();
-    let result;
-    if (!params) {
-      result = this.#stoolap.dbQuery(this.#ptr, sql);
-    } else if (Array.isArray(params)) {
-      result =
-        params.length === 0
-          ? this.#stoolap.dbQuery(this.#ptr, sql)
-          : this.#stoolap.dbQuery(this.#ptr, sql, params);
-    } else {
-      const p = processParams(sql, params);
-      result = p.values
-        ? this.#stoolap.dbQuery(this.#ptr, p.sql, p.values)
-        : this.#stoolap.dbQuery(this.#ptr, sql);
-    }
+    const result = this.#queryAll(sql, params, 0);
     return Array.isArray(result) ? result : parseRows(result);
   }
 
-  // -- QueryOne --
-
   async queryOne(sql, params) {
     const rows = await this.query(sql, params);
     return rows.length === 0 ? null : rows[0];
@@ -222,19 +247,14 @@ class Database {
 
   queryOneSync(sql, params) {
     this.#checkOpen();
-    if (!params) return this.#stoolap.dbQueryOne(this.#ptr, sql);
-    if (Array.isArray(params)) {
-      if (params.length === 0) return this.#stoolap.dbQueryOne(this.#ptr, sql);
-      return this.#stoolap.dbQueryOne(this.#ptr, sql, params);
+    const result = this.#queryAll(sql, params, 1);
+    if (result instanceof ArrayBuffer) {
+      const rows = parseRows(result);
+      return rows.length === 0 ? null : rows[0];
     }
-    const p = processParams(sql, params);
-    return p.values
-      ? this.#stoolap.dbQueryOne(this.#ptr, p.sql, p.values)
-      : this.#stoolap.dbQueryOne(this.#ptr, sql);
+    return result;
   }
 
-  // -- QueryRaw --
-
   async queryRaw(sql, params) {
     let buf;
     if (!params) {
@@ -255,25 +275,25 @@ class Database {
 
   queryRawSync(sql, params) {
     this.#checkOpen();
-    let buf;
-    if (!params) {
-      buf = this.#stoolap.dbQueryBuf(this.#ptr, sql);
-    } else if (Array.isArray(params)) {
-      buf =
-        params.length === 0
-          ? this.#stoolap.dbQueryBuf(this.#ptr, sql)
-          : this.#stoolap.dbQueryBuf(this.#ptr, sql, params);
+    return parseRaw(this.#queryAll(sql, params, 2));
+  }
+
+  /**
+   * Open a cursor over the query's rows. Iterate it with `for...of` or
+   * `for await...of`; rows are fetched in chunks as the loop advances.
+   */
+  iterate(sql, params) {
+    this.#checkOpen();
+    let ptr;
+    if (params && !Array.isArray(params)) {
+      const [names, values] = splitNamed(params);
+      ptr = this.#stoolap.rowsOpenNamed(this.#ptr, null, sql, names, values);
     } else {
-      const p = processParams(sql, params);
-      buf = p.values
-        ? this.#stoolap.dbQueryBuf(this.#ptr, p.sql, p.values)
-        : this.#stoolap.dbQueryBuf(this.#ptr, sql);
+      ptr = this.#stoolap.rowsOpen(this.#ptr, null, sql, params);
     }
-    return parseRaw(buf);
+    return new RowCursor(this.#stoolap, ptr, this);
   }
 
-  // -- Batch --
-
   executeBatchSync(sql, paramsArray) {
     this.#checkOpen();
     if (paramsArray.length > 0 && Array.isArray(paramsArray[0])) {
@@ -298,6 +318,7 @@ class Database {
 
   prepare(sql) {
     this.#checkOpen();
+    if (typeof sql !== "string") throw new TypeError("sql must be a string");
     const prepared = rewriteNamedParams(sql);
     const [stmtPtr, queryOneIntBound] = this.#stoolap.dbPrepare(
       this.#ptr,
@@ -309,7 +330,6 @@ class Database {
       queryOneIntBound,
       this,
       sql,
-      prepared.sql,
       prepared.names,
     );
     this._trackChild(stmt);
@@ -338,30 +358,35 @@ class Database {
   async close() {
     if (this.#closingPromise) return this.#closingPromise;
     if (!this.#ptr) return;
-
-    this.#closeChildren();
     const ptr = this.#ptr;
     this.#ptr = null;
-
-    const wait =
+    /* let queued work on the handle, its statements and its transactions
+     * land before anything is released */
+    const settling = [];
+    for (const child of this.#children) {
+      if (child._settle) settling.push(child._settle());
+    }
+    const wait = Promise.all(settling).then(() =>
       this.#pendingAsync === 0
-        ? Promise.resolve()
+        ? undefined
         : new Promise((resolve) => {
             this.#closeWaiter = resolve;
-          });
-
+          }),
+    );
     this.#closingPromise = wait
-      .then(() => this.#stoolap.dbCloseAsync(ptr))
+      .then(() => {
+        this.#closeChildren();
+        return this.#stoolap.dbCloseAsync(ptr);
+      })
       .finally(() => {
         this.#closingPromise = null;
         this.#closeWaiter = null;
       });
-
     return this.#closingPromise;
   }
 
   closeSync() {
-    if (this.#closingPromise || this.#pendingAsync > 0) {
+    if (this.#closingPromise || this.#pendingAsync > 0 || this.#childrenPending()) {
       throw new Error("Database has pending async operations");
     }
     if (this.#ptr) {

package/lib/ffi.js

@@ -53,6 +53,7 @@ function loadStoolap() {
 
   _stoolap = require("../build/Release/stoolap.node");
   _stoolap.loadLibrary(findLibraryPath());
+  _stoolap.setRowFactoryMaker(require("./parse").makeRowFactory);
   return _stoolap;
 }
 
@@ -161,6 +162,9 @@ function processParams(sql, params) {
   return { sql: rewritten, values };
 }
 
+/* Rewrite :name placeholders to $n for a prepared statement. The engine binds
+ * prepared statements faster by position than by name, so a statement keeps
+ * its parameter names and binds an object's values positionally. */
 function rewriteNamedParams(sql) {
   const seen = new Map();
   const names = [];
@@ -171,13 +175,21 @@ function rewriteNamedParams(sql) {
     }
     return `$${seen.get(name)}`;
   });
-
   return names.length === 0 ? { sql, names: null } : { sql: rewritten, names };
 }
 
+/* Split a named-parameter object into parallel name and value arrays. */
+function splitNamed(params) {
+  const names = Object.keys(params);
+  const values = new Array(names.length);
+  for (let i = 0; i < names.length; i++) values[i] = params[names[i]];
+  return [names, values];
+}
+
 module.exports = {
   loadStoolap,
   processParams,
   rewriteNamedParams,
+  splitNamed,
   findLibraryPath,
 };

package/lib/iterator.js

@@ -0,0 +1,111 @@
+"use strict";
+
+const { parseRows, parseHeader, makeRowFactory, parseRowsCached } = require("./parse");
+
+const FIRST_CHUNK = 32;
+const MAX_CHUNK = 256;
+
+/**
+ * A row cursor over an open result set. Rows cross the FFI boundary in
+ * packed chunks (32 rows, doubling to 256), so a scan costs one call per
+ * chunk instead of one per row. Iterate it synchronously or with
+ * `for await`; the cursor closes itself when iteration ends, breaks or
+ * throws, and `close()` ends it early.
+ */
+class RowCursor {
+  #stoolap = null;
+  #ptr = null;
+  #owner = null;
+  #chunk = FIRST_CHUNK;
+  #buf = null;
+  #meta = null;
+  #factory = null;
+  #pending = false;
+  #closeWanted = false;
+  #ref = null;
+
+  constructor(stoolap, ptr, owner) {
+    this.#stoolap = stoolap;
+    this.#ptr = ptr;
+    this.#owner = owner;
+    /* the owner holds only a weak reference: an abandoned cursor can be
+     * collected, and the native handle's finalizer closes it */
+    this.#ref = new WeakRef(this);
+    owner._trackCursor(this.#ref);
+  }
+
+  #release() {
+    if (this.#ptr) {
+      this.#stoolap.rowsClose(this.#ptr);
+      this.#ptr = null;
+    }
+    this.#closeWanted = false;
+    this.#owner._untrackCursor(this.#ref);
+  }
+
+  /** Called by the owner when it closes. A chunk in flight closes the cursor when it lands. */
+  _invalidate() {
+    if (this.#pending) this.#closeWanted = true;
+    else this.#release();
+  }
+
+  close() {
+    if (this.#pending) this.#closeWanted = true;
+    else this.#release();
+  }
+
+  get closed() {
+    return this.#ptr === null;
+  }
+
+  #parse(buf) {
+    if (!this.#meta) {
+      this.#meta = parseHeader(buf);
+      this.#factory = makeRowFactory(this.#meta.colNames);
+    }
+    const rows = this.#factory
+      ? parseRowsCached(buf, this.#meta.headerOff, this.#factory, this.#meta.colCount)
+      : parseRows(buf);
+    if (this.#chunk < MAX_CHUNK) this.#chunk *= 2;
+    return rows;
+  }
+
+  *[Symbol.iterator]() {
+    try {
+      while (this.#ptr) {
+        const buf = this.#owner._rowsChunk(this.#ptr, this.#chunk, this.#buf);
+        if (!buf) break;
+        this.#buf = buf;
+        const rows = this.#parse(buf);
+        for (let i = 0; i < rows.length; i++) yield rows[i];
+      }
+    } finally {
+      this.close();
+    }
+  }
+
+  async *[Symbol.asyncIterator]() {
+    try {
+      while (this.#ptr) {
+        this.#pending = true;
+        let buf;
+        try {
+          buf = await this.#owner._rowsChunkAsync(this.#ptr, this.#chunk);
+        } finally {
+          this.#pending = false;
+        }
+        if (this.#closeWanted) {
+          this.#release();
+          break;
+        }
+        if (!buf) break;
+        const rows = this.#parse(buf);
+        for (let i = 0; i < rows.length; i++) yield rows[i];
+      }
+    } finally {
+      this.close();
+    }
+  }
+}
+
+module.exports = { RowCursor };