@leonardovida-md/drizzle-neo-duckdb 1.0.2 → 1.0.3

package/README.md

@@ -7,7 +7,7 @@
 [![npm version](https://img.shields.io/npm/v/@leonardovida-md/drizzle-neo-duckdb)](https://www.npmjs.com/package/@leonardovida-md/drizzle-neo-duckdb)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 
-[Documentation](./docs) • [Examples](./example) • [Contributing](#contributing)
+[Documentation](https://leonardovida-md.github.io/drizzle-neo-duckdb/) • [Examples](./example) • [Contributing](#contributing)
 
 </div>
 
@@ -119,7 +119,14 @@ Drizzle DuckDB uses `drizzle-orm/pg-core` for schema definitions since DuckDB's
 
 ```typescript
 import { sql } from 'drizzle-orm';
-import { integer, text, boolean, timestamp, pgTable, pgSchema } from 'drizzle-orm/pg-core';
+import {
+  integer,
+  text,
+  boolean,
+  timestamp,
+  pgTable,
+  pgSchema,
+} from 'drizzle-orm/pg-core';
 
 // Tables in default schema
 const posts = pgTable('posts', {
@@ -181,7 +188,7 @@ const products = pgTable('products', {
 });
 ```
 
-See [Column Types Documentation](./docs/columns.md) for complete reference.
+See [Column Types Documentation](https://leonardovida-md.github.io/drizzle-neo-duckdb/api/columns) for complete reference.
 
 ## Querying
 
@@ -189,10 +196,15 @@ All standard Drizzle query methods work:
 
 ```typescript
 // Select
-const users = await db.select().from(usersTable).where(eq(usersTable.active, true));
+const users = await db
+  .select()
+  .from(usersTable)
+  .where(eq(usersTable.active, true));
 
 // Insert
-await db.insert(usersTable).values({ name: 'Alice', email: 'alice@example.com' });
+await db
+  .insert(usersTable)
+  .values({ name: 'Alice', email: 'alice@example.com' });
 
 // Insert with returning
 const inserted = await db
@@ -201,7 +213,10 @@ const inserted = await db
   .returning({ id: usersTable.id });
 
 // Update
-await db.update(usersTable).set({ name: 'Updated' }).where(eq(usersTable.id, 1));
+await db
+  .update(usersTable)
+  .set({ name: 'Updated' })
+  .where(eq(usersTable.id, 1));
 
 // Delete
 await db.delete(usersTable).where(eq(usersTable.id, 1));
@@ -228,7 +243,9 @@ const results = await db
 const results = await db
   .select()
   .from(products)
-  .where(duckDbArrayContained(products.tags, ['electronics', 'sale', 'featured']));
+  .where(
+    duckDbArrayContained(products.tags, ['electronics', 'sale', 'featured'])
+  );
 
 // Check if arrays overlap
 const results = await db
@@ -258,7 +275,7 @@ import { migrate } from '@leonardovida-md/drizzle-neo-duckdb';
 await migrate(db, { migrationsFolder: './drizzle' });
 ```
 
-Migration metadata is stored in `drizzle.__drizzle_migrations` by default. See [Migrations Documentation](./docs/migrations.md) for configuration options.
+Migration metadata is stored in `drizzle.__drizzle_migrations` by default. See [Migrations Documentation](https://leonardovida-md.github.io/drizzle-neo-duckdb/guide/migrations) for configuration options.
 
 ## Schema Introspection
 
@@ -283,7 +300,7 @@ const result = await introspect(db, {
 console.log(result.files.schemaTs);
 ```
 
-See [Introspection Documentation](./docs/introspection.md) for all options.
+See [Introspection Documentation](https://leonardovida-md.github.io/drizzle-neo-duckdb/guide/introspection) for all options.
 
 ## Configuration Options
 
@@ -307,22 +324,23 @@ const db = drizzle(connection, {
 
 This connector aims for compatibility with Drizzle's Postgres driver but has some differences:
 
-| Feature | Status |
-|---------|--------|
-| Basic CRUD operations | Full support |
-| Joins and subqueries | Full support |
-| Transactions | No savepoints (nested transactions reuse outer) |
-| JSON/JSONB columns | Use `duckDbJson()` instead |
-| Prepared statements | No statement caching |
-| Streaming results | Results are materialized |
+| Feature               | Status                                                     |
+| --------------------- | ---------------------------------------------------------- |
+| Basic CRUD operations | Full support                                               |
+| Joins and subqueries  | Full support                                               |
+| Transactions          | No savepoints (nested transactions reuse outer)            |
+| JSON/JSONB columns    | Use `duckDbJson()` instead                                 |
+| Prepared statements   | No statement caching                                       |
+| Streaming results     | Materialized by default; use `executeBatches()` for chunks |
 
-See [Limitations Documentation](./docs/limitations.md) for details.
+See [Limitations Documentation](https://leonardovida-md.github.io/drizzle-neo-duckdb/guide/limitations) for details.
 
 ## Examples
 
 - **[MotherDuck NYC Taxi](./example/motherduck-nyc.ts)** — Query the built-in NYC taxi dataset from MotherDuck cloud
 
 Run examples:
+
 ```bash
 MOTHERDUCK_TOKEN=your_token bun example/motherduck-nyc.ts
 ```

package/dist/client.d.ts

@@ -8,3 +8,15 @@ export interface PrepareParamsOptions {
 export declare function prepareParams(params: unknown[], options?: PrepareParamsOptions): unknown[];
 export declare function closeClientConnection(connection: DuckDBConnection): Promise<void>;
 export declare function executeOnClient(client: DuckDBClientLike, query: string, params: unknown[]): Promise<RowData[]>;
+export interface ExecuteInBatchesOptions {
+    rowsPerChunk?: number;
+}
+/**
+ * Stream results from DuckDB in batches to avoid fully materializing rows in JS.
+ */
+export declare function executeInBatches(client: DuckDBClientLike, query: string, params: unknown[], options?: ExecuteInBatchesOptions): AsyncGenerator<RowData[], void, void>;
+/**
+ * Return columnar results when the underlying node-api exposes an Arrow/columnar API.
+ * Falls back to column-major JS arrays when Arrow is unavailable.
+ */
+export declare function executeArrowOnClient(client: DuckDBClientLike, query: string, params: unknown[]): Promise<unknown>;

package/dist/columns.d.ts

@@ -1,5 +1,6 @@
 import { type SQL } from 'drizzle-orm';
 import type { SQLWrapper } from 'drizzle-orm/sql/sql';
+import { type ListValueWrapper, type ArrayValueWrapper, type MapValueWrapper, type BlobValueWrapper, type JsonValueWrapper } from './value-wrappers.ts';
 type IntColType = 'SMALLINT' | 'INTEGER' | 'BIGINT' | 'HUGEINT' | 'USMALLINT' | 'UINTEGER' | 'UBIGINT' | 'UHUGEINT' | 'INT' | 'INT16' | 'INT32' | 'INT64' | 'INT128' | 'LONG' | 'VARINT';
 type FloatColType = 'FLOAT' | 'DOUBLE';
 type StringColType = 'STRING' | 'VARCHAR' | 'TEXT';
@@ -16,7 +17,7 @@ export declare const duckDbList: <TData = unknown>(name: string, elementType: An
     dataType: "custom";
     columnType: "PgCustomColumn";
     data: TData[];
-    driverParam: string | unknown[] | SQL<unknown>;
+    driverParam: string | ListValueWrapper | unknown[];
     enumValues: undefined;
 }>;
 export declare const duckDbArray: <TData = unknown>(name: string, elementType: AnyColType, fixedLength?: number) => import("drizzle-orm/pg-core").PgCustomColumnBuilder<{
@@ -24,7 +25,7 @@ export declare const duckDbArray: <TData = unknown>(name: string, elementType: A
     dataType: "custom";
     columnType: "PgCustomColumn";
     data: TData[];
-    driverParam: string | unknown[] | SQL<unknown>;
+    driverParam: string | unknown[] | ArrayValueWrapper;
     enumValues: undefined;
 }>;
 export declare const duckDbMap: <TData extends Record<string, any>>(name: string, valueType: AnyColType | ListColType | ArrayColType) => import("drizzle-orm/pg-core").PgCustomColumnBuilder<{
@@ -32,7 +33,7 @@ export declare const duckDbMap: <TData extends Record<string, any>>(name: string
     dataType: "custom";
     columnType: "PgCustomColumn";
     data: TData;
-    driverParam: TData;
+    driverParam: MapValueWrapper | TData;
     enumValues: undefined;
 }>;
 export declare const duckDbStruct: <TData extends Record<string, any>>(name: string, schema: Record<string, Primitive>) => import("drizzle-orm/pg-core").PgCustomColumnBuilder<{
@@ -43,12 +44,19 @@ export declare const duckDbStruct: <TData extends Record<string, any>>(name: str
     driverParam: TData;
     enumValues: undefined;
 }>;
+/**
+ * JSON column type that wraps values and delays JSON.stringify() to binding time.
+ * This ensures consistent handling with other wrapped types.
+ *
+ * Note: DuckDB stores JSON as VARCHAR internally, so the final binding
+ * is always a stringified JSON value.
+ */
 export declare const duckDbJson: <TData = unknown>(name: string) => import("drizzle-orm/pg-core").PgCustomColumnBuilder<{
     name: string;
     dataType: "custom";
     columnType: "PgCustomColumn";
     data: TData;
-    driverParam: string | SQL<unknown>;
+    driverParam: string | JsonValueWrapper | SQL<unknown>;
     enumValues: undefined;
 }>;
 export declare const duckDbBlob: {
@@ -56,24 +64,24 @@ export declare const duckDbBlob: {
         name: "";
         dataType: "custom";
         columnType: "PgCustomColumn";
-        data: Buffer;
-        driverParam: unknown;
+        data: Buffer<ArrayBufferLike>;
+        driverParam: BlobValueWrapper;
         enumValues: undefined;
     }>;
     <TConfig extends Record<string, any>>(fieldConfig?: TConfig | undefined): import("drizzle-orm/pg-core").PgCustomColumnBuilder<{
         name: "";
         dataType: "custom";
         columnType: "PgCustomColumn";
-        data: Buffer;
-        driverParam: unknown;
+        data: Buffer<ArrayBufferLike>;
+        driverParam: BlobValueWrapper;
         enumValues: undefined;
     }>;
     <TName extends string>(dbName: TName, fieldConfig?: unknown): import("drizzle-orm/pg-core").PgCustomColumnBuilder<{
         name: TName;
         dataType: "custom";
         columnType: "PgCustomColumn";
-        data: Buffer;
-        driverParam: unknown;
+        data: Buffer<ArrayBufferLike>;
+        driverParam: BlobValueWrapper;
         enumValues: undefined;
     }>;
 };

package/dist/driver.d.ts

@@ -4,10 +4,12 @@ import { PgDatabase } from 'drizzle-orm/pg-core/db';
 import type { SelectedFields } from 'drizzle-orm/pg-core/query-builders';
 import { type ExtractTablesWithRelations, type RelationalSchemaConfig, type TablesRelationalConfig } from 'drizzle-orm/relations';
 import { type DrizzleConfig } from 'drizzle-orm/utils';
+import type { SQL } from 'drizzle-orm/sql/sql';
 import type { DuckDBClientLike, DuckDBQueryResultHKT, DuckDBTransaction } from './session.ts';
 import { DuckDBSession } from './session.ts';
 import { DuckDBDialect } from './dialect.ts';
 import { DuckDBSelectBuilder } from './select-builder.ts';
+import type { ExecuteInBatchesOptions, RowData } from './client.ts';
 export interface PgDriverOptions {
     logger?: Logger;
     rewriteArrays?: boolean;
@@ -33,5 +35,7 @@ export declare class DuckDBDatabase<TFullSchema extends Record<string, unknown>
     constructor(dialect: DuckDBDialect, session: DuckDBSession<TFullSchema, TSchema>, schema: RelationalSchemaConfig<TSchema> | undefined);
     select(): DuckDBSelectBuilder<undefined>;
     select<TSelection extends SelectedFields>(fields: TSelection): DuckDBSelectBuilder<TSelection>;
+    executeBatches<T extends RowData = RowData>(query: SQL, options?: ExecuteInBatchesOptions): AsyncGenerator<T[], void, void>;
+    executeArrow(query: SQL): Promise<unknown>;
     transaction<T>(transaction: (tx: DuckDBTransaction<TFullSchema, TSchema>) => Promise<T>): Promise<T>;
 }

package/dist/duckdb-introspect.mjs

@@ -311,8 +311,78 @@ import { TransactionRollbackError } from "drizzle-orm/errors";
 
 // src/client.ts
 import {
-  listValue
+  listValue as listValue2,
+  timestampValue as timestampValue2
 } from "@duckdb/node-api";
+
+// src/value-wrappers.ts
+import {
+  listValue,
+  arrayValue,
+  structValue,
+  mapValue,
+  blobValue,
+  timestampValue,
+  timestampTZValue
+} from "@duckdb/node-api";
+var DUCKDB_VALUE_MARKER = Symbol.for("drizzle-duckdb:value");
+function dateToMicros(value) {
+  if (value instanceof Date) {
+    return BigInt(value.getTime()) * 1000n;
+  }
+  let normalized = value;
+  if (!value.includes("T") && value.includes(" ")) {
+    normalized = value.replace(" ", "T");
+  }
+  if (!normalized.endsWith("Z") && !/[+-]\d{2}:?\d{2}$/.test(normalized)) {
+    normalized += "Z";
+  }
+  const date = new Date(normalized);
+  if (isNaN(date.getTime())) {
+    throw new Error(`Invalid timestamp string: ${value}`);
+  }
+  return BigInt(date.getTime()) * 1000n;
+}
+function toUint8Array(data) {
+  return data instanceof Uint8Array && !(data instanceof Buffer) ? data : new Uint8Array(data);
+}
+function convertStructEntries(data, toValue) {
+  const entries = {};
+  for (const [key, val] of Object.entries(data)) {
+    entries[key] = toValue(val);
+  }
+  return entries;
+}
+function convertMapEntries(data, toValue) {
+  return Object.entries(data).map(([key, val]) => ({
+    key,
+    value: toValue(val)
+  }));
+}
+function wrapperToNodeApiValue(wrapper, toValue) {
+  switch (wrapper.kind) {
+    case "list":
+      return listValue(wrapper.data.map(toValue));
+    case "array":
+      return arrayValue(wrapper.data.map(toValue));
+    case "struct":
+      return structValue(convertStructEntries(wrapper.data, toValue));
+    case "map":
+      return mapValue(convertMapEntries(wrapper.data, toValue));
+    case "timestamp":
+      return wrapper.withTimezone ? timestampTZValue(dateToMicros(wrapper.data)) : timestampValue(dateToMicros(wrapper.data));
+    case "blob":
+      return blobValue(toUint8Array(wrapper.data));
+    case "json":
+      return JSON.stringify(wrapper.data);
+    default: {
+      const _exhaustive = wrapper;
+      throw new Error(`Unknown wrapper kind: ${_exhaustive.kind}`);
+    }
+  }
+}
+
+// src/client.ts
 function isPgArrayLiteral(value) {
   return value.startsWith("{") && value.endsWith("}");
 }
@@ -341,30 +411,78 @@ function prepareParams(params, options = {}) {
   });
 }
 function toNodeApiValue(value) {
+  if (value == null)
+    return null;
+  const t = typeof value;
+  if (t === "string" || t === "number" || t === "bigint" || t === "boolean") {
+    return value;
+  }
+  if (t === "object" && DUCKDB_VALUE_MARKER in value) {
+    return wrapperToNodeApiValue(value, toNodeApiValue);
+  }
   if (Array.isArray(value)) {
-    return listValue(value.map((inner) => toNodeApiValue(inner)));
+    return listValue2(value.map((inner) => toNodeApiValue(inner)));
+  }
+  if (value instanceof Date) {
+    return timestampValue2(BigInt(value.getTime()) * 1000n);
   }
   return value;
 }
-async function executeOnClient(client, query, params) {
-  const values = params.length > 0 ? params.map((param) => toNodeApiValue(param)) : undefined;
-  const result = await client.run(query, values);
-  const rows = await result.getRowsJS();
-  const columns = result.columnNames();
+function deduplicateColumns(columns) {
   const seen = {};
-  const uniqueColumns = columns.map((col) => {
+  return columns.map((col) => {
     const count = seen[col] ?? 0;
     seen[col] = count + 1;
     return count === 0 ? col : `${col}_${count}`;
   });
-  return (rows ?? []).map((vals) => {
+}
+function mapRowsToObjects(columns, rows) {
+  return rows.map((vals) => {
     const obj = {};
-    uniqueColumns.forEach((col, idx) => {
+    columns.forEach((col, idx) => {
       obj[col] = vals[idx];
     });
     return obj;
   });
 }
+async function executeOnClient(client, query, params) {
+  const values = params.length > 0 ? params.map((param) => toNodeApiValue(param)) : undefined;
+  const result = await client.run(query, values);
+  const rows = await result.getRowsJS();
+  const columns = result.deduplicatedColumnNames?.() ?? result.columnNames();
+  const uniqueColumns = deduplicateColumns(columns);
+  return rows ? mapRowsToObjects(uniqueColumns, rows) : [];
+}
+async function* executeInBatches(client, query, params, options = {}) {
+  const rowsPerChunk = options.rowsPerChunk && options.rowsPerChunk > 0 ? options.rowsPerChunk : 1e5;
+  const values = params.length > 0 ? params.map((param) => toNodeApiValue(param)) : undefined;
+  const result = await client.stream(query, values);
+  const columns = result.deduplicatedColumnNames?.() ?? result.columnNames();
+  const uniqueColumns = deduplicateColumns(columns);
+  let buffer = [];
+  for await (const chunk of result.yieldRowsJs()) {
+    const objects = mapRowsToObjects(uniqueColumns, chunk);
+    for (const row of objects) {
+      buffer.push(row);
+      if (buffer.length >= rowsPerChunk) {
+        yield buffer;
+        buffer = [];
+      }
+    }
+  }
+  if (buffer.length > 0) {
+    yield buffer;
+  }
+}
+async function executeArrowOnClient(client, query, params) {
+  const values = params.length > 0 ? params.map((param) => toNodeApiValue(param)) : undefined;
+  const result = await client.run(query, values);
+  const maybeArrow = result.toArrow ?? result.getArrowTable;
+  if (typeof maybeArrow === "function") {
+    return await maybeArrow.call(result);
+  }
+  return result.getColumnsObjectJS();
+}
 
 // src/session.ts
 class DuckDBPreparedQuery extends PgPreparedQuery {
@@ -405,11 +523,7 @@ class DuckDBPreparedQuery extends PgPreparedQuery {
       this.logger.logQuery(`[duckdb] original query before array rewrite: ${this.queryString}`, params);
     }
     this.logger.logQuery(rewrittenQuery, params);
-    const {
-      fields,
-      joinsNotNullableMap,
-      customResultMapper
-    } = this;
+    const { fields, joinsNotNullableMap, customResultMapper } = this;
     const rows = await executeOnClient(this.client, rewrittenQuery, params);
     if (rows.length === 0 || !fields) {
       return rows;
@@ -469,6 +583,32 @@ class DuckDBSession extends PgSession {
     this.logger.logQuery(`[duckdb] ${arrayLiteralWarning}
 query: ${query}`, []);
   };
+  executeBatches(query, options = {}) {
+    const builtQuery = this.dialect.sqlToQuery(query);
+    const params = prepareParams(builtQuery.params, {
+      rejectStringArrayLiterals: this.rejectStringArrayLiterals,
+      warnOnStringArrayLiteral: this.rejectStringArrayLiterals ? undefined : () => this.warnOnStringArrayLiteral(builtQuery.sql)
+    });
+    const rewrittenQuery = this.rewriteArrays ? adaptArrayOperators(builtQuery.sql) : builtQuery.sql;
+    if (this.rewriteArrays && rewrittenQuery !== builtQuery.sql) {
+      this.logger.logQuery(`[duckdb] original query before array rewrite: ${builtQuery.sql}`, params);
+    }
+    this.logger.logQuery(rewrittenQuery, params);
+    return executeInBatches(this.client, rewrittenQuery, params, options);
+  }
+  async executeArrow(query) {
+    const builtQuery = this.dialect.sqlToQuery(query);
+    const params = prepareParams(builtQuery.params, {
+      rejectStringArrayLiterals: this.rejectStringArrayLiterals,
+      warnOnStringArrayLiteral: this.rejectStringArrayLiterals ? undefined : () => this.warnOnStringArrayLiteral(builtQuery.sql)
+    });
+    const rewrittenQuery = this.rewriteArrays ? adaptArrayOperators(builtQuery.sql) : builtQuery.sql;
+    if (this.rewriteArrays && rewrittenQuery !== builtQuery.sql) {
+      this.logger.logQuery(`[duckdb] original query before array rewrite: ${builtQuery.sql}`, params);
+    }
+    this.logger.logQuery(rewrittenQuery, params);
+    return executeArrowOnClient(this.client, rewrittenQuery, params);
+  }
 }
 
 class DuckDBTransaction extends PgTransaction {
@@ -492,6 +632,12 @@ class DuckDBTransaction extends PgTransaction {
   setTransaction(config) {
     return this.session.execute(sql`set transaction ${this.getTransactionConfigSQL(config)}`);
   }
+  executeBatches(query, options = {}) {
+    return this.session.executeBatches(query, options);
+  }
+  executeArrow(query) {
+    return this.session.executeArrow(query);
+  }
   async transaction(transaction) {
     const nestedTx = new DuckDBTransaction(this.dialect, this.session, this.schema, this.nestedIndex + 1);
     return transaction(nestedTx);
@@ -590,13 +736,7 @@ import { PgViewBase } from "drizzle-orm/pg-core/view-base";
 import { SQL as SQL4 } from "drizzle-orm/sql/sql";
 
 // src/sql/selection.ts
-import {
-  Column as Column2,
-  SQL as SQL3,
-  getTableName as getTableName2,
-  is as is3,
-  sql as sql3
-} from "drizzle-orm";
+import { Column as Column2, SQL as SQL3, getTableName as getTableName2, is as is3, sql as sql3 } from "drizzle-orm";
 function mapEntries(obj, prefix, fullJoin = false) {
   return Object.fromEntries(Object.entries(obj).filter(([key]) => key !== "enableRLS").map(([key, value]) => {
     const qualified = prefix ? `${prefix}.${key}` : key;
@@ -737,6 +877,12 @@ class DuckDBDatabase extends PgDatabase {
       dialect: this.dialect
     });
   }
+  executeBatches(query, options = {}) {
+    return this.session.executeBatches(query, options);
+  }
+  executeArrow(query) {
+    return this.session.executeArrow(query);
+  }
   async transaction(transaction) {
     return await this.session.transaction(transaction);
   }
@@ -1046,7 +1192,9 @@ function mapDuckDbType(column, imports, options) {
   }
   if (upper === "BIGINT" || upper === "INT8" || upper === "UBIGINT") {
     imports.pgCore.add("bigint");
-    return { builder: `bigint(${columnName(column.name)})` };
+    return {
+      builder: `bigint(${columnName(column.name)}, { mode: 'number' })`
+    };
   }
   const decimalMatch = /^DECIMAL\((\d+),(\d+)\)/i.exec(upper);
   const numericMatch = /^NUMERIC\((\d+),(\d+)\)/i.exec(upper);

package/dist/index.d.ts

@@ -3,3 +3,6 @@ export * from './session.ts';
 export * from './columns.ts';
 export * from './migrator.ts';
 export * from './introspect.ts';
+export * from './client.ts';
+export * from './olap.ts';
+export * from './value-wrappers.ts';