@leonardovida-md/drizzle-neo-duckdb 1.0.2 → 1.1.0

package/dist/introspect.d.ts

@@ -61,5 +61,6 @@ export interface IntrospectResult {
         relationsTs?: string;
     };
 }
+export declare const DEFAULT_IMPORT_BASE = "@leonardovida-md/drizzle-neo-duckdb/helpers";
 export declare function introspect(db: DuckDBDatabase, opts?: IntrospectOptions): Promise<IntrospectResult>;
 export {};

package/dist/olap.d.ts

@@ -0,0 +1,46 @@
+import { Subquery, type SQLWrapper } from 'drizzle-orm';
+import type { AnyPgColumn, PgTable } from 'drizzle-orm/pg-core';
+import type { PgViewBase } from 'drizzle-orm/pg-core/view-base';
+import { SQL } from 'drizzle-orm/sql/sql';
+import type { DuckDBDatabase } from './driver.ts';
+export declare const countN: (expr?: SQLWrapper) => SQL<number>;
+export declare const sumN: (expr: SQLWrapper) => SQL<number>;
+export declare const avgN: (expr: SQLWrapper) => SQL<number>;
+export declare const sumDistinctN: (expr: SQLWrapper) => SQL<number>;
+export declare const percentileCont: (p: number, expr: SQLWrapper) => SQL<number>;
+export declare const median: (expr: SQLWrapper) => SQL<number>;
+export declare const anyValue: <T = unknown>(expr: SQLWrapper) => SQL<T>;
+type PartitionOrder = {
+    partitionBy?: SQLWrapper | SQLWrapper[];
+    orderBy?: SQLWrapper | SQLWrapper[];
+} | undefined;
+export declare const rowNumber: (options?: PartitionOrder) => SQL<number>;
+export declare const rank: (options?: PartitionOrder) => SQL<number>;
+export declare const denseRank: (options?: PartitionOrder) => SQL<number>;
+export declare const lag: <T = unknown>(expr: SQLWrapper, offset?: number, defaultValue?: SQLWrapper, options?: PartitionOrder) => SQL<T>;
+export declare const lead: <T = unknown>(expr: SQLWrapper, offset?: number, defaultValue?: SQLWrapper, options?: PartitionOrder) => SQL<T>;
+type ValueExpr = SQL | SQL.Aliased | AnyPgColumn;
+type GroupKey = ValueExpr;
+type MeasureMap = Record<string, ValueExpr>;
+type NonAggMap = Record<string, ValueExpr>;
+export declare class OlapBuilder {
+    private db;
+    private source?;
+    private keys;
+    private measureMap;
+    private nonAggregates;
+    private wrapNonAggWithAnyValue;
+    private orderByClauses;
+    constructor(db: DuckDBDatabase);
+    from(source: PgTable | Subquery | PgViewBase | SQL): this;
+    groupBy(keys: GroupKey[]): this;
+    measures(measures: MeasureMap): this;
+    selectNonAggregates(fields: NonAggMap, options?: {
+        anyValue?: boolean;
+    }): this;
+    orderBy(...clauses: ValueExpr[]): this;
+    build(): any;
+    run(): any;
+}
+export declare const olap: (db: DuckDBDatabase) => OlapBuilder;
+export {};

package/dist/pool.d.ts

@@ -0,0 +1,22 @@
+import { DuckDBInstance } from '@duckdb/node-api';
+import { type DuckDBConnectionPool } from './client.ts';
+/** Pool size presets for different MotherDuck instance types */
+export type PoolPreset = 'pulse' | 'standard' | 'jumbo' | 'mega' | 'giga' | 'local' | 'memory';
+/** Pool sizes optimized for each MotherDuck instance type */
+export declare const POOL_PRESETS: Record<PoolPreset, number>;
+export interface DuckDBPoolConfig {
+    /** Maximum concurrent connections. Defaults to 4. */
+    size?: number;
+}
+/**
+ * Resolve pool configuration to a concrete size.
+ * Returns false if pooling is disabled.
+ */
+export declare function resolvePoolSize(pool: DuckDBPoolConfig | PoolPreset | false | undefined): number | false;
+export interface DuckDBConnectionPoolOptions {
+    /** Maximum concurrent connections. Defaults to 4. */
+    size?: number;
+}
+export declare function createDuckDBConnectionPool(instance: DuckDBInstance, options?: DuckDBConnectionPoolOptions): DuckDBConnectionPool & {
+    size: number;
+};

package/dist/session.d.ts

@@ -9,6 +9,7 @@ import { type Query, SQL } from 'drizzle-orm/sql/sql';
 import type { Assume } from 'drizzle-orm/utils';
 import type { DuckDBDialect } from './dialect.ts';
 import type { DuckDBClientLike, RowData } from './client.ts';
+import { type ExecuteInBatchesOptions } from './client.ts';
 export type { DuckDBClientLike, RowData } from './client.ts';
 export declare class DuckDBPreparedQuery<T extends PreparedQueryConfig> extends PgPreparedQuery<T> {
     private client;
@@ -47,12 +48,16 @@ export declare class DuckDBSession<TFullSchema extends Record<string, unknown> =
     prepareQuery<T extends PreparedQueryConfig = PreparedQueryConfig>(query: Query, fields: SelectedFieldsOrdered | undefined, name: string | undefined, isResponseInArrayMode: boolean, customResultMapper?: (rows: unknown[][]) => T['execute']): PgPreparedQuery<T>;
     transaction<T>(transaction: (tx: DuckDBTransaction<TFullSchema, TSchema>) => Promise<T>): Promise<T>;
     private warnOnStringArrayLiteral;
+    executeBatches<T extends RowData = RowData>(query: SQL, options?: ExecuteInBatchesOptions): AsyncGenerator<GenericRowData<T>[], void, void>;
+    executeArrow(query: SQL): Promise<unknown>;
 }
 export declare class DuckDBTransaction<TFullSchema extends Record<string, unknown>, TSchema extends TablesRelationalConfig> extends PgTransaction<DuckDBQueryResultHKT, TFullSchema, TSchema> {
     static readonly [entityKind]: string;
     rollback(): never;
     getTransactionConfigSQL(config: PgTransactionConfig): SQL;
     setTransaction(config: PgTransactionConfig): Promise<void>;
+    executeBatches<T extends RowData = RowData>(query: SQL, options?: ExecuteInBatchesOptions): AsyncGenerator<GenericRowData<T>[], void, void>;
+    executeArrow(query: SQL): Promise<unknown>;
     transaction<T>(transaction: (tx: DuckDBTransaction<TFullSchema, TSchema>) => Promise<T>): Promise<T>;
 }
 export type GenericRowData<T extends RowData = RowData> = T;

package/dist/sql/query-rewriters.d.ts

@@ -1,2 +1 @@
 export declare function adaptArrayOperators(query: string): string;
-export declare function queryAdapter(query: string): string;

package/dist/utils.d.ts

@@ -1,3 +1,3 @@
 export { aliasFields } from './sql/selection.ts';
-export { adaptArrayOperators, queryAdapter } from './sql/query-rewriters.ts';
+export { adaptArrayOperators } from './sql/query-rewriters.ts';
 export { mapResultRow } from './sql/result-mapper.ts';

package/dist/value-wrappers-core.d.ts

@@ -0,0 +1,42 @@
+/**
+ * DuckDB wrapper value helpers that are safe for client-side bundles.
+ * These utilities only tag values; conversion to native bindings lives
+ * in value-wrappers.ts to avoid pulling @duckdb/node-api into browsers.
+ */
+export declare const DUCKDB_VALUE_MARKER: unique symbol;
+export type DuckDBValueKind = 'list' | 'array' | 'struct' | 'map' | 'timestamp' | 'blob' | 'json';
+export interface DuckDBValueWrapper<TKind extends DuckDBValueKind = DuckDBValueKind, TData = unknown> {
+    readonly [DUCKDB_VALUE_MARKER]: true;
+    readonly kind: TKind;
+    readonly data: TData;
+}
+export interface ListValueWrapper extends DuckDBValueWrapper<'list', unknown[]> {
+    readonly elementType?: string;
+}
+export interface ArrayValueWrapper extends DuckDBValueWrapper<'array', unknown[]> {
+    readonly elementType?: string;
+    readonly fixedLength?: number;
+}
+export interface StructValueWrapper extends DuckDBValueWrapper<'struct', Record<string, unknown>> {
+    readonly schema?: Record<string, string>;
+}
+export interface MapValueWrapper extends DuckDBValueWrapper<'map', Record<string, unknown>> {
+    readonly valueType?: string;
+}
+export interface TimestampValueWrapper extends DuckDBValueWrapper<'timestamp', Date | string> {
+    readonly withTimezone: boolean;
+    readonly precision?: number;
+}
+export interface BlobValueWrapper extends DuckDBValueWrapper<'blob', Buffer | Uint8Array> {
+}
+export interface JsonValueWrapper extends DuckDBValueWrapper<'json', unknown> {
+}
+export type AnyDuckDBValueWrapper = ListValueWrapper | ArrayValueWrapper | StructValueWrapper | MapValueWrapper | TimestampValueWrapper | BlobValueWrapper | JsonValueWrapper;
+export declare function isDuckDBWrapper(value: unknown): value is AnyDuckDBValueWrapper;
+export declare function wrapList(data: unknown[], elementType?: string): ListValueWrapper;
+export declare function wrapArray(data: unknown[], elementType?: string, fixedLength?: number): ArrayValueWrapper;
+export declare function wrapStruct(data: Record<string, unknown>, schema?: Record<string, string>): StructValueWrapper;
+export declare function wrapMap(data: Record<string, unknown>, valueType?: string): MapValueWrapper;
+export declare function wrapTimestamp(data: Date | string, withTimezone: boolean, precision?: number): TimestampValueWrapper;
+export declare function wrapBlob(data: Buffer | Uint8Array): BlobValueWrapper;
+export declare function wrapJson(data: unknown): JsonValueWrapper;

package/dist/value-wrappers.d.ts

@@ -0,0 +1,8 @@
+import { type DuckDBValue } from '@duckdb/node-api';
+import { type AnyDuckDBValueWrapper } from './value-wrappers-core.ts';
+/**
+ * Convert a wrapper to a DuckDB Node API value.
+ * Uses exhaustive switch for compile-time safety.
+ */
+export declare function wrapperToNodeApiValue(wrapper: AnyDuckDBValueWrapper, toValue: (v: unknown) => DuckDBValue): DuckDBValue;
+export { DUCKDB_VALUE_MARKER, isDuckDBWrapper, wrapArray, wrapBlob, wrapJson, wrapList, wrapMap, wrapStruct, wrapTimestamp, type AnyDuckDBValueWrapper, type DuckDBValueWrapper, type ArrayValueWrapper, type BlobValueWrapper, type JsonValueWrapper, type ListValueWrapper, type MapValueWrapper, type StructValueWrapper, type TimestampValueWrapper, type DuckDBValueKind, } from './value-wrappers-core.ts';

package/package.json

@@ -3,14 +3,17 @@
   "module": "./dist/index.mjs",
   "main": "./dist/index.mjs",
   "types": "./dist/index.d.ts",
-  "version": "1.0.2",
+  "version": "1.1.0",
   "description": "A drizzle ORM client for use with DuckDB. Based on drizzle's Postgres client.",
   "type": "module",
   "scripts": {
-    "build": "bun build --target=node ./src/index.ts --outfile=./dist/index.mjs --packages=external && bun build --target=node ./src/bin/duckdb-introspect.ts --outfile=./dist/duckdb-introspect.mjs --packages=external && bun run build:declarations",
+    "build": "bun build --target=node ./src/index.ts --outfile=./dist/index.mjs --packages=external && bun build --target=node ./src/helpers.ts --outfile=./dist/helpers.mjs --packages=external && bun build --target=node ./src/bin/duckdb-introspect.ts --outfile=./dist/duckdb-introspect.mjs --packages=external && bun run build:declarations",
     "build:declarations": "tsc --emitDeclarationOnly --project tsconfig.types.json",
     "test": "vitest",
-    "t": "vitest --watch --ui"
+    "t": "vitest --watch --ui",
+    "bench": "vitest bench --runInBand test/perf",
+    "perf:run": "bun run scripts/run-perf.ts",
+    "perf:compare": "bun run scripts/compare-perf.ts"
   },
   "bin": {
     "duckdb-introspect": "dist/duckdb-introspect.mjs"
@@ -33,7 +36,8 @@
     "prettier": "^3.5.3",
     "typescript": "^5.8.2",
     "uuid": "^10.0.0",
-    "vitest": "^1.6.0"
+    "vitest": "^1.6.0",
+    "tinybench": "^2.7.1"
   },
   "repository": {
     "type": "git",
@@ -44,6 +48,10 @@
       "types": "./dist/index.d.ts",
       "import": "./dist/index.mjs"
     },
+    "./helpers": {
+      "types": "./dist/helpers.d.ts",
+      "import": "./dist/helpers.mjs"
+    },
     "./package.json": "./package.json"
   },
   "sideEffects": false,

package/src/bin/duckdb-introspect.ts

@@ -40,11 +40,17 @@ function parseArgs(argv: string[]): CliOptions {
         break;
       case '--schema':
       case '--schemas':
-        options.schemas = argv[++i]?.split(',').map((s) => s.trim()).filter(Boolean);
+        options.schemas = argv[++i]
+          ?.split(',')
+          .map((s) => s.trim())
+          .filter(Boolean);
         break;
       case '--out':
       case '--outFile':
-        options.outFile = path.resolve(process.cwd(), argv[++i] ?? 'drizzle/schema.ts');
+        options.outFile = path.resolve(
+          process.cwd(),
+          argv[++i] ?? 'drizzle/schema.ts'
+        );
         break;
       case '--include-views':
       case '--includeViews':
@@ -133,7 +139,10 @@ async function main() {
 
     console.log(`Wrote schema to ${options.outFile}`);
   } finally {
-    if ('closeSync' in connection && typeof connection.closeSync === 'function') {
+    if (
+      'closeSync' in connection &&
+      typeof connection.closeSync === 'function'
+    ) {
       connection.closeSync();
     }
   }

package/src/client.ts

@@ -1,12 +1,30 @@
 import {
   listValue,
+  timestampValue,
   type DuckDBConnection,
   type DuckDBValue,
 } from '@duckdb/node-api';
+import {
+  DUCKDB_VALUE_MARKER,
+  wrapperToNodeApiValue,
+  type AnyDuckDBValueWrapper,
+} from './value-wrappers.ts';
 
-export type DuckDBClientLike = DuckDBConnection;
+export type DuckDBClientLike = DuckDBConnection | DuckDBConnectionPool;
 export type RowData = Record<string, unknown>;
 
+export interface DuckDBConnectionPool {
+  acquire(): Promise<DuckDBConnection>;
+  release(connection: DuckDBConnection): void | Promise<void>;
+  close?(): Promise<void> | void;
+}
+
+export function isPool(
+  client: DuckDBClientLike
+): client is DuckDBConnectionPool {
+  return typeof (client as DuckDBConnectionPool).acquire === 'function';
+}
+
 export interface PrepareParamsOptions {
   rejectStringArrayLiterals?: boolean;
   warnOnStringArrayLiteral?: () => void;
@@ -26,8 +44,6 @@ function parsePgArrayLiteral(value: string): unknown {
   }
 }
 
-let warnedArrayLiteral = false;
-
 export function prepareParams(
   params: unknown[],
   options: PrepareParamsOptions = {}
@@ -40,8 +56,7 @@ export function prepareParams(
         );
       }
 
-      if (!warnedArrayLiteral && options.warnOnStringArrayLiteral) {
-        warnedArrayLiteral = true;
+      if (options.warnOnStringArrayLiteral) {
         options.warnOnStringArrayLiteral();
       }
       return parsePgArrayLiteral(param);
@@ -50,13 +65,62 @@ export function prepareParams(
   });
 }
 
+/**
+ * Convert a value to DuckDB Node API value.
+ * Handles wrapper types and plain values for backward compatibility.
+ * Optimized for the common case (primitives) in the hot path.
+ */
 function toNodeApiValue(value: unknown): DuckDBValue {
+  // Fast path 1: null/undefined
+  if (value == null) return null;
+
+  // Fast path 2: primitives (most common)
+  const t = typeof value;
+  if (t === 'string' || t === 'number' || t === 'bigint' || t === 'boolean') {
+    return value as DuckDBValue;
+  }
+
+  // Fast path 3: pre-wrapped DuckDB value (Symbol check ~2-3ns)
+  if (t === 'object' && DUCKDB_VALUE_MARKER in (value as object)) {
+    return wrapperToNodeApiValue(
+      value as AnyDuckDBValueWrapper,
+      toNodeApiValue
+    );
+  }
+
+  // Legacy path: plain arrays (backward compatibility)
   if (Array.isArray(value)) {
     return listValue(value.map((inner) => toNodeApiValue(inner)));
   }
+
+  // Date conversion to timestamp
+  if (value instanceof Date) {
+    return timestampValue(BigInt(value.getTime()) * 1000n);
+  }
+
+  // Fallback for unknown objects
   return value as DuckDBValue;
 }
 
+function deduplicateColumns(columns: string[]): string[] {
+  const seen: Record<string, number> = {};
+  return columns.map((col) => {
+    const count = seen[col] ?? 0;
+    seen[col] = count + 1;
+    return count === 0 ? col : `${col}_${count}`;
+  });
+}
+
+function mapRowsToObjects(columns: string[], rows: unknown[][]): RowData[] {
+  return rows.map((vals) => {
+    const obj: Record<string, unknown> = {};
+    columns.forEach((col, idx) => {
+      obj[col] = vals[idx];
+    });
+    return obj;
+  }) as RowData[];
+}
+
 export async function closeClientConnection(
   connection: DuckDBConnection
 ): Promise<void> {
@@ -65,10 +129,7 @@ export async function closeClientConnection(
     return;
   }
 
-  if (
-    'closeSync' in connection &&
-    typeof connection.closeSync === 'function'
-  ) {
+  if ('closeSync' in connection && typeof connection.closeSync === 'function') {
     connection.closeSync();
     return;
   }
@@ -86,25 +147,119 @@ export async function executeOnClient(
   query: string,
   params: unknown[]
 ): Promise<RowData[]> {
+  if (isPool(client)) {
+    const connection = await client.acquire();
+    try {
+      return await executeOnClient(connection, query, params);
+    } finally {
+      await client.release(connection);
+    }
+  }
+
   const values =
     params.length > 0
       ? (params.map((param) => toNodeApiValue(param)) as DuckDBValue[])
       : undefined;
   const result = await client.run(query, values);
   const rows = await result.getRowsJS();
-  const columns = result.columnNames();
-  const seen: Record<string, number> = {};
-  const uniqueColumns = columns.map((col) => {
-    const count = seen[col] ?? 0;
-    seen[col] = count + 1;
-    return count === 0 ? col : `${col}_${count}`;
-  });
+  const columns =
+    // prefer deduplicated names when available (Node API >=1.4.2)
+    result.deduplicatedColumnNames?.() ?? result.columnNames();
+  const uniqueColumns = deduplicateColumns(columns);
 
-  return (rows ?? []).map((vals) => {
-    const obj: Record<string, unknown> = {};
-    uniqueColumns.forEach((col, idx) => {
-      obj[col] = vals[idx];
-    });
-    return obj;
-  }) as RowData[];
+  return rows ? mapRowsToObjects(uniqueColumns, rows) : [];
+}
+
+export interface ExecuteInBatchesOptions {
+  rowsPerChunk?: number;
+}
+
+/**
+ * Stream results from DuckDB in batches to avoid fully materializing rows in JS.
+ */
+export async function* executeInBatches(
+  client: DuckDBClientLike,
+  query: string,
+  params: unknown[],
+  options: ExecuteInBatchesOptions = {}
+): AsyncGenerator<RowData[], void, void> {
+  if (isPool(client)) {
+    const connection = await client.acquire();
+    try {
+      yield* executeInBatches(connection, query, params, options);
+      return;
+    } finally {
+      await client.release(connection);
+    }
+  }
+
+  const rowsPerChunk =
+    options.rowsPerChunk && options.rowsPerChunk > 0
+      ? options.rowsPerChunk
+      : 100_000;
+  const values =
+    params.length > 0
+      ? (params.map((param) => toNodeApiValue(param)) as DuckDBValue[])
+      : undefined;
+
+  const result = await client.stream(query, values);
+  const columns =
+    // prefer deduplicated names when available (Node API >=1.4.2)
+    result.deduplicatedColumnNames?.() ?? result.columnNames();
+  const uniqueColumns = deduplicateColumns(columns);
+
+  let buffer: RowData[] = [];
+
+  for await (const chunk of result.yieldRowsJs()) {
+    const objects = mapRowsToObjects(uniqueColumns, chunk);
+    for (const row of objects) {
+      buffer.push(row);
+      if (buffer.length >= rowsPerChunk) {
+        yield buffer;
+        buffer = [];
+      }
+    }
+  }
+
+  if (buffer.length > 0) {
+    yield buffer;
+  }
+}
+
+/**
+ * Return columnar results when the underlying node-api exposes an Arrow/columnar API.
+ * Falls back to column-major JS arrays when Arrow is unavailable.
+ */
+export async function executeArrowOnClient(
+  client: DuckDBClientLike,
+  query: string,
+  params: unknown[]
+): Promise<unknown> {
+  if (isPool(client)) {
+    const connection = await client.acquire();
+    try {
+      return await executeArrowOnClient(connection, query, params);
+    } finally {
+      await client.release(connection);
+    }
+  }
+
+  const values =
+    params.length > 0
+      ? (params.map((param) => toNodeApiValue(param)) as DuckDBValue[])
+      : undefined;
+  const result = await client.run(query, values);
+
+  // Runtime detection for Arrow API support (optional method, not in base type)
+  const maybeArrow =
+    (result as unknown as { toArrow?: () => Promise<unknown> }).toArrow ??
+    (result as unknown as { getArrowTable?: () => Promise<unknown> })
+      .getArrowTable;
+
+  if (typeof maybeArrow === 'function') {
+    return await maybeArrow.call(result);
+  }
+
+  // Fallback: return column-major JS arrays to avoid per-row object creation.
+  return result.getColumnsObjectJS();
 }