@leonardovida-md/drizzle-neo-duckdb 1.0.1 → 1.0.3

package/dist/olap.d.ts

@@ -0,0 +1,46 @@
+import { Subquery, type SQLWrapper } from 'drizzle-orm';
+import type { AnyPgColumn, PgTable } from 'drizzle-orm/pg-core';
+import type { PgViewBase } from 'drizzle-orm/pg-core/view-base';
+import { SQL } from 'drizzle-orm/sql/sql';
+import type { DuckDBDatabase } from './driver.ts';
+export declare const countN: (expr?: SQLWrapper) => SQL<number>;
+export declare const sumN: (expr: SQLWrapper) => SQL<number>;
+export declare const avgN: (expr: SQLWrapper) => SQL<number>;
+export declare const sumDistinctN: (expr: SQLWrapper) => SQL<number>;
+export declare const percentileCont: (p: number, expr: SQLWrapper) => SQL<number>;
+export declare const median: (expr: SQLWrapper) => SQL<number>;
+export declare const anyValue: <T = unknown>(expr: SQLWrapper) => SQL<T>;
+type PartitionOrder = {
+    partitionBy?: SQLWrapper | SQLWrapper[];
+    orderBy?: SQLWrapper | SQLWrapper[];
+} | undefined;
+export declare const rowNumber: (options?: PartitionOrder) => SQL<number>;
+export declare const rank: (options?: PartitionOrder) => SQL<number>;
+export declare const denseRank: (options?: PartitionOrder) => SQL<number>;
+export declare const lag: <T = unknown>(expr: SQLWrapper, offset?: number, defaultValue?: SQLWrapper, options?: PartitionOrder) => SQL<T>;
+export declare const lead: <T = unknown>(expr: SQLWrapper, offset?: number, defaultValue?: SQLWrapper, options?: PartitionOrder) => SQL<T>;
+type ValueExpr = SQL | SQL.Aliased | AnyPgColumn;
+type GroupKey = ValueExpr;
+type MeasureMap = Record<string, ValueExpr>;
+type NonAggMap = Record<string, ValueExpr>;
+export declare class OlapBuilder {
+    private db;
+    private source?;
+    private keys;
+    private measureMap;
+    private nonAggregates;
+    private wrapNonAggWithAnyValue;
+    private orderByClauses;
+    constructor(db: DuckDBDatabase);
+    from(source: PgTable | Subquery | PgViewBase | SQL): this;
+    groupBy(keys: GroupKey[]): this;
+    measures(measures: MeasureMap): this;
+    selectNonAggregates(fields: NonAggMap, options?: {
+        anyValue?: boolean;
+    }): this;
+    orderBy(...clauses: ValueExpr[]): this;
+    build(): any;
+    run(): any;
+}
+export declare const olap: (db: DuckDBDatabase) => OlapBuilder;
+export {};

package/dist/session.d.ts

@@ -9,6 +9,7 @@ import { type Query, SQL } from 'drizzle-orm/sql/sql';
 import type { Assume } from 'drizzle-orm/utils';
 import type { DuckDBDialect } from './dialect.ts';
 import type { DuckDBClientLike, RowData } from './client.ts';
+import { type ExecuteInBatchesOptions } from './client.ts';
 export type { DuckDBClientLike, RowData } from './client.ts';
 export declare class DuckDBPreparedQuery<T extends PreparedQueryConfig> extends PgPreparedQuery<T> {
     private client;
@@ -47,12 +48,16 @@ export declare class DuckDBSession<TFullSchema extends Record<string, unknown> =
     prepareQuery<T extends PreparedQueryConfig = PreparedQueryConfig>(query: Query, fields: SelectedFieldsOrdered | undefined, name: string | undefined, isResponseInArrayMode: boolean, customResultMapper?: (rows: unknown[][]) => T['execute']): PgPreparedQuery<T>;
     transaction<T>(transaction: (tx: DuckDBTransaction<TFullSchema, TSchema>) => Promise<T>): Promise<T>;
     private warnOnStringArrayLiteral;
+    executeBatches<T extends RowData = RowData>(query: SQL, options?: ExecuteInBatchesOptions): AsyncGenerator<GenericRowData<T>[], void, void>;
+    executeArrow(query: SQL): Promise<unknown>;
 }
 export declare class DuckDBTransaction<TFullSchema extends Record<string, unknown>, TSchema extends TablesRelationalConfig> extends PgTransaction<DuckDBQueryResultHKT, TFullSchema, TSchema> {
     static readonly [entityKind]: string;
     rollback(): never;
     getTransactionConfigSQL(config: PgTransactionConfig): SQL;
     setTransaction(config: PgTransactionConfig): Promise<void>;
+    executeBatches<T extends RowData = RowData>(query: SQL, options?: ExecuteInBatchesOptions): AsyncGenerator<GenericRowData<T>[], void, void>;
+    executeArrow(query: SQL): Promise<unknown>;
     transaction<T>(transaction: (tx: DuckDBTransaction<TFullSchema, TSchema>) => Promise<T>): Promise<T>;
 }
 export type GenericRowData<T extends RowData = RowData> = T;

package/dist/value-wrappers.d.ts

@@ -0,0 +1,104 @@
+import { type DuckDBValue } from '@duckdb/node-api';
+/**
+ * Symbol used to identify wrapped DuckDB values for native binding.
+ * Uses Symbol.for() to ensure cross-module compatibility.
+ */
+export declare const DUCKDB_VALUE_MARKER: unique symbol;
+/**
+ * Type identifier for each wrapper kind.
+ */
+export type DuckDBValueKind = 'list' | 'array' | 'struct' | 'map' | 'timestamp' | 'blob' | 'json';
+/**
+ * Base interface for all tagged DuckDB value wrappers.
+ */
+export interface DuckDBValueWrapper<TKind extends DuckDBValueKind = DuckDBValueKind, TData = unknown> {
+    readonly [DUCKDB_VALUE_MARKER]: true;
+    readonly kind: TKind;
+    readonly data: TData;
+}
+/**
+ * List wrapper - maps to DuckDBListValue
+ */
+export interface ListValueWrapper extends DuckDBValueWrapper<'list', unknown[]> {
+    readonly elementType?: string;
+}
+/**
+ * Array wrapper (fixed size) - maps to DuckDBArrayValue
+ */
+export interface ArrayValueWrapper extends DuckDBValueWrapper<'array', unknown[]> {
+    readonly elementType?: string;
+    readonly fixedLength?: number;
+}
+/**
+ * Struct wrapper - maps to DuckDBStructValue
+ */
+export interface StructValueWrapper extends DuckDBValueWrapper<'struct', Record<string, unknown>> {
+    readonly schema?: Record<string, string>;
+}
+/**
+ * Map wrapper - maps to DuckDBMapValue
+ */
+export interface MapValueWrapper extends DuckDBValueWrapper<'map', Record<string, unknown>> {
+    readonly valueType?: string;
+}
+/**
+ * Timestamp wrapper - maps to DuckDBTimestampValue or DuckDBTimestampTZValue
+ */
+export interface TimestampValueWrapper extends DuckDBValueWrapper<'timestamp', Date | string> {
+    readonly withTimezone: boolean;
+    readonly precision?: number;
+}
+/**
+ * Blob wrapper - maps to DuckDBBlobValue
+ */
+export interface BlobValueWrapper extends DuckDBValueWrapper<'blob', Buffer | Uint8Array> {
+}
+/**
+ * JSON wrapper - delays JSON.stringify() to binding time.
+ * DuckDB stores JSON as VARCHAR internally.
+ */
+export interface JsonValueWrapper extends DuckDBValueWrapper<'json', unknown> {
+}
+/**
+ * Union of all wrapper types for exhaustive type checking.
+ */
+export type AnyDuckDBValueWrapper = ListValueWrapper | ArrayValueWrapper | StructValueWrapper | MapValueWrapper | TimestampValueWrapper | BlobValueWrapper | JsonValueWrapper;
+/**
+ * Type guard to check if a value is a tagged DuckDB wrapper.
+ * Optimized for fast detection in the hot path.
+ */
+export declare function isDuckDBWrapper(value: unknown): value is AnyDuckDBValueWrapper;
+/**
+ * Create a list wrapper for variable-length lists.
+ */
+export declare function wrapList(data: unknown[], elementType?: string): ListValueWrapper;
+/**
+ * Create an array wrapper for fixed-length arrays.
+ */
+export declare function wrapArray(data: unknown[], elementType?: string, fixedLength?: number): ArrayValueWrapper;
+/**
+ * Create a struct wrapper for named field structures.
+ */
+export declare function wrapStruct(data: Record<string, unknown>, schema?: Record<string, string>): StructValueWrapper;
+/**
+ * Create a map wrapper for key-value maps.
+ */
+export declare function wrapMap(data: Record<string, unknown>, valueType?: string): MapValueWrapper;
+/**
+ * Create a timestamp wrapper.
+ */
+export declare function wrapTimestamp(data: Date | string, withTimezone: boolean, precision?: number): TimestampValueWrapper;
+/**
+ * Create a blob wrapper for binary data.
+ */
+export declare function wrapBlob(data: Buffer | Uint8Array): BlobValueWrapper;
+/**
+ * Create a JSON wrapper that delays JSON.stringify() to binding time.
+ * This ensures consistent handling with other wrapped types.
+ */
+export declare function wrapJson(data: unknown): JsonValueWrapper;
+/**
+ * Convert a wrapper to a DuckDB Node API value.
+ * Uses exhaustive switch for compile-time safety.
+ */
+export declare function wrapperToNodeApiValue(wrapper: AnyDuckDBValueWrapper, toValue: (v: unknown) => DuckDBValue): DuckDBValue;

package/package.json

@@ -3,14 +3,17 @@
   "module": "./dist/index.mjs",
   "main": "./dist/index.mjs",
   "types": "./dist/index.d.ts",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "description": "A drizzle ORM client for use with DuckDB. Based on drizzle's Postgres client.",
   "type": "module",
   "scripts": {
     "build": "bun build --target=node ./src/index.ts --outfile=./dist/index.mjs --packages=external && bun build --target=node ./src/bin/duckdb-introspect.ts --outfile=./dist/duckdb-introspect.mjs --packages=external && bun run build:declarations",
     "build:declarations": "tsc --emitDeclarationOnly --project tsconfig.types.json",
     "test": "vitest",
-    "t": "vitest --watch --ui"
+    "t": "vitest --watch --ui",
+    "bench": "vitest bench --runInBand test/perf",
+    "perf:run": "bun run scripts/run-perf.ts",
+    "perf:compare": "bun run scripts/compare-perf.ts"
   },
   "bin": {
     "duckdb-introspect": "dist/duckdb-introspect.mjs"
@@ -33,7 +36,8 @@
     "prettier": "^3.5.3",
     "typescript": "^5.8.2",
     "uuid": "^10.0.0",
-    "vitest": "^1.6.0"
+    "vitest": "^1.6.0",
+    "tinybench": "^2.7.1"
   },
   "repository": {
     "type": "git",

package/src/bin/duckdb-introspect.ts

@@ -8,6 +8,8 @@ import { introspect } from '../introspect.ts';
 
 interface CliOptions {
   url?: string;
+  database?: string;
+  allDatabases: boolean;
   schemas?: string[];
   outFile: string;
   includeViews: boolean;
@@ -18,6 +20,7 @@ interface CliOptions {
 function parseArgs(argv: string[]): CliOptions {
   const options: CliOptions = {
     outFile: path.resolve(process.cwd(), 'drizzle/schema.ts'),
+    allDatabases: false,
     includeViews: false,
     useCustomTimeTypes: true,
   };
@@ -28,13 +31,26 @@ function parseArgs(argv: string[]): CliOptions {
       case '--url':
         options.url = argv[++i];
         break;
+      case '--database':
+      case '--db':
+        options.database = argv[++i];
+        break;
+      case '--all-databases':
+        options.allDatabases = true;
+        break;
       case '--schema':
       case '--schemas':
-        options.schemas = argv[++i]?.split(',').map((s) => s.trim()).filter(Boolean);
+        options.schemas = argv[++i]
+          ?.split(',')
+          .map((s) => s.trim())
+          .filter(Boolean);
         break;
       case '--out':
       case '--outFile':
-        options.outFile = path.resolve(process.cwd(), argv[++i] ?? 'drizzle/schema.ts');
+        options.outFile = path.resolve(
+          process.cwd(),
+          argv[++i] ?? 'drizzle/schema.ts'
+        );
         break;
       case '--include-views':
       case '--includeViews':
@@ -68,11 +84,27 @@ Usage:
 
 Options:
   --url            DuckDB database path (e.g. :memory:, ./local.duckdb, md:)
+  --database, --db Database/catalog to introspect (default: current database)
+  --all-databases  Introspect all attached databases (not just current)
   --schema         Comma separated schema list (defaults to all non-system schemas)
   --out            Output file (default: ./drizzle/schema.ts)
   --include-views  Include views in the generated schema
   --use-pg-time    Use pg-core timestamp/date/time instead of DuckDB custom helpers
   --import-base    Override import path for duckdb helpers (default: package name)
+
+Database Filtering:
+  By default, only tables from the current database are introspected. This prevents
+  returning tables from all attached databases in MotherDuck workspaces.
+
+  Use --database to specify a different database, or --all-databases to introspect
+  all attached databases.
+
+Examples:
+  # Local DuckDB file
+  bun x duckdb-introspect --url ./my-database.duckdb --out ./schema.ts
+
+  # MotherDuck (requires MOTHERDUCK_TOKEN env var)
+  MOTHERDUCK_TOKEN=xxx bun x duckdb-introspect --url md: --database my_cloud_db --out ./schema.ts
 `);
 }
 
@@ -94,6 +126,8 @@ async function main() {
 
   try {
     const result = await introspect(db, {
+      database: options.database,
+      allDatabases: options.allDatabases,
       schemas: options.schemas,
       includeViews: options.includeViews,
       useCustomTimeTypes: options.useCustomTimeTypes,
@@ -105,7 +139,10 @@ async function main() {
 
     console.log(`Wrote schema to ${options.outFile}`);
   } finally {
-    if ('closeSync' in connection && typeof connection.closeSync === 'function') {
+    if (
+      'closeSync' in connection &&
+      typeof connection.closeSync === 'function'
+    ) {
       connection.closeSync();
     }
   }

package/src/client.ts

@@ -1,8 +1,14 @@
 import {
   listValue,
+  timestampValue,
   type DuckDBConnection,
   type DuckDBValue,
 } from '@duckdb/node-api';
+import {
+  DUCKDB_VALUE_MARKER,
+  wrapperToNodeApiValue,
+  type AnyDuckDBValueWrapper,
+} from './value-wrappers.ts';
 
 export type DuckDBClientLike = DuckDBConnection;
 export type RowData = Record<string, unknown>;
@@ -50,13 +56,62 @@ export function prepareParams(
   });
 }
 
+/**
+ * Convert a value to DuckDB Node API value.
+ * Handles wrapper types and plain values for backward compatibility.
+ * Optimized for the common case (primitives) in the hot path.
+ */
 function toNodeApiValue(value: unknown): DuckDBValue {
+  // Fast path 1: null/undefined
+  if (value == null) return null;
+
+  // Fast path 2: primitives (most common)
+  const t = typeof value;
+  if (t === 'string' || t === 'number' || t === 'bigint' || t === 'boolean') {
+    return value as DuckDBValue;
+  }
+
+  // Fast path 3: pre-wrapped DuckDB value (Symbol check ~2-3ns)
+  if (t === 'object' && DUCKDB_VALUE_MARKER in (value as object)) {
+    return wrapperToNodeApiValue(
+      value as AnyDuckDBValueWrapper,
+      toNodeApiValue
+    );
+  }
+
+  // Legacy path: plain arrays (backward compatibility)
   if (Array.isArray(value)) {
     return listValue(value.map((inner) => toNodeApiValue(inner)));
   }
+
+  // Date conversion to timestamp
+  if (value instanceof Date) {
+    return timestampValue(BigInt(value.getTime()) * 1000n);
+  }
+
+  // Fallback for unknown objects
   return value as DuckDBValue;
 }
 
+function deduplicateColumns(columns: string[]): string[] {
+  const seen: Record<string, number> = {};
+  return columns.map((col) => {
+    const count = seen[col] ?? 0;
+    seen[col] = count + 1;
+    return count === 0 ? col : `${col}_${count}`;
+  });
+}
+
+function mapRowsToObjects(columns: string[], rows: unknown[][]): RowData[] {
+  return rows.map((vals) => {
+    const obj: Record<string, unknown> = {};
+    columns.forEach((col, idx) => {
+      obj[col] = vals[idx];
+    });
+    return obj;
+  }) as RowData[];
+}
+
 export async function closeClientConnection(
   connection: DuckDBConnection
 ): Promise<void> {
@@ -65,10 +120,7 @@ export async function closeClientConnection(
     return;
   }
 
-  if (
-    'closeSync' in connection &&
-    typeof connection.closeSync === 'function'
-  ) {
+  if ('closeSync' in connection && typeof connection.closeSync === 'function') {
     connection.closeSync();
     return;
   }
@@ -92,19 +144,84 @@ export async function executeOnClient(
       : undefined;
   const result = await client.run(query, values);
   const rows = await result.getRowsJS();
-  const columns = result.columnNames();
-  const seen: Record<string, number> = {};
-  const uniqueColumns = columns.map((col) => {
-    const count = seen[col] ?? 0;
-    seen[col] = count + 1;
-    return count === 0 ? col : `${col}_${count}`;
-  });
+  const columns =
+    // prefer deduplicated names when available (Node API >=1.4.2)
+    result.deduplicatedColumnNames?.() ?? result.columnNames();
+  const uniqueColumns = deduplicateColumns(columns);
 
-  return (rows ?? []).map((vals) => {
-    const obj: Record<string, unknown> = {};
-    uniqueColumns.forEach((col, idx) => {
-      obj[col] = vals[idx];
-    });
-    return obj;
-  }) as RowData[];
+  return rows ? mapRowsToObjects(uniqueColumns, rows) : [];
+}
+
+export interface ExecuteInBatchesOptions {
+  rowsPerChunk?: number;
+}
+
+/**
+ * Stream results from DuckDB in batches to avoid fully materializing rows in JS.
+ */
+export async function* executeInBatches(
+  client: DuckDBClientLike,
+  query: string,
+  params: unknown[],
+  options: ExecuteInBatchesOptions = {}
+): AsyncGenerator<RowData[], void, void> {
+  const rowsPerChunk =
+    options.rowsPerChunk && options.rowsPerChunk > 0
+      ? options.rowsPerChunk
+      : 100_000;
+  const values =
+    params.length > 0
+      ? (params.map((param) => toNodeApiValue(param)) as DuckDBValue[])
+      : undefined;
+
+  const result = await client.stream(query, values);
+  const columns =
+    // prefer deduplicated names when available (Node API >=1.4.2)
+    result.deduplicatedColumnNames?.() ?? result.columnNames();
+  const uniqueColumns = deduplicateColumns(columns);
+
+  let buffer: RowData[] = [];
+
+  for await (const chunk of result.yieldRowsJs()) {
+    const objects = mapRowsToObjects(uniqueColumns, chunk);
+    for (const row of objects) {
+      buffer.push(row);
+      if (buffer.length >= rowsPerChunk) {
+        yield buffer;
+        buffer = [];
+      }
+    }
+  }
+
+  if (buffer.length > 0) {
+    yield buffer;
+  }
+}
+
+/**
+ * Return columnar results when the underlying node-api exposes an Arrow/columnar API.
+ * Falls back to column-major JS arrays when Arrow is unavailable.
+ */
+export async function executeArrowOnClient(
+  client: DuckDBClientLike,
+  query: string,
+  params: unknown[]
+): Promise<unknown> {
+  const values =
+    params.length > 0
+      ? (params.map((param) => toNodeApiValue(param)) as DuckDBValue[])
+      : undefined;
+  const result = await client.run(query, values);
+
+  const maybeArrow =
+    (result as unknown as { toArrow?: () => Promise<unknown> }).toArrow ??
+    (result as unknown as { getArrowTable?: () => Promise<unknown> })
+      .getArrowTable;
+
+  if (typeof maybeArrow === 'function') {
+    return await maybeArrow.call(result);
+  }
+
+  // Fallback: return column-major JS arrays to avoid per-row object creation.
+  return result.getColumnsObjectJS();
 }