@leonardovida-md/drizzle-neo-duckdb 1.1.4 → 1.2.1

package/dist/operators.d.ts

@@ -0,0 +1,8 @@
+/**
+ * DuckDB-native array operators. Generate DuckDB-compatible SQL directly
+ * without query rewriting.
+ */
+import { type SQL, type SQLWrapper } from 'drizzle-orm';
+export declare function arrayHasAll<T>(column: SQLWrapper, values: T[] | SQLWrapper): SQL;
+export declare function arrayHasAny<T>(column: SQLWrapper, values: T[] | SQLWrapper): SQL;
+export declare function arrayContainedBy<T>(column: SQLWrapper, values: T[] | SQLWrapper): SQL;

package/dist/options.d.ts

@@ -1,6 +1,3 @@
-export type RewriteArraysMode = 'auto' | 'always' | 'never';
-export type RewriteArraysOption = boolean | RewriteArraysMode;
-export declare function resolveRewriteArraysOption(value?: RewriteArraysOption): RewriteArraysMode;
 export type PrepareCacheOption = boolean | number | {
     size?: number;
 };

package/dist/session.d.ts

@@ -10,7 +10,7 @@ import type { Assume } from 'drizzle-orm/utils';
 import type { DuckDBDialect } from './dialect.ts';
 import type { DuckDBClientLike, RowData } from './client.ts';
 import { type ExecuteBatchesRawChunk, type ExecuteInBatchesOptions } from './client.ts';
-import type { PreparedStatementCacheConfig, RewriteArraysMode } from './options.ts';
+import type { PreparedStatementCacheConfig } from './options.ts';
 export type { DuckDBClientLike, RowData } from './client.ts';
 export declare class DuckDBPreparedQuery<T extends PreparedQueryConfig> extends PgPreparedQuery<T> {
     private client;
@@ -21,19 +21,17 @@ export declare class DuckDBPreparedQuery<T extends PreparedQueryConfig> extends
     private fields;
     private _isResponseInArrayMode;
     private customResultMapper;
-    private rewriteArraysMode;
     private rejectStringArrayLiterals;
     private prepareCache;
     private warnOnStringArrayLiteral?;
     static readonly [entityKind]: string;
-    constructor(client: DuckDBClientLike, dialect: DuckDBDialect, queryString: string, params: unknown[], logger: Logger, fields: SelectedFieldsOrdered | undefined, _isResponseInArrayMode: boolean, customResultMapper: ((rows: unknown[][]) => T['execute']) | undefined, rewriteArraysMode: RewriteArraysMode, rejectStringArrayLiterals: boolean, prepareCache: PreparedStatementCacheConfig | undefined, warnOnStringArrayLiteral?: ((sql: string) => void) | undefined);
+    constructor(client: DuckDBClientLike, dialect: DuckDBDialect, queryString: string, params: unknown[], logger: Logger, fields: SelectedFieldsOrdered | undefined, _isResponseInArrayMode: boolean, customResultMapper: ((rows: unknown[][]) => T['execute']) | undefined, rejectStringArrayLiterals: boolean, prepareCache: PreparedStatementCacheConfig | undefined, warnOnStringArrayLiteral?: ((sql: string) => void) | undefined);
     execute(placeholderValues?: Record<string, unknown> | undefined): Promise<T['execute']>;
     all(placeholderValues?: Record<string, unknown> | undefined): Promise<T['all']>;
     isResponseInArrayMode(): boolean;
 }
 export interface DuckDBSessionOptions {
     logger?: Logger;
-    rewriteArrays?: RewriteArraysMode;
     rejectStringArrayLiterals?: boolean;
     prepareCache?: PreparedStatementCacheConfig;
 }
@@ -44,7 +42,6 @@ export declare class DuckDBSession<TFullSchema extends Record<string, unknown> =
     static readonly [entityKind]: string;
     protected dialect: DuckDBDialect;
     private logger;
-    private rewriteArraysMode;
     private rejectStringArrayLiterals;
     private prepareCache;
     private hasWarnedArrayLiteral;

package/dist/sql/ast-transformer.d.ts

@@ -0,0 +1,31 @@
+/**
+ * AST-based SQL transformer for DuckDB compatibility.
+ *
+ * Transforms:
+ * - Array operators: @>, <@, && -> array_has_all(), array_has_any()
+ * - JOIN column qualification: "col" = "col" -> "left"."col" = "right"."col"
+ *
+ * Performance optimizations:
+ * - LRU cache for transformed queries (avoids re-parsing identical queries)
+ * - Smart heuristics to skip JOIN qualification when not needed
+ * - Early exit when no transformation is required
+ */
+export type TransformResult = {
+    sql: string;
+    transformed: boolean;
+};
+export declare function transformSQL(query: string): TransformResult;
+/**
+ * Clear the transformation cache. Useful for testing or memory management.
+ */
+export declare function clearTransformCache(): void;
+/**
+ * Get current cache statistics for monitoring.
+ */
+export declare function getTransformCacheStats(): {
+    size: number;
+    maxSize: number;
+};
+export declare function needsTransformation(query: string): boolean;
+export { transformArrayOperators } from './visitors/array-operators.ts';
+export { qualifyJoinColumns } from './visitors/column-qualifier.ts';

package/dist/sql/visitors/array-operators.d.ts

@@ -0,0 +1,5 @@
+/**
+ * AST visitor to transform Postgres array operators to DuckDB functions.
+ */
+import type { AST } from 'node-sql-parser';
+export declare function transformArrayOperators(ast: AST | AST[]): boolean;

package/dist/sql/visitors/column-qualifier.d.ts

@@ -0,0 +1,10 @@
+/**
+ * AST visitor to qualify unqualified column references in JOIN ON clauses.
+ *
+ * Performance optimizations:
+ * - Early exit when no unqualified columns found in ON clause
+ * - Skip processing if all columns are already qualified
+ * - Minimal tree traversal when possible
+ */
+import type { AST } from 'node-sql-parser';
+export declare function qualifyJoinColumns(ast: AST | AST[]): boolean;

package/dist/utils.d.ts

@@ -1,3 +1,2 @@
 export { aliasFields } from './sql/selection.ts';
-export { adaptArrayOperators } from './sql/query-rewriters.ts';
 export { mapResultRow } from './sql/result-mapper.ts';

package/package.json

@@ -3,7 +3,7 @@
   "module": "./dist/index.mjs",
   "main": "./dist/index.mjs",
   "types": "./dist/index.d.ts",
-  "version": "1.1.4",
+  "version": "1.2.1",
   "description": "A drizzle ORM client for use with DuckDB. Based on drizzle's Postgres client.",
   "type": "module",
   "scripts": {
@@ -11,7 +11,7 @@
     "build:declarations": "tsc --emitDeclarationOnly --project tsconfig.types.json",
     "test": "vitest",
     "t": "vitest --watch --ui",
-    "bench": "vitest bench --runInBand test/perf",
+    "bench": "vitest bench --run test/perf --pool=threads --poolOptions.threads.singleThread=true --no-file-parallelism",
     "perf:run": "bun run scripts/run-perf.ts",
     "perf:compare": "bun run scripts/compare-perf.ts"
   },
@@ -79,6 +79,7 @@
     "dist/**/*.d.ts"
   ],
   "dependencies": {
-    "@duckdb/node-bindings-darwin-arm64": "^1.4.2-r.1"
+    "@duckdb/node-bindings-darwin-arm64": "^1.4.2-r.1",
+    "node-sql-parser": "^5.3.13"
   }
 }

package/src/dialect.ts

@@ -15,9 +15,13 @@ import {
 } from 'drizzle-orm/pg-core';
 import {
   sql,
+  SQL,
   type DriverValueEncoder,
   type QueryTypingsValue,
 } from 'drizzle-orm';
+import type { QueryWithTypings } from 'drizzle-orm/sql/sql';
+
+import { transformSQL } from './sql/ast-transformer.ts';
 
 const enum SavepointSupport {
   Unknown = 0,
@@ -181,4 +185,20 @@ export class DuckDBDialect extends PgDialect {
       return 'none';
     }
   }
+
+  override sqlToQuery(
+    sqlObj: SQL,
+    invokeSource?: 'indexes' | undefined
+  ): QueryWithTypings {
+    // First, let the parent generate the SQL string
+    const result = super.sqlToQuery(sqlObj, invokeSource);
+
+    // Apply AST-based transformations for DuckDB compatibility
+    const transformed = transformSQL(result.sql);
+
+    return {
+      ...result,
+      sql: transformed.sql,
+    };
+  }
 }

package/src/driver.ts

@@ -37,16 +37,12 @@ import {
 } from './pool.ts';
 import {
   resolvePrepareCacheOption,
-  resolveRewriteArraysOption,
   type PreparedStatementCacheConfig,
   type PrepareCacheOption,
-  type RewriteArraysMode,
-  type RewriteArraysOption,
 } from './options.ts';
 
 export interface PgDriverOptions {
   logger?: Logger;
-  rewriteArrays?: RewriteArraysMode;
   rejectStringArrayLiterals?: boolean;
   prepareCache?: PreparedStatementCacheConfig;
 }
@@ -65,7 +61,6 @@ export class DuckDBDriver {
   ): DuckDBSession<Record<string, unknown>, TablesRelationalConfig> {
     return new DuckDBSession(this.client, this.dialect, schema, {
       logger: this.options.logger,
-      rewriteArrays: this.options.rewriteArrays ?? 'auto',
       rejectStringArrayLiterals: this.options.rejectStringArrayLiterals,
       prepareCache: this.options.prepareCache,
     });
@@ -83,7 +78,6 @@ export interface DuckDBConnectionConfig {
 export interface DuckDBDrizzleConfig<
   TSchema extends Record<string, unknown> = Record<string, never>,
 > extends DrizzleConfig<TSchema> {
-  rewriteArrays?: RewriteArraysOption;
   rejectStringArrayLiterals?: boolean;
   prepareCache?: PrepareCacheOption;
   /** Pool configuration. Use preset name, size config, or false to disable. */
@@ -126,7 +120,6 @@ function createFromClient<
   instance?: DuckDBInstance
 ): DuckDBDatabase<TSchema, ExtractTablesWithRelations<TSchema>> {
   const dialect = new DuckDBDialect();
-  const rewriteArraysMode = resolveRewriteArraysOption(config.rewriteArrays);
   const prepareCache = resolvePrepareCacheOption(config.prepareCache);
 
   const logger =
@@ -148,7 +141,6 @@ function createFromClient<
 
   const driver = new DuckDBDriver(client, dialect, {
     logger,
-    rewriteArrays: rewriteArraysMode,
     rejectStringArrayLiterals: config.rejectStringArrayLiterals,
     prepareCache,
   });

package/src/index.ts

@@ -8,3 +8,4 @@ export * from './pool.ts';
 export * from './olap.ts';
 export * from './value-wrappers.ts';
 export * from './options.ts';
+export * from './operators.ts';

package/src/operators.ts

@@ -0,0 +1,27 @@
+/**
+ * DuckDB-native array operators. Generate DuckDB-compatible SQL directly
+ * without query rewriting.
+ */
+
+import { sql, type SQL, type SQLWrapper } from 'drizzle-orm';
+
+export function arrayHasAll<T>(
+  column: SQLWrapper,
+  values: T[] | SQLWrapper
+): SQL {
+  return sql`array_has_all(${column}, ${values})`;
+}
+
+export function arrayHasAny<T>(
+  column: SQLWrapper,
+  values: T[] | SQLWrapper
+): SQL {
+  return sql`array_has_any(${column}, ${values})`;
+}
+
+export function arrayContainedBy<T>(
+  column: SQLWrapper,
+  values: T[] | SQLWrapper
+): SQL {
+  return sql`array_has_all(${values}, ${column})`;
+}

package/src/options.ts

@@ -1,18 +1,3 @@
-export type RewriteArraysMode = 'auto' | 'always' | 'never';
-
-export type RewriteArraysOption = boolean | RewriteArraysMode;
-
-const DEFAULT_REWRITE_ARRAYS_MODE: RewriteArraysMode = 'auto';
-
-export function resolveRewriteArraysOption(
-  value?: RewriteArraysOption
-): RewriteArraysMode {
-  if (value === undefined) return DEFAULT_REWRITE_ARRAYS_MODE;
-  if (value === true) return 'auto';
-  if (value === false) return 'never';
-  return value;
-}
-
 export type PrepareCacheOption = boolean | number | { size?: number };
 
 export interface PreparedStatementCacheConfig {

package/src/session.ts

@@ -15,10 +15,6 @@ import type {
 } from 'drizzle-orm/relations';
 import { fillPlaceholders, type Query, SQL, sql } from 'drizzle-orm/sql/sql';
 import type { Assume } from 'drizzle-orm/utils';
-import {
-  adaptArrayOperators,
-  qualifyJoinColumns,
-} from './sql/query-rewriters.ts';
 import { mapResultRow } from './sql/result-mapper.ts';
 import { TransactionRollbackError } from 'drizzle-orm/errors';
 import type { DuckDBDialect } from './dialect.ts';
@@ -39,10 +35,7 @@ import {
 } from './client.ts';
 import { isPool } from './client.ts';
 import type { DuckDBConnection } from '@duckdb/node-api';
-import type {
-  PreparedStatementCacheConfig,
-  RewriteArraysMode,
-} from './options.ts';
+import type { PreparedStatementCacheConfig } from './options.ts';
 
 export type { DuckDBClientLike, RowData } from './client.ts';
 
@@ -56,34 +49,6 @@ function isSavepointSyntaxError(error: unknown): boolean {
   );
 }
 
-function rewriteQuery(
-  mode: RewriteArraysMode,
-  query: string
-): { sql: string; rewritten: boolean } {
-  if (mode === 'never') {
-    return { sql: query, rewritten: false };
-  }
-
-  let result = query;
-  let wasRewritten = false;
-
-  // Rewrite Postgres array operators to DuckDB functions
-  const arrayRewritten = adaptArrayOperators(result);
-  if (arrayRewritten !== result) {
-    result = arrayRewritten;
-    wasRewritten = true;
-  }
-
-  // Qualify unqualified column references in JOIN ON clauses
-  const joinQualified = qualifyJoinColumns(result);
-  if (joinQualified !== result) {
-    result = joinQualified;
-    wasRewritten = true;
-  }
-
-  return { sql: result, rewritten: wasRewritten };
-}
-
 export class DuckDBPreparedQuery<
   T extends PreparedQueryConfig,
 > extends PgPreparedQuery<T> {
@@ -100,7 +65,6 @@ export class DuckDBPreparedQuery<
     private customResultMapper:
       | ((rows: unknown[][]) => T['execute'])
       | undefined,
-    private rewriteArraysMode: RewriteArraysMode,
     private rejectStringArrayLiterals: boolean,
     private prepareCache: PreparedStatementCacheConfig | undefined,
     private warnOnStringArrayLiteral?: (sql: string) => void
@@ -121,19 +85,7 @@ export class DuckDBPreparedQuery<
           : undefined,
       }
     );
-    const { sql: rewrittenQuery, rewritten: didRewrite } = rewriteQuery(
-      this.rewriteArraysMode,
-      this.queryString
-    );
-
-    if (didRewrite) {
-      this.logger.logQuery(
-        `[duckdb] original query before array rewrite: ${this.queryString}`,
-        params
-      );
-    }
-
-    this.logger.logQuery(rewrittenQuery, params);
+    this.logger.logQuery(this.queryString, params);
 
     const { fields, joinsNotNullableMap, customResultMapper } =
       this as typeof this & { joinsNotNullableMap?: Record<string, boolean> };
@@ -141,7 +93,7 @@ export class DuckDBPreparedQuery<
     if (fields) {
       const { rows } = await executeArraysOnClient(
         this.client,
-        rewrittenQuery,
+        this.queryString,
         params,
         { prepareCache: this.prepareCache }
       );
@@ -157,7 +109,7 @@ export class DuckDBPreparedQuery<
           );
     }
 
-    const rows = await executeOnClient(this.client, rewrittenQuery, params, {
+    const rows = await executeOnClient(this.client, this.queryString, params, {
       prepareCache: this.prepareCache,
     });
 
@@ -177,7 +129,6 @@ export class DuckDBPreparedQuery<
 
 export interface DuckDBSessionOptions {
   logger?: Logger;
-  rewriteArrays?: RewriteArraysMode;
   rejectStringArrayLiterals?: boolean;
   prepareCache?: PreparedStatementCacheConfig;
 }
@@ -190,7 +141,6 @@ export class DuckDBSession<
 
   protected override dialect: DuckDBDialect;
   private logger: Logger;
-  private rewriteArraysMode: RewriteArraysMode;
   private rejectStringArrayLiterals: boolean;
   private prepareCache: PreparedStatementCacheConfig | undefined;
   private hasWarnedArrayLiteral = false;
@@ -205,12 +155,10 @@ export class DuckDBSession<
     super(dialect);
     this.dialect = dialect;
     this.logger = options.logger ?? new NoopLogger();
-    this.rewriteArraysMode = options.rewriteArrays ?? 'auto';
     this.rejectStringArrayLiterals = options.rejectStringArrayLiterals ?? false;
     this.prepareCache = options.prepareCache;
     this.options = {
       ...options,
-      rewriteArrays: this.rewriteArraysMode,
       prepareCache: this.prepareCache,
     };
   }
@@ -232,7 +180,6 @@ export class DuckDBSession<
       fields,
       isResponseInArrayMode,
       customResultMapper,
-      this.rewriteArraysMode,
       this.rejectStringArrayLiterals,
       this.prepareCache,
       this.rejectStringArrayLiterals ? undefined : this.warnOnStringArrayLiteral
@@ -326,23 +273,12 @@ export class DuckDBSession<
         ? undefined
         : () => this.warnOnStringArrayLiteral(builtQuery.sql),
     });
-    const { sql: rewrittenQuery, rewritten: didRewrite } = rewriteQuery(
-      this.rewriteArraysMode,
-      builtQuery.sql
-    );
 
-    if (didRewrite) {
-      this.logger.logQuery(
-        `[duckdb] original query before array rewrite: ${builtQuery.sql}`,
-        params
-      );
-    }
-
-    this.logger.logQuery(rewrittenQuery, params);
+    this.logger.logQuery(builtQuery.sql, params);
 
     return executeInBatches(
       this.client,
-      rewrittenQuery,
+      builtQuery.sql,
       params,
       options
     ) as AsyncGenerator<GenericRowData<T>[], void, void>;
@@ -361,21 +297,10 @@ export class DuckDBSession<
         ? undefined
         : () => this.warnOnStringArrayLiteral(builtQuery.sql),
     });
-    const { sql: rewrittenQuery, rewritten: didRewrite } = rewriteQuery(
-      this.rewriteArraysMode,
-      builtQuery.sql
-    );
 
-    if (didRewrite) {
-      this.logger.logQuery(
-        `[duckdb] original query before array rewrite: ${builtQuery.sql}`,
-        params
-      );
-    }
+    this.logger.logQuery(builtQuery.sql, params);
 
-    this.logger.logQuery(rewrittenQuery, params);
-
-    return executeInBatchesRaw(this.client, rewrittenQuery, params, options);
+    return executeInBatchesRaw(this.client, builtQuery.sql, params, options);
   }
 
   async executeArrow(query: SQL): Promise<unknown> {
@@ -388,21 +313,10 @@ export class DuckDBSession<
         ? undefined
         : () => this.warnOnStringArrayLiteral(builtQuery.sql),
     });
-    const { sql: rewrittenQuery, rewritten: didRewrite } = rewriteQuery(
-      this.rewriteArraysMode,
-      builtQuery.sql
-    );
-
-    if (didRewrite) {
-      this.logger.logQuery(
-        `[duckdb] original query before array rewrite: ${builtQuery.sql}`,
-        params
-      );
-    }
 
-    this.logger.logQuery(rewrittenQuery, params);
+    this.logger.logQuery(builtQuery.sql, params);
 
-    return executeArrowOnClient(this.client, rewrittenQuery, params);
+    return executeArrowOnClient(this.client, builtQuery.sql, params);
   }
 
   markRollbackOnly(): void {

package/src/sql/ast-transformer.ts

@@ -0,0 +1,144 @@
+/**
+ * AST-based SQL transformer for DuckDB compatibility.
+ *
+ * Transforms:
+ * - Array operators: @>, <@, && -> array_has_all(), array_has_any()
+ * - JOIN column qualification: "col" = "col" -> "left"."col" = "right"."col"
+ *
+ * Performance optimizations:
+ * - LRU cache for transformed queries (avoids re-parsing identical queries)
+ * - Smart heuristics to skip JOIN qualification when not needed
+ * - Early exit when no transformation is required
+ */
+
+import nodeSqlParser from 'node-sql-parser';
+const { Parser } = nodeSqlParser;
+import type { AST } from 'node-sql-parser';
+
+import { transformArrayOperators } from './visitors/array-operators.ts';
+import { qualifyJoinColumns } from './visitors/column-qualifier.ts';
+
+const parser = new Parser();
+
+export type TransformResult = {
+  sql: string;
+  transformed: boolean;
+};
+
+// LRU cache for transformed SQL queries
+// Key: original SQL, Value: transformed result
+const CACHE_SIZE = 500;
+const transformCache = new Map<string, TransformResult>();
+
+function getCachedOrTransform(
+  query: string,
+  transform: () => TransformResult
+): TransformResult {
+  const cached = transformCache.get(query);
+  if (cached) {
+    // Move to end for LRU behavior
+    transformCache.delete(query);
+    transformCache.set(query, cached);
+    return cached;
+  }
+
+  const result = transform();
+
+  // Add to cache with LRU eviction
+  if (transformCache.size >= CACHE_SIZE) {
+    // Delete oldest entry (first key in Map iteration order)
+    const oldestKey = transformCache.keys().next().value;
+    if (oldestKey) {
+      transformCache.delete(oldestKey);
+    }
+  }
+  transformCache.set(query, result);
+
+  return result;
+}
+
+const DEBUG_ENV = 'DRIZZLE_DUCKDB_DEBUG_AST';
+
+function hasJoin(query: string): boolean {
+  return /\bjoin\b/i.test(query);
+}
+
+function debugLog(message: string, payload?: unknown): void {
+  if (process?.env?.[DEBUG_ENV]) {
+    // eslint-disable-next-line no-console
+    console.debug('[duckdb-ast]', message, payload ?? '');
+  }
+}
+
+export function transformSQL(query: string): TransformResult {
+  const needsArrayTransform =
+    query.includes('@>') || query.includes('<@') || query.includes('&&');
+  const needsJoinTransform =
+    hasJoin(query) || /\bupdate\b/i.test(query) || /\bdelete\b/i.test(query);
+
+  if (!needsArrayTransform && !needsJoinTransform) {
+    return { sql: query, transformed: false };
+  }
+
+  // Use cache for repeated queries
+  return getCachedOrTransform(query, () => {
+    try {
+      const ast = parser.astify(query, { database: 'PostgreSQL' });
+
+      let transformed = false;
+
+      if (needsArrayTransform) {
+        transformed = transformArrayOperators(ast) || transformed;
+      }
+
+      if (needsJoinTransform) {
+        transformed = qualifyJoinColumns(ast) || transformed;
+      }
+
+      if (!transformed) {
+        debugLog('AST parsed but no transformation applied', {
+          join: needsJoinTransform,
+        });
+        return { sql: query, transformed: false };
+      }
+
+      const transformedSql = parser.sqlify(ast, { database: 'PostgreSQL' });
+
+      return { sql: transformedSql, transformed: true };
+    } catch (err) {
+      debugLog('AST transform failed; returning original SQL', {
+        error: (err as Error).message,
+      });
+      return { sql: query, transformed: false };
+    }
+  });
+}
+
+/**
+ * Clear the transformation cache. Useful for testing or memory management.
+ */
+export function clearTransformCache(): void {
+  transformCache.clear();
+}
+
+/**
+ * Get current cache statistics for monitoring.
+ */
+export function getTransformCacheStats(): { size: number; maxSize: number } {
+  return { size: transformCache.size, maxSize: CACHE_SIZE };
+}
+
+export function needsTransformation(query: string): boolean {
+  const lower = query.toLowerCase();
+  return (
+    query.includes('@>') ||
+    query.includes('<@') ||
+    query.includes('&&') ||
+    lower.includes('join') ||
+    lower.includes('update') ||
+    lower.includes('delete')
+  );
+}
+
+export { transformArrayOperators } from './visitors/array-operators.ts';
+export { qualifyJoinColumns } from './visitors/column-qualifier.ts';