duckdb 1.4.2.0 → 1.4.4.0

data/lib/duckdb/scalar_function.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module DuckDB
+  # DuckDB::ScalarFunction encapsulates DuckDB's scalar function
+  #
+  # @note DuckDB::ScalarFunction is experimental.
+  # @note DuckDB::ScalarFunction must be used with threads=1 in DuckDB.
+  #       Set this with: connection.execute('SET threads=1')
+  class ScalarFunction
+    # Create and configure a scalar function in one call
+    #
+    # @param name [String, Symbol] the function name
+    # @param return_type [DuckDB::LogicalType] the return type
+    # @param parameter_type [DuckDB::LogicalType, nil] single parameter type (use this OR parameter_types)
+    # @param parameter_types [Array<DuckDB::LogicalType>, nil] multiple parameter types
+    # @yield [*args] the function implementation
+    # @return [DuckDB::ScalarFunction] configured scalar function ready to register
+    # @raise [ArgumentError] if block is not provided or both parameter_type and parameter_types are specified
+    #
+    # @example Single parameter function
+    #   sf = DuckDB::ScalarFunction.create(
+    #     name: :triple,
+    #     return_type: DuckDB::LogicalType::INTEGER,
+    #     parameter_type: DuckDB::LogicalType::INTEGER
+    #   ) { |v| v * 3 }
+    #
+    # @example Multiple parameters
+    #   sf = DuckDB::ScalarFunction.create(
+    #     name: :add,
+    #     return_type: DuckDB::LogicalType::INTEGER,
+    #     parameter_types: [DuckDB::LogicalType::INTEGER, DuckDB::LogicalType::INTEGER]
+    #   ) { |a, b| a + b }
+    #
+    # @example No parameters (constant function)
+    #   sf = DuckDB::ScalarFunction.create(
+    #     name: :random_num,
+    #     return_type: DuckDB::LogicalType::INTEGER
+    #   ) { rand(100) }
+    def self.create(name:, return_type:, parameter_type: nil, parameter_types: nil, &) # rubocop:disable Metrics/MethodLength
+      raise ArgumentError, 'Block required' unless block_given?
+      raise ArgumentError, 'Cannot specify both parameter_type and parameter_types' if parameter_type && parameter_types
+
+      params = if parameter_type
+                 [parameter_type]
+               elsif parameter_types
+                 parameter_types
+               else
+                 []
+               end
+
+      sf = new
+      sf.name = name.to_s
+      sf.return_type = return_type
+      params.each { |type| sf.add_parameter(type) }
+      sf.set_function(&)
+      sf
+    end
+
+    # Supported types for scalar function parameters and return values
+    SUPPORTED_TYPES = %i[
+      bigint
+      blob
+      boolean
+      date
+      double
+      float
+      integer
+      smallint
+      time
+      timestamp
+      tinyint
+      ubigint
+      uinteger
+      usmallint
+      utinyint
+      varchar
+    ].freeze
+
+    private_constant :SUPPORTED_TYPES
+
+    # Adds a parameter to the scalar function.
+    # Currently supports BIGINT, BLOB, BOOLEAN, DATE, DOUBLE, FLOAT, INTEGER, SMALLINT, TIME, TIMESTAMP, TINYINT,
+    # UBIGINT, UINTEGER, USMALLINT, UTINYINT, and VARCHAR types.
+    #
+    # @param logical_type [DuckDB::LogicalType] the parameter type
+    # @return [DuckDB::ScalarFunction] self
+    # @raise [DuckDB::Error] if the type is not supported
+    def add_parameter(logical_type)
+      raise DuckDB::Error, 'logical_type must be a DuckDB::LogicalType' unless logical_type.is_a?(DuckDB::LogicalType)
+
+      unless SUPPORTED_TYPES.include?(logical_type.type)
+        type_list = SUPPORTED_TYPES.map(&:upcase).join(', ')
+        raise DuckDB::Error,
+              "Only #{type_list} parameter types are currently supported"
+      end
+
+      _add_parameter(logical_type)
+    end
+
+    # Sets the return type for the scalar function.
+    # Currently supports BIGINT, BLOB, BOOLEAN, DATE, DOUBLE, FLOAT, INTEGER, SMALLINT, TIME, TIMESTAMP, TINYINT,
+    # UBIGINT, UINTEGER, USMALLINT, UTINYINT, and VARCHAR types.
+    #
+    # @param logical_type [DuckDB::LogicalType] the return type
+    # @return [DuckDB::ScalarFunction] self
+    # @raise [DuckDB::Error] if the type is not supported
+    def return_type=(logical_type)
+      raise DuckDB::Error, 'logical_type must be a DuckDB::LogicalType' unless logical_type.is_a?(DuckDB::LogicalType)
+
+      unless SUPPORTED_TYPES.include?(logical_type.type)
+        type_list = SUPPORTED_TYPES.map(&:upcase).join(', ')
+        raise DuckDB::Error,
+              "Only #{type_list} return types are currently supported"
+      end
+
+      _set_return_type(logical_type)
+    end
+  end
+end

data/lib/duckdb/table_function.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+module DuckDB
+  #
+  # The DuckDB::TableFunction encapsulates a DuckDB table function.
+  #
+  # NOTE: DuckDB::TableFunction is experimental now.
+  #
+  #   require 'duckdb'
+  #
+  #   db = DuckDB::Database.new
+  #   conn = db.connect
+  #
+  #   # Low-level API:
+  #   tf = DuckDB::TableFunction.new
+  #   tf.name = 'my_function'
+  #   tf.add_parameter(DuckDB::LogicalType::BIGINT)
+  #
+  #   tf.bind do |bind_info|
+  #     bind_info.add_result_column('value', DuckDB::LogicalType::BIGINT)
+  #   end
+  #
+  #   tf.execute do |func_info, output|
+  #     # Fill output data...
+  #     0  # Return 0 to signal done
+  #   end
+  #
+  #   conn.register_table_function(tf)
+  #
+  #   # High-level API (recommended):
+  #   tf = DuckDB::TableFunction.create(
+  #     name: 'my_function',
+  #     parameters: [DuckDB::LogicalType::BIGINT],
+  #     columns: { 'value' => DuckDB::LogicalType::BIGINT }
+  #   ) do |func_info, output|
+  #     # Fill output data...
+  #     0  # Return row count (0 when done)
+  #   end
+  #
+  class TableFunction
+    # TableFunction#initialize is defined in C extension
+
+    @table_adapters = {}
+
+    class << self
+      #
+      # Creates a new table function with a declarative API.
+      #
+      # @param name [String] The name of the table function
+      # @param parameters [Array<LogicalType>, Hash<String, LogicalType>] Function parameters (optional)
+      # @param columns [Hash<String, LogicalType>] Output columns (required)
+      # @yield [func_info, output] The execute block that generates data
+      # @yieldparam func_info [FunctionInfo] Function execution context
+      # @yieldparam output [DataChunk] Output data chunk to fill
+      # @yieldreturn [Integer] Number of rows generated (0 when done)
+      # @return [TableFunction] The configured table function
+      #
+      # @example Simple range function
+      #   tf = TableFunction.create(
+      #     name: 'my_range',
+      #     parameters: [LogicalType::BIGINT],
+      #     columns: { 'value' => LogicalType::BIGINT }
+      #   ) do |func_info, output|
+      #     # Generate data...
+      #     0  # Signal done
+      #   end
+      #
+      # @example Function that returns data
+      #   tf = TableFunction.create(
+      #     name: 'my_function',
+      #     columns: { 'value' => LogicalType::BIGINT }
+      #   ) do |func_info, output|
+      #     vec = output.get_vector(0)
+      #     # Fill vector...
+      #     3  # Return row count
+      #   end
+      #
+      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+      def create(name:, columns:, parameters: nil, &)
+        raise ArgumentError, 'name is required' unless name
+        raise ArgumentError, 'columns are required' unless columns
+        raise ArgumentError, 'block is required' unless block_given?
+
+        tf = new
+        tf.name = name
+
+        # Add parameters (positional or named)
+        if parameters
+          case parameters
+          when Array
+            parameters.each { |type| tf.add_parameter(type) }
+          when Hash
+            parameters.each { |param_name, type| tf.add_named_parameter(param_name, type) }
+          else
+            raise ArgumentError, 'parameters must be Array or Hash'
+          end
+        end
+
+        # Set bind callback to add result columns
+        tf.bind do |bind_info|
+          columns.each do |col_name, col_type|
+            bind_info.add_result_column(col_name, col_type)
+          end
+        end
+
+        # Set init callback (required by DuckDB)
+        tf.init do |_init_info|
+          # No-op
+        end
+
+        # Set execute callback - user's block returns row count
+        tf.execute do |func_info, output|
+          size = yield(func_info, output)
+          output.size = Integer(size)
+        end
+
+        tf
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+      # Registers a table adapter for a Ruby class.
+      #
+      # The adapter is used by +DuckDB::Connection#expose_as_table+ to convert
+      # instances of +klass+ into a DuckDB table function. The adapter must respond
+      # to +call(object, name, columns: nil)+ and return a +DuckDB::TableFunction+.
+      #
+      # == Implementing a Table Adapter
+      #
+      # An adapter is any object that responds to +call(object, name, columns: nil)+.
+      # The +columns:+ keyword argument allows callers to override the column schema;
+      # the adapter should fall back to its own schema detection when it is +nil+.
+      #
+      # The execute block passed to +DuckDB::TableFunction.create+ must:
+      # - Write one batch of rows into +output+ per call
+      # - Return the number of rows written as an +Integer+
+      # - Return +0+ to signal that all data has been exhausted
+      #
+      # @example Minimal adapter for CSV objects
+      #   class CSVTableAdapter
+      #     def call(csv, name, columns: nil)
+      #       columns ||= infer_columns(csv)
+      #
+      #       DuckDB::TableFunction.create(name:, columns:) do |_func_info, output|
+      #         row = csv.readline
+      #         if row
+      #           row.each_with_index { |cell, i| output.set_value(i, 0, cell[1]) }
+      #           1  # wrote one row
+      #         else
+      #           csv.rewind
+      #           0  # signal end of data
+      #         end
+      #       end
+      #     end
+      #
+      #     private
+      #
+      #     def infer_columns(csv)
+      #       headers = csv.first.headers
+      #       csv.rewind
+      #       headers.each_with_object({}) { |h, hsh| hsh[h] = DuckDB::LogicalType::VARCHAR }
+      #     end
+      #   end
+      #
+      #   # Register and use:
+      #   DuckDB::TableFunction.add_table_adapter(CSV, CSVTableAdapter.new)
+      #   con.execute('SET threads=1')
+      #   con.expose_as_table(csv, 'csv_table')
+      #   con.query('SELECT * FROM csv_table()').to_a
+      #
+      # @param klass [Class] the Ruby class to register an adapter for (e.g. +CSV+)
+      # @param adapter [#call] the adapter object
+      # @return [void]
+      #
+      def add_table_adapter(klass, adapter)
+        @table_adapters[klass] = adapter
+      end
+
+      # Returns the table adapter registered for the given class, or +nil+ if none.
+      #
+      # @param klass [Class] the Ruby class to look up
+      # @return [#call, nil] the registered adapter, or +nil+ if not found
+      #
+      # @example
+      #   adapter = DuckDB::TableFunction.table_adapter_for(CSV)
+      #
+      def table_adapter_for(klass)
+        @table_adapters[klass]
+      end
+    end
+  end
+end

data/lib/duckdb/vector.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module DuckDB
+  #
+  # The DuckDB::Vector represents a column vector in a data chunk.
+  #
+  # Vectors store the actual data for table function output.
+  #
+  # Example:
+  #
+  #   vector = output.get_vector(0)
+  #   vector.assign_string_element(0, 'hello')
+  #   vector.assign_string_element(1, 'world')
+  #
+  # rubocop:disable Lint/EmptyClass
+  class Vector
+    # All methods are defined in C extension (ext/duckdb/vector.c)
+  end
+  # rubocop:enable Lint/EmptyClass
+end

data/lib/duckdb/version.rb

@@ -3,5 +3,5 @@
 module DuckDB
   # The version string of ruby-duckdb.
   # Currently, ruby-duckdb is NOT semantic versioning.
-  VERSION = '1.4.2.0'
+  VERSION = '1.4.4.0'
 end

data/lib/duckdb.rb

@@ -14,8 +14,16 @@ require 'duckdb/appender'
 require 'duckdb/config'
 require 'duckdb/column'
 require 'duckdb/logical_type'
+require 'duckdb/scalar_function'
+require 'duckdb/bind_info'
+require 'duckdb/init_info'
+require 'duckdb/function_info'
+require 'duckdb/vector'
+require 'duckdb/data_chunk'
+require 'duckdb/table_function'
 require 'duckdb/infinity'
 require 'duckdb/instance_cache'
+require 'duckdb/casting'
 
 # DuckDB provides Ruby interface of DuckDB.
 module DuckDB

data/sample/async_query.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'duckdb'
 
 DuckDB::Database.open do |db|

data/sample/async_query_stream.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'duckdb'
 
 DuckDB::Database.open do |db|

data/sample/issue922.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'duckdb'
+require 'polars-df'
+
+class PolarsDataFrameTableAdapter
+  def call(data_frame, name, columns: nil)
+    columns ||= infer_columns(data_frame)
+    DuckDB::TableFunction.create(name:, columns:, &execute_block(data_frame))
+  end
+
+  private
+
+  def execute_block(data_frame)
+    counter = 0
+    height = data_frame.height
+    width = data_frame.width
+    proc do |_func_info, output|
+      next counter = 0 if counter >= height
+
+      write_row(data_frame, output, counter, width)
+      counter += 1
+      1
+    end
+  end
+
+  def write_row(data_frame, output, counter, width)
+    width.times { |index| output.set_value(index, 0, data_frame[counter, index]) }
+  end
+
+  def infer_columns(data_frame)
+    data_frame.columns.to_h { |header| [header, DuckDB::LogicalType::VARCHAR] }
+  end
+end
+
+DuckDB::TableFunction.add_table_adapter(Polars::DataFrame, PolarsDataFrameTableAdapter.new)
+
+df = Polars::DataFrame.new(
+  {
+    a: [1, 2, 3],
+    b: %w[one two three]
+  }
+)
+
+db = DuckDB::Database.open
+con = db.connect
+con.query('SET threads=1')
+con.expose_as_table(df, 'polars_df')
+result = con.query('SELECT * FROM polars_df()').to_a
+p result
+puts result.first.first == '1'
+
+con.close
+db.close

data/sample/issue922_benchmark.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+# rubocop:disable Style/OneClassPerFile
+require 'duckdb'
+require 'polars-df'
+require 'tmpdir'
+
+class PolarsDataFrameTableAdapter
+  def call(data_frame, name, columns: nil)
+    columns ||= infer_columns(data_frame)
+    DuckDB::TableFunction.create(name:, columns:, &execute_block(data_frame))
+  end
+
+  private
+
+  def execute_block(data_frame)
+    counter = 0
+    height = data_frame.height
+    width = data_frame.width
+    proc do |_func_info, output|
+      next counter = 0 if counter >= height
+
+      write_row(data_frame, output, counter, width)
+      counter += 1
+      1
+    end
+  end
+
+  def write_row(data_frame, output, counter, width)
+    width.times { |index| output.set_value(index, 0, data_frame[counter, index]) }
+  end
+
+  def infer_columns(data_frame)
+    data_frame.columns.to_h { |header| [header, DuckDB::LogicalType::VARCHAR] }
+  end
+end
+
+# Batch approach: write BATCH_SIZE rows per execute call to reduce Ruby<->C crossings
+class PolarsDataFrameBatchTableAdapter
+  BATCH_SIZE = 2048
+
+  def call(data_frame, name, columns: nil)
+    columns ||= infer_columns(data_frame)
+    DuckDB::TableFunction.create(name:, columns:, &execute_block(data_frame))
+  end
+
+  private
+
+  def execute_block(data_frame)
+    offset = 0
+    height = data_frame.height
+    width = data_frame.width
+    proc do |_func_info, output|
+      next offset = 0 if offset >= height
+
+      rows = [height - offset, BATCH_SIZE].min
+      write_batch(data_frame, output, offset, rows, width)
+      offset += rows
+      rows
+    end
+  end
+
+  def write_batch(data_frame, output, offset, rows, width)
+    rows.times do |row_idx|
+      width.times { |col_idx| output.set_value(col_idx, row_idx, data_frame[offset + row_idx, col_idx]) }
+    end
+  end
+
+  def infer_columns(data_frame)
+    data_frame.columns.to_h { |header| [header, DuckDB::LogicalType::VARCHAR] }
+  end
+end
+
+# Optimized batch approach: pre-extract columns as Ruby arrays to avoid
+# repeated Polars FFI calls, and use assign_string_element to skip type dispatch
+class PolarsDataFrameOptimizedTableAdapter
+  BATCH_SIZE = 2048
+
+  def call(data_frame, name, columns: nil)
+    columns ||= infer_columns(data_frame)
+    DuckDB::TableFunction.create(name:, columns:, &execute_block(data_frame))
+  end
+
+  private
+
+  # rubocop:disable Metrics/MethodLength
+  def execute_block(data_frame)
+    col_arrays = extract_columns(data_frame)
+    offset = 0
+    height = data_frame.height
+    width = data_frame.width
+    proc do |_func_info, output|
+      next offset = 0 if offset >= height
+
+      rows = [height - offset, BATCH_SIZE].min
+      vectors = width.times.map { |i| output.get_vector(i) }
+      write_batch(col_arrays, vectors, offset, rows)
+      offset += rows
+      rows
+    end
+  end
+  # rubocop:enable Metrics/MethodLength
+
+  def extract_columns(data_frame)
+    data_frame.columns.map { |col| data_frame[col].cast(Polars::Utf8).to_a }
+  end
+
+  def write_batch(col_arrays, vectors, offset, rows)
+    col_arrays.each_with_index do |col_data, col_idx|
+      vec = vectors[col_idx]
+      rows.times { |row_idx| vec.assign_string_element(row_idx, col_data[offset + row_idx].to_s) }
+    end
+  end
+
+  def infer_columns(data_frame)
+    data_frame.columns.to_h { |header| [header, DuckDB::LogicalType::VARCHAR] }
+  end
+end
+
+def query_via_parquet(con, data_frame, name, parquet_path)
+  data_frame.write_parquet(parquet_path)
+  con.query("CREATE OR REPLACE TABLE #{name} AS SELECT * FROM read_parquet('#{parquet_path}')")
+  con.query("SELECT * FROM #{name}").to_a
+end
+
+df = Polars::DataFrame.new(
+  {
+    id: 100_000.times.map { |i| i + 1 },
+    name: 100_000.times.map { |i| "Name#{i + 1}" },
+    age: 100_000.times.map { rand(0..100) }
+  }
+)
+
+db = DuckDB::Database.open
+con = db.connect
+con.query('SET threads=1')
+
+DuckDB::TableFunction.add_table_adapter(Polars::DataFrame, PolarsDataFrameTableAdapter.new)
+start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+con.expose_as_table(df, 'polars_tf')
+con.query('SELECT * FROM polars_tf()').to_a
+end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+DuckDB::TableFunction.add_table_adapter(Polars::DataFrame, PolarsDataFrameBatchTableAdapter.new)
+start_time3 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+con.expose_as_table(df, 'polars_tf_batch')
+con.query('SELECT * FROM polars_tf_batch()').to_a
+end_time3 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+DuckDB::TableFunction.add_table_adapter(Polars::DataFrame, PolarsDataFrameOptimizedTableAdapter.new)
+start_time4 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+con.expose_as_table(df, 'polars_tf_opt')
+con.query('SELECT * FROM polars_tf_opt()').to_a
+end_time4 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+parquet_path = File.join(Dir.tmpdir, 'issue922_benchmark.parquet')
+start_time2 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+query_via_parquet(con, df, 'polars_pq', parquet_path)
+end_time2 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+con.close
+db.close
+File.delete(parquet_path)
+
+puts "Time taken for table function approach (1 row/call):              #{end_time - start_time} seconds"
+puts "Time taken for table function approach (batch/call):              #{end_time3 - start_time3} seconds"
+puts "Time taken for table function approach (batch + pre-extract):     #{end_time4 - start_time4} seconds"
+puts "Time taken for parquet file   approach:                           #{end_time2 - start_time2} seconds"
+# rubocop:enable Style/OneClassPerFile

data/sample/issue930.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'duckdb'
+require 'csv'
+require 'stringio'
+
+class CSVTableAdapter
+  def call(csv, name, columns: nil)
+    columns ||= infer_columns(csv)
+    DuckDB::TableFunction.create(name:, columns:) do |_func_info, output|
+      write_row(csv, output)
+    end
+  end
+
+  private
+
+  def write_row(csv, output)
+    line = csv.readline
+    if line
+      line.each_with_index { |cell, index| output.set_value(index, 0, cell[1]) }
+      1
+    else
+      csv.rewind
+      0
+    end
+  end
+
+  def infer_columns(csv)
+    headers = csv.first.headers
+    csv.rewind
+    headers.to_h { |header| [header, DuckDB::LogicalType::VARCHAR] }
+  end
+end
+
+DuckDB::TableFunction.add_table_adapter(CSV, CSVTableAdapter.new)
+
+csv_data = "id,name,age\n1,Alice,30\n2,Bob,25\n3,Charlie,35"
+csv_data += 30_000.times.map { |i| "\n#{i + 4},Name#{i + 4},#{rand(0..100)}" }.join
+csv = CSV.new(StringIO.new(csv_data), headers: true)
+
+db = DuckDB::Database.open
+con = db.connect
+con.query('SET threads=1')
+con.expose_as_table(csv, 'csv_table')
+result = con.query('SELECT * FROM csv_table()').to_a
+
+p result
+puts result.first.first == '1'
+puts result.first[1] == 'Alice'