sequel-duckdb 0.3.0 → 0.4.0

data/lib/sequel/duckdb/helpers/copier.rb

@@ -4,7 +4,14 @@ module Sequel
   module DuckDB
     module Helpers
       # Builds COPY SQL statements for exporting DuckDB query results to files.
+      #
+      # FORMAT comes from the +format:+ option when given, otherwise from the
+      # destination's file extension. A destination with neither (for example a
+      # directory for a partitioned COPY) raises Sequel::Error.
       class Copier
+        # Option values that DuckDB accepts as a bare keyword (FORMAT PARQUET).
+        BARE_WORD = /\A[A-Za-z_][A-Za-z0-9_]*\z/
+
         def initialize(src, dst, options = Sequel::OPTS)
           @src = src
           @dst = dst
@@ -12,7 +19,7 @@ module Sequel
         end
 
         def to_sql
-          "COPY (#{source}) TO '#{@dst}' #{options_str}"
+          "COPY (#{source}) TO #{string_literal(@dst)} #{options_str}"
         end
 
         def source
@@ -23,26 +30,42 @@ module Sequel
           end
         end
 
-        def destination
-          Pathname.new(@dst).expand_path.to_s
-        end
-
         def options_str
           opts_str = { format: format }.merge(@options).map { |k, v| format_option(k, v) }.compact.join(", ").strip
           opts_str.empty? ? "" : "(#{opts_str})"
         end
 
         def format_option(key, value)
+          return "PARTITION_BY (#{Array(value).map { |name| quoted_identifier(name) }.join(", ")})" if key.to_s == "partition_by"
+
           key = key.to_s.upcase
           case value
           when true then key
           when false then nil
-          else "#{key} #{value.to_s.upcase}"
+          when Integer, Float then "#{key} #{value}"
+          when BARE_WORD then "#{key} #{value.to_s.upcase}"
+          else "#{key} #{string_literal(value)}"
           end
         end
 
         def format
-          File.extname(@dst).delete_prefix(".")
+          @options.fetch(:format) do
+            File.extname(@dst.to_s).delete_prefix(".").then do |ext|
+              raise Sequel::Error, "Cannot infer COPY format from #{@dst.to_s.inspect}: it has no file extension. Pass format:, for example format: :parquet" if ext.empty?
+
+              ext
+            end
+          end
+        end
+
+        private
+
+        def string_literal(value)
+          "'#{value.to_s.gsub("'", "''")}'"
+        end
+
+        def quoted_identifier(name)
+          "\"#{name.to_s.gsub('"', '""')}\""
         end
       end
     end

data/lib/sequel/duckdb/helpers/pathifier.rb

@@ -120,7 +120,7 @@ module Sequel
         #   db.literal(sql)
         #   # => "read_csv(['/data/a.csv','/data/b.csv'])"
         def to_sql
-          paths_sql = @paths.map { |p| "'#{p}'" }.join(",").then do |paths_arr|
+          paths_sql = @paths.map { |p| "'#{p.to_s.gsub("'", "''")}'" }.join(",").then do |paths_arr|
             Sequel::LiteralString.new("[#{paths_arr}]")
           end
 

data/lib/sequel/duckdb/instance_registry.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Sequel
+  module DuckDB
+    # One ::DuckDB::Database per database file per process.
+    #
+    # Two ::DuckDB::Database instances on one file in one process do not reliably see
+    # each other's writes, and DuckDB's file lock does not stop the second one, because
+    # fcntl locks are per process. So every Sequel::Database on a file shares the one
+    # instance held here, counted by reference. The instance closes when the last
+    # reference is released.
+    #
+    # This does not use ::DuckDB::InstanceCache. In ruby-duckdb 1.5.5.1 it returns one
+    # Ruby object to every holder, and get_or_create hangs when called again after that
+    # object was closed (seed sequel-duckdb-k82).
+    #
+    # The first Sequel::Database to open a file fixes its DuckDB settings until the
+    # instance closes. Opening it again with different settings raises SettingsConflict,
+    # as the Python, JDBC and R DuckDB clients do.
+    #
+    # In-memory databases are never registered: each one is separate by design.
+    module InstanceRegistry
+      Entry = Struct.new(:database, :references, :settings)
+
+      # Raised when a file that is open in this process is opened with different settings.
+      class SettingsConflict < Sequel::DatabaseConnectionError
+        def initialize(path, held, requested)
+          super("#{path} is already open in this process with different DuckDB settings " \
+                "(open: #{held.inspect}, requested: #{requested.inspect}). The first " \
+                "Sequel::Database to open a file fixes its settings until every " \
+                "Sequel::Database on it has disconnected.")
+        end
+      end
+
+      @mutex = Mutex.new
+      @entries = {}
+      @pid = Process.pid
+
+      class << self
+        # Open a ::DuckDB::Database that is not shared. +settings+ maps DuckDB setting
+        # names to values, both Strings (see normalize_settings).
+        def open(path, settings)
+          return ::DuckDB::Database.open(path) if settings.empty?
+
+          config = ::DuckDB::Config.new
+          settings.each { |name, value| config.set_config(name, value) }
+          ::DuckDB::Database.open(path, config: config)
+        end
+
+        # The settings Hash in the one form that acquire compares.
+        def normalize_settings(settings)
+          (settings || {}).to_h { |name, value| [ name.to_s, value.to_s ] }
+        end
+
+        # Return the shared ::DuckDB::Database for +path+ and take one reference to it.
+        # Raises SettingsConflict when +path+ is already open with other +settings+.
+        def acquire(path, settings = {})
+          key = key_for(path)
+          @mutex.synchronize do
+            forget_parent_entries
+            entry = (@entries[key] ||= Entry.new(open(path, settings), 0, settings))
+            raise SettingsConflict.new(path, entry.settings, settings) unless entry.settings == settings
+
+            entry.references += 1
+            entry.database
+          end
+        end
+
+        # Give back one reference taken by acquire. The last one closes the instance.
+        def release(path)
+          key = key_for(path)
+          @mutex.synchronize do
+            forget_parent_entries
+            entry = @entries[key]
+            return unless entry
+
+            entry.references -= 1
+            return if entry.references.positive?
+
+            @entries.delete(key)
+            entry.database.close
+          end
+        end
+
+        # The number of references held on +path+, 0 when it is not open.
+        def references(path)
+          @mutex.synchronize do
+            forget_parent_entries
+            @entries[key_for(path)]&.references || 0
+          end
+        end
+
+        private
+
+        # The real path of the file's directory, so two spellings of one file share an
+        # entry. The file itself may not exist yet.
+        def key_for(path)
+          File.join(File.realpath(File.dirname(path)), File.basename(path))
+        rescue SystemCallError
+          File.expand_path(path)
+        end
+
+        # A forked child inherits the parent's entries, but the instances belong to the
+        # parent. The child opens its own, as it did before this registry existed.
+        def forget_parent_entries
+          return if @pid == Process.pid
+
+          @entries = {}
+          @pid = Process.pid
+        end
+      end
+    end
+  end
+end

data/lib/sequel/duckdb/sql_time_tz.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Sequel
+  module DuckDB
+    # A TIMETZ value: a time of day with a UTC offset.
+    #
+    # Sequel::SQLTime has no offset, so a TIMETZ value written as one gets DuckDB's
+    # session time zone. This subclass keeps the offset and is written as
+    # 'HH:MM:SS.ffffff+HH:MM'. Like Sequel::SQLTime, its date is Sequel::SQLTime.date,
+    # so two values with the same time and offset are equal.
+    class SQLTimeTZ < Sequel::SQLTime
+      # The time of day and UTC offset of +time+, on Sequel::SQLTime.date.
+      def self.from_time(time)
+        date = Sequel::SQLTime.date
+        new(date.year, date.month, date.day, time.hour, time.min,
+            time.sec + Rational(time.nsec, 1_000_000_000), time.utc_offset)
+      end
+
+      # Two values with the same instant but different offsets are different TIMETZ
+      # values, so a model that is assigned a new offset reports the column as changed.
+      def ==(other)
+        super && other.respond_to?(:utc_offset) && utc_offset == other.utc_offset
+      end
+
+      # HH:MM:SS+HH:MM
+      def to_s(*args)
+        args.empty? ? strftime("%H:%M:%S%:z") : super
+      end
+    end
+  end
+end

data/lib/sequel/duckdb/version.rb

@@ -11,6 +11,6 @@ module Sequel
     #   puts Sequel::DuckDB::VERSION
     #
     # @since 0.1.0
-    VERSION = "0.3.0"
+    VERSION = "0.4.0"
   end
 end

data/lib/sequel/extensions/duckdb_array.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+# The duckdb_array extension writes Ruby arrays to DuckDB LIST and ARRAY columns and
+# reads them back as Sequel::DuckDB::DuckDBArray objects. It follows Sequel's
+# pg_array extension. It is opt-in: without it, the adapter's SQL and values do not
+# change.
+#
+# Load the constructor globally:
+#
+#   Sequel.extension :duckdb_array
+#
+# Load it into a Database, which also converts results, schema types and model
+# typecasting for that Database:
+#
+#   DB.extension :duckdb_array
+#
+# Write a list:
+#
+#   DB[:t].insert(l: Sequel.duckdb_array([1, 2]))  # INSERT ... VALUES ([1, 2])
+#   Sequel.duckdb_array([[1], []])                 # [[1], []]
+#
+# A plain Ruby Array still means an IN list or a condition in Sequel. Only a
+# DuckDBArray is written as a DuckDB list.
+#
+# With the extension loaded into a Database:
+#
+# * LIST and ARRAY result values (INTEGER[], INTEGER[3]) are DuckDBArray objects.
+#   Only the top-level value is wrapped. A value nested in it stays as ruby-duckdb
+#   returns it.
+# * db.schema reports an INTEGER[] or INTEGER[3] column as :integer_array. A column
+#   whose element type has no Sequel type, for example a STRUCT without the
+#   duckdb_struct extension, is reported as :array.
+# * A model typecasts an Array assigned to such a column to a DuckDBArray, and
+#   typecasts each element with the element's type.
+
+require "delegate"
+
+module Sequel
+  module DuckDB
+    # An Array that Sequel writes as a DuckDB list literal.
+    class DuckDBArray < DelegateClass(Array)
+      # Element types that get a :<type>_array schema type.
+      ELEMENT_TYPES = %i[string integer bigint float decimal boolean date datetime time interval blob uuid struct
+                         map json].freeze
+
+      # Element types whose typecast method has another name.
+      ELEMENT_TYPECASTS = { bigint: :integer }.freeze
+
+      # The element type and the dimensions of a LIST or ARRAY db_type:
+      # INTEGER[], INTEGER[3], INTEGER[][], STRUCT(a INTEGER)[].
+      DB_TYPE = /\A(.+?)(?:\[\d*\])+\z/m
+
+      # Element db_types that the adapter maps to :string only as its fallback.
+      COMPOSITE_DB_TYPE = /\A(?:STRUCT|MAP|UNION)\(/i
+
+      module DatabaseMethods
+        def self.extended(db)
+          db.instance_exec do
+            (ELEMENT_TYPES.map { |type| :"#{type}_array" } + [ :array ]).each do |type|
+              @schema_type_classes[type] = DuckDBArray
+            end
+            if respond_to?(:add_conversion_proc)
+              to_array = DuckDBArray.method(:new)
+              add_conversion_proc(:list, to_array)
+              add_conversion_proc(:array, to_array)
+            end
+          end
+        end
+
+        ELEMENT_TYPES.each do |type|
+          method_name = :"typecast_value_#{type}_array"
+          element_typecast = :"typecast_value_#{ELEMENT_TYPECASTS.fetch(type, type)}"
+          define_method(method_name) { |value| typecast_value_duckdb_array(value, element_typecast) }
+          private method_name
+        end
+
+        private
+
+        def typecast_value_array(value)
+          typecast_value_duckdb_array(value, nil)
+        end
+
+        def map_duckdb_type_to_sequel(duckdb_type)
+          element = duckdb_type[DB_TYPE, 1]
+          return super unless element
+
+          type = map_duckdb_type_to_sequel(element)
+          if ELEMENT_TYPES.include?(type) && !(type == :string && element.match?(COMPOSITE_DB_TYPE))
+            :"#{type}_array"
+          else
+            :array
+          end
+        end
+
+        def typecast_value_duckdb_array(value, element_typecast)
+          raise Sequel::InvalidValue, "invalid value for array type: #{value.inspect}" unless value.is_a?(Array) || value.is_a?(DuckDBArray)
+
+          element_typecast = nil unless element_typecast && respond_to?(element_typecast, true)
+          DuckDBArray.new(typecast_duckdb_array_elements(value, element_typecast))
+        end
+
+        # Nested arrays become plain Arrays. A nil element stays nil.
+        def typecast_duckdb_array_elements(array, element_typecast)
+          array.map do |element|
+            if element.is_a?(Array) || element.is_a?(DuckDBArray)
+              typecast_duckdb_array_elements(element, element_typecast)
+            elsif element.nil? || element_typecast.nil?
+              element
+            else
+              send(element_typecast, element)
+            end
+          end
+        end
+      end
+
+      def initialize(array)
+        super(array.to_ary)
+      end
+
+      # [1, 2]. A nested Array gives a nested list, and every other element goes
+      # through the dataset's literal_append.
+      def sql_literal_append(ds, sql)
+        append_list(ds, sql, __getobj__)
+      end
+
+      private
+
+      def append_list(ds, sql, array)
+        sql << "["
+        array.each_with_index do |element, i|
+          sql << ", " unless i.zero?
+          element.is_a?(Array) ? append_list(ds, sql, element) : ds.literal_append(sql, element)
+        end
+        sql << "]"
+      end
+    end
+  end
+
+  module SQL
+    module Builders
+      # Wraps an Array, so that Sequel writes it as a DuckDB list.
+      def duckdb_array(array)
+        Sequel::DuckDB::DuckDBArray.new(array)
+      end
+    end
+  end
+
+  Database.register_extension(:duckdb_array, DuckDB::DuckDBArray::DatabaseMethods)
+end

data/lib/sequel/extensions/duckdb_json.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+# The duckdb_json extension writes Ruby hashes and arrays to DuckDB JSON columns and
+# reads them back as Sequel::DuckDB::DuckDBJSONHash and DuckDBJSONArray objects. It
+# follows Sequel's pg_json extension. It is opt-in: without it, the adapter's SQL and
+# values do not change.
+#
+# Load the constructor globally:
+#
+#   Sequel.extension :duckdb_json
+#
+# Load it into a Database, which also converts results, schema types and model
+# typecasting for that Database:
+#
+#   DB.extension :duckdb_json
+#
+# Write a JSON value:
+#
+#   DB[:t].insert(j: Sequel.duckdb_json("a" => 1))  # INSERT ... VALUES ('{"a":1}'::JSON)
+#   Sequel.duckdb_json([1, 2])                      # '[1,2]'::JSON
+#
+# A plain Ruby Hash or Array still means a condition or an IN list in Sequel. Only a
+# DuckDBJSONHash or DuckDBJSONArray is written as JSON.
+#
+# With the extension loaded into a Database:
+#
+# * JSON result values are parsed. A JSON object becomes a DuckDBJSONHash with String
+#   keys, and a JSON array becomes a DuckDBJSONArray. A JSON string, number, boolean
+#   or null becomes the plain Ruby value. Only the top-level value is wrapped. A
+#   VARCHAR column is not parsed, even when it holds JSON text.
+# * db.schema reports a JSON column as :json. With the duckdb_array extension also
+#   loaded, it reports a JSON[] column as :json_array.
+# * A model typecasts a Hash or an Array assigned to a JSON column to the wrapper. It
+#   parses a String as JSON text.
+
+require "delegate"
+require "json"
+
+module Sequel
+  module DuckDB
+    # Included in DuckDBJSONHash and DuckDBJSONArray, so that one check finds both.
+    module DuckDBJSONObject
+      # The wrapper for a parsed JSON value. A JSON string, number, boolean or null
+      # stays the plain Ruby value.
+      def self.wrap(value)
+        case value
+        when Hash then DuckDBJSONHash.new(value)
+        when Array then DuckDBJSONArray.new(value)
+        else value
+        end
+      end
+
+      # Not Sequel.parse_json: it passes create_additions, which json 3.0 rejects.
+      # JSON.parse creates no additions by default.
+      def self.parse(json)
+        wrap(::JSON.parse(json))
+      end
+
+      module DatabaseMethods
+        def self.extended(db)
+          db.instance_exec do
+            @schema_type_classes[:json] = [ DuckDBJSONObject ]
+            add_conversion_proc(:json, DuckDBJSONObject.method(:parse)) if respond_to?(:add_conversion_proc)
+          end
+        end
+
+        private
+
+        def map_duckdb_type_to_sequel(duckdb_type)
+          duckdb_type.casecmp?("JSON") ? :json : super
+        end
+
+        def typecast_value_json(value)
+          case value
+          when DuckDBJSONObject then value
+          when Hash, Array then DuckDBJSONObject.wrap(value)
+          when String then parse_json_for_typecast(value)
+          else raise Sequel::InvalidValue, "invalid value for json type: #{value.inspect}"
+          end
+        end
+
+        def parse_json_for_typecast(json)
+          DuckDBJSONObject.parse(json)
+        rescue ::JSON::ParserError => e
+          raise Sequel.convert_exception_class(e, Sequel::InvalidValue)
+        end
+      end
+
+      # '{"a":1}'::JSON. The JSON text is a string literal, cast to JSON.
+      def sql_literal_append(ds, sql)
+        ds.literal_append(sql, Sequel.object_to_json(__getobj__))
+        sql << "::JSON"
+      end
+    end
+
+    # A Hash that Sequel writes as a DuckDB JSON object.
+    class DuckDBJSONHash < DelegateClass(Hash)
+      include DuckDBJSONObject
+
+      def initialize(hash)
+        super(hash.to_hash)
+      end
+    end
+
+    # An Array that Sequel writes as a DuckDB JSON array.
+    class DuckDBJSONArray < DelegateClass(Array)
+      include DuckDBJSONObject
+
+      def initialize(array)
+        super(array.to_ary)
+      end
+    end
+  end
+
+  module SQL
+    module Builders
+      # Wraps a Hash or an Array, so that Sequel writes it as DuckDB JSON. A wrapper
+      # is returned as it is.
+      def duckdb_json(value)
+        case value
+        when Sequel::DuckDB::DuckDBJSONObject then value
+        when Hash, Array then Sequel::DuckDB::DuckDBJSONObject.wrap(value)
+        else raise Sequel::Error, "Sequel.duckdb_json needs a Hash or an Array: #{value.inspect}"
+        end
+      end
+    end
+  end
+
+  Database.register_extension(:duckdb_json, DuckDB::DuckDBJSONObject::DatabaseMethods)
+end

data/lib/sequel/extensions/duckdb_map.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+# The duckdb_map extension writes Ruby hashes to DuckDB MAP columns and reads them
+# back as Sequel::DuckDB::DuckDBMap objects. Sequel's pg_hstore extension is the
+# PostgreSQL counterpart, but a DuckDB MAP has typed keys and values, not only
+# strings. It is opt-in: without it, the adapter's SQL and values do not change.
+#
+# Load the constructor globally:
+#
+#   Sequel.extension :duckdb_map
+#
+# Load it into a Database, which also converts results, schema types and model
+# typecasting for that Database:
+#
+#   DB.extension :duckdb_map
+#
+# Write a map:
+#
+#   DB[:t].insert(m: Sequel.duckdb_map("k" => 1))  # INSERT ... VALUES (MAP {'k': 1})
+#   Sequel.duckdb_map({})                          # MAP {}
+#
+# A plain Ruby Hash still means a condition in Sequel. Only a DuckDBMap is written as
+# a DuckDB map. A struct or list inside a map must itself be a wrapper.
+#
+# With the extension loaded into a Database:
+#
+# * MAP result values are DuckDBMap objects. The keys keep their Ruby type: a
+#   MAP(VARCHAR, INTEGER) has String keys. Only the top-level value is wrapped. A
+#   value nested in it stays as ruby-duckdb returns it.
+# * db.schema reports a MAP column, for example MAP(VARCHAR, INTEGER), as :map.
+# * A model typecasts a Hash assigned to a MAP column to a DuckDBMap.
+
+require "delegate"
+
+module Sequel
+  module DuckDB
+    # A Hash that Sequel writes as a DuckDB map literal.
+    class DuckDBMap < DelegateClass(Hash)
+      # A MAP db_type. MAP(VARCHAR, INTEGER)[] is a list, so it does not match.
+      DB_TYPE = /\AMAP\(.*\)\z/im
+
+      module DatabaseMethods
+        def self.extended(db)
+          db.instance_exec do
+            @schema_type_classes[:map] = DuckDBMap
+            add_conversion_proc(:map, DuckDBMap.method(:new)) if respond_to?(:add_conversion_proc)
+          end
+        end
+
+        private
+
+        def map_duckdb_type_to_sequel(duckdb_type)
+          duckdb_type.match?(DB_TYPE) ? :map : super
+        end
+
+        def typecast_value_map(value)
+          raise Sequel::InvalidValue, "invalid value for map type: #{value.inspect}" unless value.is_a?(Hash) || value.is_a?(DuckDBMap)
+
+          DuckDBMap.new(value)
+        end
+      end
+
+      def initialize(hash)
+        super(hash.to_hash)
+      end
+
+      # MAP {'k': 1}. Each key and each value goes through the dataset's
+      # literal_append.
+      def sql_literal_append(ds, sql)
+        sql << "MAP {"
+        each_with_index do |(key, value), i|
+          sql << ", " unless i.zero?
+          ds.literal_append(sql, key)
+          sql << ": "
+          ds.literal_append(sql, value)
+        end
+        sql << "}"
+      end
+    end
+  end
+
+  module SQL
+    module Builders
+      # Wraps a Hash, so that Sequel writes it as a DuckDB map. The keys keep their
+      # type.
+      def duckdb_map(hash)
+        Sequel::DuckDB::DuckDBMap.new(hash)
+      end
+    end
+  end
+
+  Database.register_extension(:duckdb_map, DuckDB::DuckDBMap::DatabaseMethods)
+end