sequelizer 0.1.4 → 0.2.0

data/lib/sequel/extensions/platform.rb

@@ -0,0 +1,301 @@
+# frozen_string_literal: true
+
+#
+# The platform extension provides a unified interface for platform-specific
+# database behavior. It uses KVCSV configuration files to define capabilities
+# and preferences, with Ruby classes only for function translations.
+#
+#   DB.extension :platform
+#   DB.platform.supports?(:cte)           # => true (from config)
+#   DB.platform.prefers?(:cte)            # => true (from config)
+#   DB.platform[:interim_schema]          # => nil (from config)
+#   DB.platform.date_diff(from, to)       # => Sequel expression (from code)
+#
+# Configuration is loaded from CSV files in config/platforms/:
+#   - base.csv: conservative defaults
+#   - rdbms/<adapter>.csv: RDBMS-specific overrides
+#   - Additional configs can be stacked via platform_configs option
+#
+# Related module: Sequel::Platform
+
+require 'kvcsv'
+
+module Sequel
+
+  # The Platform module provides database platform abstraction through
+  # configuration-driven capabilities and code-driven function translations.
+  module Platform
+
+    # Base platform class with KVCSV config loading and default implementations.
+    # Subclasses override function translation methods for platform-specific SQL.
+    class Base
+
+      attr_reader :db, :config
+
+      # Initialize platform with database connection and config paths.
+      #
+      # @param db [Sequel::Database] The database connection
+      # @param config_paths [Array<String>] Paths to CSV config files (stacked in order)
+      def initialize(db, *config_paths)
+        @db = db
+        @config = config_paths.empty? ? {} : KVCSV::Settings.new(*config_paths)
+      end
+
+      # Check if the platform supports a feature.
+      #
+      # @param feature [Symbol] Feature name (e.g., :cte, :temp_tables)
+      # @return [Boolean] true if supported
+      #
+      # @example
+      #   platform.supports?(:cte)           # checks supports_cte in config
+      #   platform.supports?(:temp_tables)   # checks supports_temp_tables in config
+      def supports?(feature)
+        config[:"supports_#{feature}"] || false
+      end
+
+      # Check if the platform prefers a feature (may support but not prefer).
+      #
+      # @param feature [Symbol] Feature name (e.g., :cte, :parquet)
+      # @return [Boolean] true if preferred
+      #
+      # @example
+      #   platform.prefers?(:cte)      # Spark supports CTEs but doesn't prefer them
+      #   platform.prefers?(:parquet)  # Spark prefers parquet format
+      def prefers?(feature)
+        config[:"prefers_#{feature}"] || false
+      end
+
+      # Access arbitrary config values.
+      #
+      # @param key [Symbol] Config key
+      # @return [Object] Config value or nil
+      #
+      # @example
+      #   platform[:interim_schema]           # => "scratch"
+      #   platform[:schema_switching_method]  # => "use"
+      def [](key)
+        config[key]
+      end
+
+      # Fetch config value with default.
+      #
+      # @param key [Symbol] Config key
+      # @param default [Object] Default value if key not found
+      # @return [Object] Config value or default
+      def fetch(key, default = nil)
+        config.respond_to?(:fetch) ? config.fetch(key, default) : (config[key] || default)
+      end
+
+      # ---- Function translations (override in subclasses) ----
+
+      # Calculate date difference between two dates.
+      #
+      # @param from [Symbol, Sequel::SQL::Expression] Start date
+      # @param to [Symbol, Sequel::SQL::Expression] End date
+      # @return [Sequel::SQL::Expression] Date difference expression
+      def date_diff(from, to)
+        Sequel.function(:datediff, from, to)
+      end
+
+      # Cast expression to date type.
+      #
+      # @param expr [Object] Expression to cast
+      # @return [Sequel::SQL::Cast] Cast expression
+      def cast_date(expr)
+        Sequel.cast(expr, Date)
+      end
+
+      # Parse string to date with format.
+      #
+      # @param value [Object] String value to parse
+      # @param format [String] Date format string
+      # @return [Sequel::SQL::Expression] Parsed date expression
+      def str_to_date(value, format)
+        Sequel.function(:to_date, value, format)
+      end
+
+      # Calculate days between two dates.
+      #
+      # @param from [Symbol, Sequel::SQL::Expression] Start date
+      # @param to [Symbol, Sequel::SQL::Expression] End date
+      # @return [Sequel::SQL::Expression] Days between expression
+      def days_between(from, to)
+        date_diff(from, to)
+      end
+
+    end
+
+    # PostgreSQL platform with Postgres-specific function translations.
+    class Postgres < Base
+
+      def date_diff(from, to)
+        # Postgres uses date subtraction
+        Sequel.lit('(? - ?)', to, from)
+      end
+
+      def days_between(from, to)
+        # Postgres date subtraction returns integer days
+        Sequel.lit('(? - ?)', to, from)
+      end
+
+    end
+
+    # Spark platform with Spark SQL-specific function translations.
+    class Spark < Base
+
+      def date_diff(from, to)
+        # Spark datediff has reversed argument order (end, start)
+        Sequel.function(:datediff, to, from)
+      end
+
+      def str_to_date(value, format)
+        Sequel.function(:to_date, Sequel.cast_string(value), format)
+      end
+
+    end
+
+    # Snowflake platform with Snowflake-specific function translations.
+    class Snowflake < Base
+
+      def date_diff(from, to)
+        # Snowflake requires unit parameter
+        Sequel.function(:datediff, 'day', from, to)
+      end
+
+      def days_between(from, to)
+        Sequel.function(:datediff, 'day', from, to)
+      end
+
+    end
+
+    # Athena platform (Presto/Trino based) with Athena-specific function translations.
+    class Athena < Base
+
+      def date_diff(from, to)
+        # Athena/Presto uses date_diff with unit
+        Sequel.function(:date_diff, 'day', from, to)
+      end
+
+      def days_between(from, to)
+        Sequel.function(:date_diff, 'day', from, to)
+      end
+
+    end
+
+    # Map adapter schemes to platform classes
+    PLATFORM_CLASSES = {
+      postgres: Postgres,
+      postgresql: Postgres,
+      spark: Spark,
+      athena: Athena,
+      presto: Athena,
+      trino: Athena,
+      snowflake: Snowflake,
+    }.freeze
+
+    # Map adapter schemes to config file names
+    ADAPTER_CONFIG_NAMES = {
+      postgres: 'postgres',
+      postgresql: 'postgres',
+      spark: 'spark',
+      athena: 'athena',
+      presto: 'athena',
+      trino: 'athena',
+      snowflake: 'snowflake',
+    }.freeze
+
+    class << self
+
+      # Find the config directory, searching gem paths
+      def config_dir
+        @config_dir ||= find_config_dir
+      end
+
+      # Allow overriding config dir for testing
+      attr_writer :config_dir
+
+      private
+
+      def find_config_dir
+        # Check relative to this file (gem's config)
+        gem_config = File.expand_path('../../../config/platforms', __dir__)
+        return gem_config if File.directory?(gem_config)
+
+        # Fallback to working directory
+        local_config = File.join(Dir.pwd, 'config/platforms')
+        return local_config if File.directory?(local_config)
+
+        nil
+      end
+
+    end
+
+    # Build config paths for the given adapter
+    #
+    # @param adapter_scheme [Symbol] Database adapter scheme
+    # @param extra_configs [Array<String>] Additional config paths to stack
+    # @return [Array<String>] Ordered config paths
+    def self.config_paths_for(adapter_scheme, extra_configs = [])
+      paths = []
+
+      if config_dir
+        base_config = File.join(config_dir, 'base.csv')
+        paths << base_config if File.exist?(base_config)
+
+        adapter_name = ADAPTER_CONFIG_NAMES[adapter_scheme]
+        if adapter_name
+          rdbms_config = File.join(config_dir, 'rdbms', "#{adapter_name}.csv")
+          paths << rdbms_config if File.exist?(rdbms_config)
+        end
+      end
+
+      paths.concat(extra_configs.select { |p| File.exist?(p) })
+      paths
+    end
+
+    # Build platform instance for database
+    #
+    # @param db [Sequel::Database] Database connection
+    # @param extra_configs [Array<String>] Additional config paths
+    # @return [Platform::Base] Platform instance
+    def self.build_platform(db, extra_configs = [])
+      adapter = effective_adapter(db)
+      platform_class = PLATFORM_CLASSES.fetch(adapter, Base)
+      config_paths = config_paths_for(adapter, extra_configs)
+      platform_class.new(db, *config_paths)
+    end
+
+    # Detect the effective adapter for platform selection.
+    # For mock databases, use database_type or host option; otherwise use adapter_scheme.
+    #
+    # @param db [Sequel::Database] Database connection
+    # @return [Symbol] Effective adapter type
+    def self.effective_adapter(db)
+      return db.adapter_scheme unless db.adapter_scheme == :mock
+
+      # Mock databases: try database_type first (set for known types like postgres, spark)
+      db_type = db.database_type
+      return db_type if db_type && db_type != :mock
+
+      # Fall back to host option (for unknown types like snowflake, athena)
+      db.opts[:host]
+    end
+
+    # Extension hook - called when extension is loaded
+    def self.extended(db)
+      extra_configs = db.opts[:platform_configs] || []
+      db.instance_variable_set(:@platform, build_platform(db, extra_configs))
+    end
+
+    # Access the platform instance
+    #
+    # @return [Platform::Base] Platform instance for this database
+    def platform
+      @platform
+    end
+
+  end
+
+  Database.register_extension(:platform, Platform)
+
+end

data/lib/sequel/extensions/settable.rb

@@ -1,5 +1,56 @@
+# frozen_string_literal: true
+
+#
+# The settable extension adds a convenient +set+ method to database connections
+# for executing SET statements with key-value pairs. This is particularly useful
+# for configuring database session parameters.
+#
+#   DB.extension :settable
+#   DB.set(search_path: 'public', timezone: 'UTC')
+#   # Executes: SET search_path=public
+#   #           SET timezone=UTC
+#
+#   DB.set(work_mem: '256MB')
+#   # Executes: SET work_mem=256MB
+#
+# The extension works with any database adapter and supports various value types
+# including strings, numbers, booleans, and nil values.
+#
+# Related module: Sequel::Settable
+
 module Sequel
+
+  # The Settable module provides database configuration functionality through
+  # SET statements. When loaded as an extension, it adds the +set+ method to
+  # database connections.
   module Settable
+
+    # Execute SET statements for the given options hash.
+    #
+    # Each key-value pair in the options hash is converted to a SET statement
+    # and executed against the database. Multiple options result in multiple
+    # SET statements being executed in sequence.
+    #
+    # @param opts [Hash] Hash of configuration options to set
+    # @option opts [Object] key The configuration parameter name
+    # @option opts [Object] value The value to set for the parameter
+    #
+    # @example Set a single parameter
+    #   DB.set(timezone: 'UTC')
+    #   # Executes: SET timezone=UTC
+    #
+    # @example Set multiple parameters
+    #   DB.set(search_path: 'public', work_mem: '256MB')
+    #   # Executes: SET search_path=public
+    #   #           SET work_mem=256MB
+    #
+    # @example Different value types
+    #   DB.set(port: 5432, autocommit: true, custom_setting: nil)
+    #   # Executes: SET port=5432
+    #   #           SET autocommit=true
+    #   #           SET custom_setting=
+    #
+    # @return [void]
     def set(opts = {})
       set_sql(opts).each do |sql|
         run(sql)
@@ -8,10 +59,23 @@ module Sequel
 
     private
 
+    # Generate SET SQL statements from options hash.
+    #
+    # Converts each key-value pair in the options hash into a SET SQL statement
+    # string. This is a private helper method used internally by the +set+ method.
+    #
+    # @param opts [Hash] Hash of options to convert to SET statements
+    # @return [Array<String>] Array of SET SQL statement strings
+    #
+    # @example
+    #   set_sql(timezone: 'UTC', port: 5432)
+    #   # => ["SET timezone=UTC", "SET port=5432"]
     def set_sql(opts)
       opts.map { |k, v| "SET #{k}=#{v}" }
     end
+
   end
 
   Database.register_extension(:settable, Settable)
+
 end

data/lib/sequel/extensions/sql_recorder.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+# == Overview
+#
+# The sql_recorder extension records each SQL statement sent to the database
+# in a thread-safe array accessible via the +sql_recorder+ method.
+#
+# == Usage
+#
+#   DB.extension :sql_recorder
+#   DB[:users].all
+#   DB[:posts].where(id: 1).first
+#
+#   # Access recorded SQL statements
+#   DB.sql_recorder
+#   # => ["SELECT * FROM users", "SELECT * FROM posts WHERE (id = 1) LIMIT 1"]
+#
+#   # Clear the recorded statements
+#   DB.sql_recorder.clear
+#
+# == Thread Safety
+#
+# The extension is thread-safe and uses a mutex to synchronize access to the
+# SQL recording array when multiple threads are executing queries simultaneously.
+#
+# == Compatibility
+#
+# This extension is designed to work alongside mock databases and other SQL
+# recording mechanisms. It uses the method name +sql_recorder+ to avoid
+# conflicts with existing +sqls+ methods that may be present in test frameworks.
+#
+# Related module: Sequel::SqlRecorder
+
+module Sequel
+
+  # Extension module that adds SQL recording capabilities to Sequel databases.
+  # When included, it provides a +sql_recorder+ method that returns an array
+  # of all SQL statements executed against the database.
+  module SqlRecorder
+
+    # Returns the array of recorded SQL statements.
+    #
+    # The array accumulates all SQL statements sent to the database since the
+    # extension was loaded or since the last time +clear+ was called on the array.
+    #
+    # @return [Array<String>] array of SQL statement strings
+    # @example
+    #   DB.extension :sql_recorder
+    #   DB[:users].all
+    #   DB.sql_recorder #=> ["SELECT * FROM users"]
+    attr_reader :sql_recorder
+
+    # Intercepts SQL execution to record statements.
+    #
+    # This method overrides Sequel's +log_connection_yield+ to capture each SQL
+    # statement in a thread-safe manner before delegating to the parent implementation.
+    #
+    # @param sql [String] the SQL statement being executed
+    # @param conn [Object] the database connection object
+    # @param args [Object] additional arguments (optional)
+    # @return [Object] result from the parent +log_connection_yield+ method
+    def log_connection_yield(sql, conn, args = nil)
+      @sql_recorder_mutex.synchronize { sql_recorder.push(sql) }
+      super
+    end
+
+    # Initializes the SQL recording infrastructure when the extension is loaded.
+    #
+    # Sets up the mutex for thread-safe access and initializes the SQL recording
+    # array. This method is automatically called when the extension is loaded
+    # via +DB.extension :sql_recorder+.
+    #
+    # @param db [Sequel::Database] the database instance being extended
+    def self.extended(db)
+      db.instance_exec do
+        @sql_recorder_mutex ||= Mutex.new
+        @sql_recorder ||= []
+      end
+    end
+
+  end
+
+  Database.register_extension(:sql_recorder, SqlRecorder)
+
+end

data/lib/sequel/extensions/unionize.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Sequel
+
+  # Provides efficient handling of large UNION operations.
+  #
+  # The unionize extension allows combining many datasets through UNION operations
+  # by chunking them into manageable temporary tables or views. This is particularly
+  # useful when dealing with databases that have limitations on the number of UNION
+  # operations in a single query (e.g., Spark SQL, DuckDB).
+  #
+  # @example Load the extension
+  #   DB.extension :unionize
+  #
+  # @example Basic usage
+  #   DB.unionize([dataset1, dataset2, dataset3, dataset4])
+  #
+  # @example With options
+  #   DB.unionize(datasets, chunk_size: 50, all: true, temp_table_prefix: 'my_union')
+  module Unionize
+
+    # Handles the chunking and union of multiple datasets.
+    #
+    # This class manages the process of splitting a large collection of datasets
+    # into smaller chunks, creating temporary tables/views for each chunk, and
+    # then recursively combining them until a single unified dataset is produced.
+    class Unionizer
+
+      # Default number of datasets to combine in each chunk
+      DEFAULT_CHUNK_SIZE = 100
+
+      # Represents a chunk of datasets to be combined via UNION.
+      #
+      # Each chunk handles a subset of datasets, creates a temporary table/view
+      # for the combined result, and provides access to the unified dataset.
+      class Chunk
+
+        # @!attribute [r] db
+        #   @return [Sequel::Database] The database connection
+        # @!attribute [r] dses
+        #   @return [Array<Sequel::Dataset>] The datasets in this chunk
+        # @!attribute [r] opts
+        #   @return [Hash] Options for the union operation
+        attr_reader :db, :dses, :opts
+
+        # Creates a new chunk instance.
+        #
+        # @param db [Sequel::Database] The database connection
+        # @param dses [Array<Sequel::Dataset>] The datasets to combine
+        # @param opts [Hash] Options for the union operation
+        def initialize(db, dses, opts)
+          @db = db
+          @dses = dses
+          @opts = opts
+        end
+
+        # Returns the unified dataset created by combining all datasets in this chunk.
+        #
+        # @return [Sequel::Dataset] The combined dataset
+        def union
+          @union ||= dses.reduce { |a, b| a.union(b, all: opts[:all], from_self: opts[:from_self]) }
+        end
+
+        # Generates a unique name for the temporary table/view.
+        #
+        # The name is based on a hash of the SQL query to ensure uniqueness
+        # and avoid collisions when multiple unionize operations are running.
+        #
+        # @return [Symbol] The temporary table/view name
+        def name
+          @name ||= :"#{opts[:temp_table_prefix]}_#{Digest::SHA1.hexdigest(union.sql)}"
+        end
+
+        # Creates a temporary table or view for this chunk's union result.
+        #
+        # The method used depends on the database type:
+        # - Spark: Creates a temporary view
+        # - DuckDB: Creates a temporary table
+        #
+        # @raise [RuntimeError] If the database type is not supported
+        # @return [void]
+        def create
+          if db.database_type == :spark
+            db.create_view(name, union, temp: true)
+          elsif db.database_type == :duckdb
+            db.create_table(name, temp: true, as: union)
+          else
+            raise "Unsupported database type: #{db.database_type}"
+          end
+        end
+
+      end
+
+      # @!attribute [r] db
+      #   @return [Sequel::Database] The database connection
+      attr_reader :db
+
+      # Creates a new Unionizer instance.
+      #
+      # @param db [Sequel::Database] The database connection
+      # @param ds_set [Array<Sequel::Dataset>] The datasets to combine
+      # @param opts [Hash] Options for the union operation
+      # @option opts [Integer] :chunk_size (100) Number of datasets per chunk
+      # @option opts [String] :temp_table_prefix ('temp_union') Prefix for temporary tables
+      # @option opts [Boolean] :all (false) Use UNION ALL instead of UNION
+      # @option opts [Boolean] :from_self (true) Wrap individual datasets in subqueries
+      def initialize(db, ds_set, opts = {})
+        @db = db
+        @ds_set = ds_set
+        @opts = opts
+        opts[:chunk_size] ||= DEFAULT_CHUNK_SIZE
+        opts[:temp_table_prefix] ||= 'temp_union'
+        opts[:all] ||= false
+        opts[:from_self] = opts.fetch(:from_self, true)
+      end
+
+      # Performs the unionization of datasets.
+      #
+      # This method recursively chunks the datasets, creates temporary tables/views
+      # for each chunk, and then combines them until a single dataset remains.
+      #
+      # @param dses [Array<Sequel::Dataset>] The datasets to combine (defaults to @ds_set)
+      # @return [Sequel::Dataset] The final combined dataset
+      def unionize(dses = @ds_set)
+        chunks = dses.each_slice(@opts[:chunk_size]).map do |chunk_of_dses|
+          Chunk.new(db, chunk_of_dses, @opts)
+        end
+
+        return chunks.first.union if chunks.size == 1
+
+        unionize(chunks.each(&:create).map { |chunk| db[chunk.name] })
+      end
+
+    end
+
+    # Efficiently combines multiple datasets using UNION operations.
+    #
+    # This method handles large numbers of datasets by chunking them into
+    # manageable groups, creating temporary tables/views for intermediate
+    # results, and recursively combining them until a single dataset is produced.
+    #
+    # @param ds_set [Array<Sequel::Dataset>] The datasets to combine via UNION
+    # @param opts [Hash] Options for the union operation
+    # @option opts [Integer] :chunk_size (100) Number of datasets to combine in each chunk
+    # @option opts [String] :temp_table_prefix ('temp_union') Prefix for temporary table names
+    # @option opts [Boolean] :all (false) Use UNION ALL instead of UNION (keeps duplicates)
+    # @option opts [Boolean] :from_self (true) Wrap individual datasets in subqueries
+    #
+    # @return [Sequel::Dataset] The combined dataset
+    #
+    # @example Basic union of datasets
+    #   db.unionize([ds1, ds2, ds3, ds4])
+    #
+    # @example Union all with custom chunk size
+    #   db.unionize(datasets, all: true, chunk_size: 50)
+    #
+    # @example Custom temporary table prefix
+    #   db.unionize(datasets, temp_table_prefix: 'my_union_batch')
+    def unionize(ds_set, opts = {})
+      Unionizer.new(self, ds_set, opts).unionize
+    end
+
+  end
+
+  Database.register_extension(:unionize, Unionize)
+
+end

data/lib/sequel/extensions/usable.rb

@@ -1,15 +1,44 @@
 module Sequel
+
+  # = Usable
+  #
+  # Sequel extension that provides a convenient +use+ method for switching
+  # the current database/schema context. This is particularly useful for
+  # databases that support the USE statement like MySQL, SQL Server, and
+  # some big data engines.
+  #
+  # @example
+  #   db.extension :usable
+  #   db.use(:my_schema)
+  #   # Executes: USE `my_schema`
   module Usable
+
+    # Switches to the specified database or schema.
+    #
+    # Executes a USE statement to change the current database context.
+    # The schema name is properly quoted using the database's identifier
+    # quoting rules.
+    #
+    # @param schema_name [Symbol, String] the name of the schema/database to use
+    # @example
+    #   db.use(:production_db)
+    #   db.use('test_schema')
     def use(schema_name)
       run(use_sql(schema_name))
     end
 
     private
 
+    # Generates the USE SQL statement for the given schema name.
+    #
+    # @param schema_name [Symbol, String] the schema name to use
+    # @return [String] the USE SQL statement
     def use_sql(schema_name)
-      "USE #{quote_identifier(schema_name)}"
+      "USE #{literal(schema_name)}"
     end
+
   end
 
   Database.register_extension(:usable, Usable)
+
 end