sequelizer 0.1.4 → 0.2.0

data/lib/sequel/extensions/db_opts.rb

@@ -1,41 +1,102 @@
 module Sequel
+
+  # = DbOpts
+  #
+  # Sequel extension that provides database-specific options handling.
+  # This extension allows setting database-specific configuration options
+  # that get applied during connection establishment.
+  #
+  # The extension looks for options in the database configuration that match
+  # the pattern `{database_type}_db_opt_{option_name}` and converts them to
+  # appropriate SQL SET statements.
+  #
+  # @example
+  #   # For PostgreSQL, options like:
+  #   postgres_db_opt_work_mem: '256MB'
+  #   postgres_db_opt_shared_preload_libraries: 'pg_stat_statements'
+  #
+  #   # Will generate:
+  #   SET work_mem='256MB'
+  #   SET shared_preload_libraries='pg_stat_statements'
   module DbOpts
+
+    # Handles extraction and application of database-specific options.
     class DbOptions
-      attr :db
+
+      # @!attribute [r] db
+      #   @return [Sequel::Database] the database instance
+      attr_reader :db
+
+      # Creates a new DbOptions instance for the given database.
+      #
+      # @param db [Sequel::Database] the database to configure
       def initialize(db)
         db.extension :settable
         @db = db
       end
 
+      # Returns a hash of database-specific options extracted from the database configuration.
+      #
+      # @return [Hash] hash of option names to values
       def to_hash
         @_to_hash ||= extract_db_opts
       end
 
+      # Extracts database-specific options from the database configuration.
+      #
+      # Looks for options matching the pattern `{database_type}_db_opt_{option_name}`
+      # and extracts the option name and value.
+      #
+      # @return [Hash] extracted options with symbolic keys
       def extract_db_opts
         opt_regexp = /^#{db.database_type}_db_opt_/i
 
-        Hash[db.opts.select { |k, _| k.to_s.match(opt_regexp) }.map { |k, v| [k.to_s.gsub(opt_regexp, '').to_sym, prep_value(k, v)] }]
+        db.opts.select do |k, _|
+          k.to_s.match(opt_regexp)
+        end.to_h { |k, v| [k.to_s.gsub(opt_regexp, '').to_sym, prep_value(k, v)] }
       end
 
+      # Applies the database options to the given connection.
+      #
+      # Executes the SQL statements generated from the options on the connection.
+      #
+      # @param c [Object] the database connection
       def apply(c)
         sql_statements.each do |stmt|
           db.send(:log_connection_execute, c, stmt)
         end
       end
 
-      def prep_value(k, v)
-        v =~ /\W/ ? db.literal("#{v}") : v
+      # Prepares a value for use in SQL statements.
+      #
+      # Values containing non-word characters are treated as literals and quoted,
+      # while simple values are used as-is.
+      #
+      # @param _k [Symbol] the option key (unused)
+      # @param v [Object] the option value
+      # @return [String] the prepared value
+      def prep_value(_k, v)
+        v =~ /\W/ ? db.literal(v.to_s) : v
       end
 
+      # Generates SQL SET statements for the database options.
+      #
+      # @return [Array<String>] array of SQL SET statements
       def sql_statements
         db.send(:set_sql, to_hash)
       end
+
     end
 
+    # Returns a DbOptions instance for this database.
+    #
+    # @return [DbOptions] the database options handler
     def db_opts
       @db_opts ||= DbOptions.new(self)
     end
+
   end
 
   Database.register_extension(:db_opts, DbOpts)
+
 end

data/lib/sequel/extensions/funky.rb

@@ -0,0 +1,136 @@
+module Sequel
+
+  module Funky
+
+    class FunkyBase
+
+      def initialize(db)
+        @db = db
+      end
+
+      def to_strptime(format)
+        return format if format =~ /%/
+
+        format.gsub('yyyy', '%Y').gsub('MM', '%m').gsub('dd', '%d')
+      end
+
+      def from_strptime(format)
+        return format unless format =~ /%/
+
+        format.gsub('%Y', 'yyyy').gsub('%m', 'MM').gsub('%d', 'dd')
+      end
+
+    end
+
+    class FunkySpark < FunkyBase
+
+      def str_to_date(value, format, try: false) # rubocop:disable Lint/UnusedMethodArgument
+        Sequel.function(:to_date, Sequel.cast_string(value), format)
+      end
+
+      def hash(*)
+        Sequel.function(:xxhash64, *)
+      end
+
+      def make_json_column(ds, key_column, value_column)
+        json_object_col = Sequel.function(
+          :named_struct,
+          'key',
+          key_column,
+          'value',
+          value_column,
+        ).then do |json_object_col|
+          Sequel.function(
+            :collect_list,
+            json_object_col,
+          )
+        end.then do |list_col|
+          Sequel.function(
+            :map_from_entries,
+            list_col,
+          )
+        end.then do |map_from_entries_col|
+          Sequel.function(
+            :to_json,
+            map_from_entries_col,
+          )
+        end
+        ds.from_self
+          .select(json_object_col)
+          .limit(1)
+      end
+
+      def collect_list(column)
+        Sequel.function(:collect_list, column)
+      end
+
+    end
+
+    class FunkyDuckDB < FunkyBase
+
+      def str_to_date(value, format, try: false)
+        strptime_func = try ? :try_strptime : :strptime
+        Sequel.function(strptime_func, Sequel.cast_string(value), to_strptime(format))
+      end
+
+      def hash(*)
+        Sequel.function(:hash, concat(*)).then do |hash_val|
+          Sequel.case(
+            { hash_val <= 9_223_372_036_854_775_807 => hash_val },
+            hash_val - 18_446_744_073_709_551_616,
+          ).cast(:bigint)
+        end
+      end
+
+      def concat(*)
+        Sequel.function(:concat, *)
+      end
+
+      def make_json_column(ds, key_column, value_column)
+        json_object_col = Sequel.function(
+          :json_object,
+          key_column,
+          value_column,
+        ).then do |json_object_col|
+          Sequel.function(
+            :list,
+            json_object_col,
+          )
+        end
+        ds.from_self
+          .select(json_object_col)
+          .limit(1)
+      end
+
+      def collect_list(column)
+        Sequel.function(:list, column)
+      end
+
+    end
+
+    def self.extended(db)
+      db.instance_exec do
+        @funky = get_funky(db.database_type)
+      end
+    end
+
+    def funky
+      @funky
+    end
+
+    def get_funky(database_type)
+      case database_type
+      when :spark
+        FunkySpark.new(self)
+      when :duckdb
+        FunkyDuckDB.new(self)
+      else
+        raise "No known functions for #{database_type}"
+      end
+    end
+
+  end
+
+  Database.register_extension(:funky, Funky)
+
+end

data/lib/sequel/extensions/make_readyable.rb

@@ -1,60 +1,99 @@
 module Sequel
+
+  # = MakeReadyable
+  #
+  # Sequel extension that provides database readiness functionality,
+  # primarily geared towards Spark SQL-based databases. This extension
+  # allows setting up temporary views and schema configurations to prepare
+  # a database for use.
+  #
+  # @example Basic schema usage
+  #   db.extension :make_readyable
+  #   db.make_ready(use_schema: :my_schema)
+  #
+  # @example Search path with schema precedence
+  #   db.make_ready(search_path: [:schema1, :schema2])
+  #
+  # @example External file sources
+  #   db.make_ready(search_path: [Pathname.new('data.parquet')])
   module MakeReadyable
-    ##
-    # This method is primarily geared towards Spark SQL-based databases.
+
+    # Prepares the database by setting up schemas, views, and external data sources.
     #
+    # This method is primarily geared towards Spark SQL-based databases.
     # Given some options, prepares a set of views to represent a set
     # of tables across a collection of different schemas and external,
     # unmanaged tables.
     #
+    # @param opts [Hash] the options used to prepare the database
+    # @option opts [Symbol] :use_schema The schema to be used as the primary schema
+    # @option opts [Array] :search_path A set of symbols (schemas) or Pathnames (external files)
+    # @option opts [Array] :only Limit view creation to these tables only
+    # @option opts [Array] :except Skip view creation for these tables
+    #
+    # @example Set primary schema
     #   DB.make_ready(use_schema: :schema)
     #   # => USE `schema`
     #
-    # When using search_path, tables from previous schema override tables
-    # from the next schema.  This is analogous to the way Unix searches 
-    # the PATH variable for programs.
-    #
-    # Assuming the following tables: schema1.a, schema2.a, schema2.b
-    #
+    # @example Search path with precedence
+    #   # Assuming tables: schema1.a, schema2.a, schema2.b
     #   DB.make_ready(search_path: [:schema1, :schema2])
     #   # => CREATE TEMPORARY VIEW `a` AS SELECT * FROM `schema1`.`a;`
     #   # => CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b;`
     #
-    # When using Pathnames, the extension on the file becomes the format
-    # to try to read from the file.  
-    #
+    # @example External file sources
     #   DB.make_ready(search_path: [Pathname.new("c.parquet"), Pathname.new("d.orc")])
     #   # => CREATE TEMPORARY VIEW `c` USING parquet OPTIONS ('path'='c.parquet')
     #   # => CREATE TEMPORARY VIEW `d` USING orc OPTIONS ('path'='d.orc')
-    #
-    # @param [Hash] opts the options used to prepare the database
-    # @option opts [String] :use_schema The schema to be used as the primary schema
-    # @option opts [Array] :search_path A set of sympbols (to represent schemas) or Pathnames (to represent externally managed data files)
     def make_ready(opts = {})
       ReadyMaker.new(self, opts).run
     end
+
   end
 
-  private 
+  # = ReadyMaker
+  #
+  # Internal class that handles the actual database preparation logic.
+  # This class processes the make_ready options and executes the necessary
+  # SQL statements to set up schemas, views, and external data sources.
   class ReadyMaker
+
+    # @!attribute [r] db
+    #   @return [Sequel::Database] the database instance
+    # @!attribute [r] opts
+    #   @return [Hash] the preparation options
     attr_reader :db, :opts
 
+    # Creates a new ReadyMaker instance.
+    #
+    # @param db [Sequel::Database] the database to prepare
+    # @param opts [Hash] the preparation options
     def initialize(db, opts)
       @db = db
       @opts = opts
     end
-  
+
+    # Executes the database preparation process.
+    #
+    # This method handles:
+    # 1. Setting the primary schema if specified
+    # 2. Processing the search path to create views
+    # 3. Handling table filtering (only/except options)
     def run
       if opts[:use_schema]
         db.extension :usable
         db.use(opts[:use_schema])
       end
       only_tables = Array(opts[:only])
-      created_views = (Array(opts[:except]) || [])
-      (opts[:search_path] || []).each do |schema|
-        schema = schema.is_a?(Pathname) ? schema : schema.to_sym
+      created_views = Array(opts[:except]) || []
+      (opts[:search_path] || []).flatten.each do |schema|
+        schema = schema.to_sym unless schema.is_a?(Pathname)
         source = get_source(db, schema)
-        tables = source.tables(schema: schema) - created_views
+        tables = if schema.is_a?(Pathname)
+                   source.tables - created_views
+                 else
+                   source.tables(schema: schema) - created_views
+                 end
         tables &= only_tables unless only_tables.empty?
         tables.each do |table|
           create_view(source, table, schema)
@@ -63,46 +102,123 @@ module Sequel
       end
     end
 
+    # Creates a temporary view for the given table.
+    #
+    # @param source [Object] the source (database or FileSourcerer)
+    # @param table [Symbol] the table name
+    # @param schema [Symbol, Pathname] the schema or file path
     def create_view(source, table, schema)
       if schema.to_s =~ %r{/}
         source.create_view(table, temp: true)
       else
+        # For schema-based tables, just create temporary views
+        # This extension is primarily for Spark SQL-based databases
         source.create_view(table, db[Sequel.qualify(schema, table)], temp: true)
       end
     end
 
+    # Gets the appropriate source handler for the schema.
+    #
+    # @param db [Sequel::Database] the database instance
+    # @param schema [Symbol, Pathname] the schema or file path
+    # @return [Sequel::Database, FileSourcerer] the source handler
     def get_source(db, schema)
       if schema.to_s =~ %r{/}
-        FileSourcerer.new(db, Pathname.new(schema))
+        FileSourcerer.new(db, Pathname.new(schema.to_s))
       else
         db
       end
     end
 
+    # = FileSourcerer
+    #
+    # Handles external file sources for the make_ready functionality.
+    # This class creates temporary views that read from external files
+    # like Parquet, ORC, etc.
     class FileSourcerer
+
+      # @!attribute [r] db
+      #   @return [Sequel::Database] the database instance
+      # @!attribute [r] schema
+      #   @return [Pathname] the file path
       attr_reader :db, :schema
+
+      # Creates a new FileSourcerer instance.
+      #
+      # @param db [Sequel::Database] the database instance
+      # @param schema [Pathname] the file path
       def initialize(db, schema)
         @db = db
         @schema = schema
       end
 
-      def tables(opts = {})
-        [schema.basename(".*").to_s.to_sym]
+      # Returns the table name derived from the file name.
+      #
+      # @param _opts [Hash] unused options parameter
+      # @return [Array<Symbol>] array containing the table name
+      def tables(_opts = {})
+        [schema.basename(schema.extname).to_s.to_sym]
       end
 
+      # Creates a temporary view that reads from the external file.
+      #
+      # @param table [Symbol] the table/view name
+      # @param opts [Hash] additional options to merge
       def create_view(table, opts = {})
-        db.create_view(table, {
-          temp: true,
-          using: format,
-          options: { path: schema.expand_path }
-        }.merge(opts))
+        case db.database_type
+        when :spark
+          # Spark SQL uses USING clause for external tables
+          db.create_view(table, {
+            temp: true,
+            using: format,
+            options: { path: schema.expand_path },
+          }.merge(opts))
+        when :duckdb
+          # DuckDB uses direct file reading with read_* functions
+          create_duckdb_view(table, opts)
+        else
+          raise Sequel::Error, "External file sources are not supported on #{db.database_type}"
+        end
       end
 
+      private
+
+      # Creates a view for DuckDB to read external files
+      #
+      # @param table [Symbol] the table/view name
+      # @param _opts [Hash] additional options to merge (currently unused for DuckDB)
+      def create_duckdb_view(table, _opts)
+        file_path = if schema.directory?
+                      schema.expand_path.join('**').join("*.#{format}").to_s
+                    else
+                      schema.expand_path.to_s
+                    end
+        read_function = case format
+                        when 'parquet'
+                          :read_parquet
+                        when 'csv'
+                          :read_csv_auto
+                        when 'json'
+                          :read_json_auto
+                        else
+                          raise Sequel::Error, "Unsupported file format '#{format}' for DuckDB"
+                        end
+
+        # DuckDB doesn't support TEMPORARY views, use regular CREATE VIEW
+        db.create_view(table, db.from(Sequel.function(read_function, file_path)))
+      end
+
+      # Returns the file format based on the file extension.
+      #
+      # @return [String] the file format (e.g., 'parquet', 'orc')
       def format
-        schema.extname[1..-1]
+        schema.extname[1..]
       end
+
     end
+
   end
 
   Database.register_extension(:make_readyable, MakeReadyable)
+
 end

data/lib/sequel/extensions/more_sql.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Sequel
+
+  # Provides additional SQL helper methods for database operations.
+  #
+  # The more_sql extension adds convenience methods for SQL operations that
+  # aren't covered by Sequel's core functionality, particularly schema-related
+  # operations like CREATE SCHEMA.
+  #
+  # @example Load the extension
+  #   DB.extension :more_sql
+  #
+  # @example Create a schema
+  #   DB.create_schema(:analytics)
+  #   # Executes: CREATE SCHEMA "analytics"
+  #
+  # @example Create schema if it doesn't exist
+  #   DB.create_schema(:staging, if_not_exists: true)
+  #   # Executes: CREATE SCHEMA IF NOT EXISTS "staging"
+  module MoreSql
+
+    # Creates a database schema.
+    #
+    # Generates and executes a CREATE SCHEMA statement with optional
+    # IF NOT EXISTS clause for idempotent schema creation.
+    #
+    # @param schema_name [Symbol, String] The name of the schema to create
+    # @param opts [Hash] Options for schema creation
+    # @option opts [Boolean] :if_not_exists (false) Only create the schema if it doesn't already exist
+    #
+    # @return [nil]
+    #
+    # @example Basic schema creation
+    #   DB.create_schema(:reports)
+    #   # Executes: CREATE SCHEMA "reports"
+    #
+    # @example Idempotent schema creation
+    #   DB.create_schema(:analytics, if_not_exists: true)
+    #   # Executes: CREATE SCHEMA IF NOT EXISTS "analytics"
+    #
+    # @example With string schema name
+    #   DB.create_schema('user_data')
+    #   # Executes: CREATE SCHEMA "user_data"
+    def create_schema(schema_name, opts = {})
+      run(create_schema_sql(schema_name, opts))
+    end
+
+    private
+
+    # Generates the SQL for creating a schema.
+    #
+    # Builds a CREATE SCHEMA statement with proper identifier quoting
+    # and optional IF NOT EXISTS clause.
+    #
+    # @param schema_name [Symbol, String] The name of the schema to create
+    # @param opts [Hash] Options for schema creation
+    # @option opts [Boolean] :if_not_exists (false) Include IF NOT EXISTS clause
+    #
+    # @return [String] The CREATE SCHEMA SQL statement
+    #
+    # @example
+    #   create_schema_sql(:test, if_not_exists: true)
+    #   # => 'CREATE SCHEMA IF NOT EXISTS "test"'
+    def create_schema_sql(schema_name, opts)
+      sql = 'CREATE SCHEMA '
+      sql += 'IF NOT EXISTS ' if opts[:if_not_exists]
+      sql += literal(schema_name)
+      sql
+    end
+
+  end
+
+  Database.register_extension(:more_sql, MoreSql)
+
+end