umbrellio-utils 1.12.0 → 1.13.0

data/lib/umbrellio_utils/click_house/backends/base.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module UmbrellioUtils
+  module ClickHouse
+    module Backends
+      # Abstract backend. Each concrete backend (Legacy for the `click_house`
+      # gem, Native for the `clickhouse-native` gem) implements the low-level
+      # ops (execute / query / insert / describe_table / server_version /
+      # tables / create_database / drop_database / config / logger) and a
+      # SERVER_ERROR constant used by `log_errors`.
+      class Base
+        include Singleton
+
+        # Concrete backends implement the low-level ops (execute / query /
+        # insert / describe_table / server_version / tables / admin_execute
+        # / config / logger) and define SERVER_ERROR.
+
+        def from(source, db_name: self.db_name)
+          ds =
+            case source
+            when Symbol
+              DB.from(db_name == self.db_name ? SQL[source] : SQL[db_name][source])
+            when nil
+              DB.dataset
+            else
+              DB.from(source)
+            end
+          ds.clone(ch: true)
+        end
+
+        def count(dataset)
+          query_value(dataset.select(SQL.ch_count))
+        end
+
+        def db_name
+          config.database.to_sym
+        end
+
+        def create_database(name, if_not_exists: false, cluster: nil, engine: nil)
+          admin_execute(
+            format(
+              "CREATE DATABASE %<exists>s %<name>s %<cluster>s %<engine>s",
+              exists: if_not_exists ? "IF NOT EXISTS" : "",
+              name:,
+              cluster: cluster ? "ON CLUSTER #{cluster}" : "",
+              engine: engine ? "ENGINE = #{engine}" : "",
+            ),
+          )
+        end
+
+        def drop_database(name, if_exists: false, cluster: nil)
+          admin_execute(
+            format(
+              "DROP DATABASE %<exists>s %<name>s %<cluster>s",
+              exists: if_exists ? "IF EXISTS" : "",
+              name:,
+              cluster: cluster ? "ON CLUSTER #{cluster}" : "",
+            ),
+          )
+        end
+
+        # Returns the `ON CLUSTER <name> [SYNC]` clause for DDL, or "" if
+        # `UmbrellioUtils.config.clickhouse_cluster` is blank or we're in
+        # a Rails test env. Test-env suppression saves hundreds of ms per
+        # DDL on a single-node CH (each ON CLUSTER op blocks waiting for
+        # replicas that don't exist). The cluster *name* is still used
+        # by callers like Distributed engine declarations, regardless of
+        # this clause.
+        def on_cluster(sync: false)
+          name = UmbrellioUtils.config.clickhouse_cluster
+          return "" if name.blank?
+          return "" if defined?(Rails) && Rails.env.test?
+          sync ? "ON CLUSTER #{name} SYNC" : "ON CLUSTER #{name}"
+        end
+
+        def truncate_table!(table_name, db_name: self.db_name)
+          execute("TRUNCATE TABLE #{db_name}.#{table_name} #{on_cluster(sync: true)}")
+        end
+
+        def drop_table!(table_name, db_name: self.db_name)
+          execute("DROP TABLE #{db_name}.#{table_name} #{on_cluster(sync: true)}")
+        end
+
+        def optimize_table!(table_name, db_name: self.db_name)
+          Timeout.timeout(UmbrellioUtils.config.ch_optimize_timeout) do
+            execute("OPTIMIZE TABLE #{db_name}.#{table_name} #{on_cluster} FINAL")
+          end
+        end
+
+        def parse_value(value, type:)
+          case type
+          when /Array/ then Array.wrap(value)
+          when /DateTime/
+            case value
+            when String then value.present? ? Time.zone.parse(value) : nil
+            else value
+            end
+          when /String/ then value&.to_s
+          else value
+          end
+        end
+
+        def pg_table_connection(table, schema: "public")
+          host = ENV["PGHOST"] || DB.opts[:host].presence || "localhost"
+          port = DB.opts[:port] || 5432
+          # Etc.getlogin returns "root" under non-TTY shells (e.g. rake from
+          # a CI runner), which is almost never a real PG role. Prefer $USER.
+          login = ENV["USER"].presence || Etc.getlogin
+          database = DB.opts[:database].presence || login
+          username = DB.opts[:user].presence || login
+          password = DB.opts[:password]
+          SQL.func(:postgresql, "#{host}:#{port}", database, table, username, password, schema)
+        end
+
+        def populate_temp_table!(temp_table_name, dataset, schema: "public")
+          execute(<<~SQL.squish)
+            INSERT INTO TABLE FUNCTION #{DB.literal(pg_table_connection(temp_table_name, schema:))}
+            #{dataset.sql}
+          SQL
+        end
+
+        def with_temp_table(
+          dataset, temp_table_name:, primary_key: [:id], primary_key_types: [:integer], **, &
+        )
+          unless DB.table_exists?(temp_table_name)
+            UmbrellioUtils::Database.create_temp_table(
+              nil, primary_key:, primary_key_types:, temp_table_name:, &
+            )
+            populate_temp_table!(temp_table_name, dataset)
+          end
+          UmbrellioUtils::Database.with_temp_table(nil, primary_key:, temp_table_name:, **, &)
+        end
+
+        protected
+
+        def log_errors(sql)
+          yield
+        rescue self.class::SERVER_ERROR => e
+          logger.error("ClickHouse error: #{e.inspect}\nSQL: #{sql}")
+          raise e
+        end
+
+        def sql_for(dataset)
+          return dataset if dataset.is_a?(String)
+          unless ch_dataset?(dataset)
+            raise "Non-ClickHouse dataset: #{dataset.inspect}. " \
+                  "You should use `CH.from` instead of `DB`"
+          end
+          dataset.sql
+        end
+
+        def ch_dataset?(dataset)
+          case dataset
+          when Sequel::Dataset
+            dataset.opts[:ch] && Array(dataset.opts[:from]).all? { |x| ch_dataset?(x) }
+          when Sequel::SQL::AliasedExpression
+            ch_dataset?(dataset.expression)
+          when Sequel::SQL::Identifier, Sequel::SQL::QualifiedIdentifier
+            true
+          else
+            raise "Unknown dataset type: #{dataset.inspect}"
+          end
+        end
+
+        def normalize_identifier(name)
+          name = name.value if name.is_a?(Sequel::SQL::Identifier)
+          name.to_s
+        end
+
+        def full_table_name(table_name, db_name)
+          "#{db_name}.#{normalize_identifier(table_name)}"
+        end
+      end
+    end
+  end
+end

data/lib/umbrellio_utils/click_house/backends/legacy.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require "click_house"
+
+module UmbrellioUtils
+  module ClickHouse
+    module Backends
+      # Adapter for the umbrellio/click_house gem (HTTP driver).
+      class Legacy < Base
+        include Memery
+
+        SERVER_ERROR = ::ClickHouse::Error
+
+        def execute(sql, host: nil, **opts)
+          log_errors(sql) { client(host).execute(sql, params: opts) }
+        end
+
+        def query(dataset, host: nil, **opts)
+          sql = sql_for(dataset)
+          log_errors(sql) do
+            select_all(sql, host:, **opts).map { |x| Misc::StrictHash[x.symbolize_keys] }
+          end
+        end
+
+        def query_value(dataset, host: nil, **opts)
+          sql = sql_for(dataset)
+          log_errors(sql) { select_value(sql, host:, **opts) }
+        end
+
+        def query_each(dataset, host: nil, **, &)
+          query(dataset, host:, **).each(&)
+        end
+
+        def insert(table_name, db_name: self.db_name, rows: [])
+          client.insert(full_table_name(table_name, db_name), rows, format: "JSONEachRow")
+        end
+
+        def describe_table(table_name, db_name: self.db_name)
+          sql = "DESCRIBE TABLE #{full_table_name(table_name, db_name)} FORMAT JSON"
+          log_errors(sql) { select_all(sql).map { |x| Misc::StrictHash[x.symbolize_keys] } }
+        end
+
+        def server_version
+          select_value("SELECT version()").to_f
+        end
+
+        def tables
+          client.tables
+        end
+
+        # Legacy HTTP driver can issue DDL directly; no admin side-channel
+        # needed. Base#create_database / #drop_database call this.
+        def admin_execute(sql)
+          client.execute(sql)
+        end
+
+        def config
+          client.config
+        end
+
+        def logger
+          client.config.logger
+        end
+
+        private
+
+        def client(host = nil)
+          cfg = ::ClickHouse.config
+          cfg.host = resolve(host) if host
+          ::ClickHouse::Connection.new(cfg)
+        end
+        memoize :client, ttl: 1.minute
+
+        def resolve(host)
+          IPSocket.getaddress(host)
+        rescue => e
+          Exceptions.notify!(e, raise_errors: false)
+          config.host
+        end
+
+        def select_all(sql, host: nil, **opts)
+          response = client(host).get(body: sql, query: { default_format: "JSON", **opts })
+          ::ClickHouse::Response::Factory.response(response, client(host).config)
+        end
+
+        def select_value(...)
+          select_all(...).first.to_a.dig(0, -1)
+        end
+      end
+    end
+  end
+end

data/lib/umbrellio_utils/click_house/backends/native.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "clickhouse-native"
+require_relative "../config"
+
+module UmbrellioUtils
+  module ClickHouse
+    module Backends
+      # Adapter for the clickhouse-native gem (TCP driver).
+      #
+      # Intentional differences from the HTTP-era module:
+      #   - Values returned by query / query_value are real Ruby types
+      #     (Time, Integer, etc.), not JSON-stringified.
+      #   - The `host:` kwarg on execute / query / query_value is accepted
+      #     for source compatibility but ignored — hostname is bound at
+      #     Pool construction, not per query.
+      class Native < Base
+        SERVER_ERROR = ::ClickhouseNative::ServerError
+
+        # Server-side error codes that mean "object doesn't exist". Used by
+        # describe_table callers that want to tolerate eager-load against a
+        # database that hasn't been created yet (e.g. rake ch:create).
+        UNKNOWN_TABLE = 60
+        UNKNOWN_DATABASE = 81
+
+        def execute(sql, host: nil, **_opts) # rubocop:disable Lint/UnusedMethodArgument
+          sql_string = sql.is_a?(String) ? sql : sql.sql
+          log_errors(sql_string) { pool.execute(sql_string) }
+        end
+
+        def query(dataset, host: nil, **_opts) # rubocop:disable Lint/UnusedMethodArgument
+          sql = sql_for(dataset)
+          log_errors(sql) { pool.query(sql) }
+        end
+
+        def query_value(dataset, host: nil, **_opts) # rubocop:disable Lint/UnusedMethodArgument
+          sql = sql_for(dataset)
+          log_errors(sql) { pool.query_value(sql) }
+        end
+
+        def query_each(dataset, host: nil, **_opts, &) # rubocop:disable Lint/UnusedMethodArgument
+          sql = sql_for(dataset)
+          log_errors(sql) { pool.query_each(sql, &) }
+        end
+
+        def insert(table_name, db_name: self.db_name, rows: [])
+          return if rows.empty?
+          pool.insert(normalize_identifier(table_name), rows, db_name: db_name.to_s)
+        end
+
+        def describe_table(table_name, db_name: self.db_name)
+          pool.describe_table(normalize_identifier(table_name), db_name: db_name.to_s)
+        end
+
+        def server_version
+          pool.with(&:server_version).to_f
+        end
+
+        def tables
+          pool.query("SHOW TABLES").pluck(:name)
+        end
+
+        def config
+          ::ClickHouse.config
+        end
+
+        # Read through pool so test mocks of `pool` also redirect `db_name`.
+        def db_name
+          pool.database.to_sym
+        end
+
+        def logger
+          @logger ||= UmbrellioUtils.config.clickhouse_native_logger ||
+                      (defined?(Rails) && Rails.logger) ||
+                      Logger.new($stdout)
+        end
+
+        def pool
+          @pool ||= ::ClickhouseNative::Pool.new(
+            **client_options(database: (config[:database] || "default").to_s),
+            pool_size: Integer(config[:pool_size] || 5),
+            pool_timeout: Integer(config[:pool_timeout] || 10),
+            settings: UmbrellioUtils.config.clickhouse_native_settings || {},
+          )
+        end
+
+        # DDL that creates/drops the configured database can't run through
+        # the main pool (which is bound to that database). Open a one-shot
+        # client connected to the always-present "default" db instead.
+        def admin_execute(sql)
+          admin = ::ClickhouseNative::Client.new(**client_options(database: "default"))
+          admin.execute(sql)
+        ensure
+          admin&.close
+        end
+
+        private
+
+        def client_options(database:)
+          {
+            host: config[:host] || "localhost",
+            port: Integer(config[:port] || 9000),
+            database:,
+            user: (config[:username] || "default").to_s,
+            password: (config[:password] || "").to_s,
+            logger:,
+          }
+        end
+      end
+    end
+  end
+end

data/lib/umbrellio_utils/click_house/backends.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require_relative "backends/base"
+
+module UmbrellioUtils
+  module ClickHouse
+    module Backends
+      autoload :Legacy, "umbrellio_utils/click_house/backends/legacy"
+      autoload :Native, "umbrellio_utils/click_house/backends/native"
+    end
+  end
+end

data/lib/umbrellio_utils/click_house/config.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# Provides `::ClickHouse.config` when the legacy `click_house` gem is
+# not loaded. The legacy gem defines `::ClickHouse::Connection` and its
+# own `::ClickHouse.config`; we only step in when it's absent (typical
+# for consumers that have migrated to the `clickhouse-native` gem).
+unless defined?(ClickHouse::Connection)
+  module ClickHouse
+    def self.config
+      @config ||= Rails.application.config_for(:clickhouse)
+    end
+  end
+end

data/lib/umbrellio_utils/click_house.rb

@@ -1,188 +1,50 @@
 # frozen_string_literal: true
 
 module UmbrellioUtils
+  # Polymorphic ClickHouse facade. The active backend is picked up from
+  # `UmbrellioUtils.config.clickhouse_backend` — `:legacy` routes through
+  # the `click_house` gem (HTTP), `:native` through the `clickhouse-native`
+  # gem (TCP). Both backends expose the same public surface so consumer
+  # code (including UmbrellioUtils::Migrations) is backend-agnostic.
   module ClickHouse
-    include Memery
-
     extend self
 
-    delegate :create_database, :drop_database, :tables, :config, to: :client
-
-    def insert(table_name, db_name: self.db_name, rows: [])
-      client.insert(full_table_name(table_name, db_name), rows, format: "JSONEachRow")
-    end
-
-    def from(source, db_name: self.db_name)
-      ds =
-        case source
-        when Symbol
-          DB.from(db_name == self.db_name ? SQL[source] : SQL[db_name][source])
-        when nil
-          DB.dataset
-        else
-          DB.from(source)
-        end
-
-      ds.clone(ch: true)
-    end
-
-    def execute(sql, host: nil, **opts)
-      log_errors(sql) do
-        client(host).execute(sql, params: opts)
-      end
-    end
-
-    def query(dataset, host: nil, **opts)
-      sql = sql_for(dataset)
-
-      log_errors(sql) do
-        select_all(sql, host:, **opts).map { |x| Misc::StrictHash[x.symbolize_keys] }
-      end
-    end
-
-    def query_value(dataset, host: nil, **opts)
-      sql = sql_for(dataset)
-
-      log_errors(sql) do
-        select_value(sql, host:, **opts)
-      end
-    end
-
-    def count(dataset)
-      query_value(dataset.select(SQL.ch_count))
-    end
-
-    def optimize_table!(table_name, db_name: self.db_name)
-      Timeout.timeout(UmbrellioUtils.config.ch_optimize_timeout) do
-        execute("OPTIMIZE TABLE #{db_name}.#{table_name} ON CLUSTER click_cluster FINAL")
-      end
-    end
-
-    def truncate_table!(table_name, db_name: self.db_name)
-      execute("TRUNCATE TABLE #{db_name}.#{table_name} ON CLUSTER click_cluster SYNC")
-    end
-
-    def drop_table!(table_name, db_name: self.db_name)
-      execute("DROP TABLE #{db_name}.#{table_name} ON CLUSTER click_cluster SYNC")
-    end
-
-    def describe_table(table_name, db_name: self.db_name)
-      sql = "DESCRIBE TABLE #{full_table_name(table_name, db_name)} FORMAT JSON"
+    autoload :Backends, "umbrellio_utils/click_house/backends"
 
-      log_errors(sql) do
-        select_all(sql).map { |x| Misc::StrictHash[x.symbolize_keys] }
-      end
-    end
+    VALID_BACKENDS = %i[legacy native].freeze
 
-    def db_name
-      client.config.database.to_sym
-    end
+    DELEGATED = %i[
+      execute query query_value query_each count insert
+      from describe_table server_version tables
+      create_database drop_database db_name config
+      truncate_table! drop_table! optimize_table! on_cluster
+      parse_value pg_table_connection populate_temp_table! with_temp_table
+    ].freeze
 
-    def parse_value(value, type:)
-      case type
-      when /String/
-        value&.to_s
-      when /DateTime/
-        Time.zone.parse(value) if value
-      else
-        value
+    DELEGATED.each do |method_name|
+      define_method(method_name) do |*args, **kwargs, &block|
+        backend.public_send(method_name, *args, **kwargs, &block)
       end
     end
 
-    def server_version
-      select_value("SELECT version()").to_f
+    def backend
+      @backend ||= backend_for(UmbrellioUtils.config.clickhouse_backend)
     end
 
-    def pg_table_connection(table)
-      host = ENV["PGHOST"] || DB.opts[:host].presence || "localhost"
-      port = DB.opts[:port] || 5432
-      database = DB.opts[:database]
-      username = DB.opts[:user]
-      password = DB.opts[:password]
-
-      Sequel.function(:postgresql, "#{host}:#{port}", database, table, username, password)
-    end
-
-    def with_temp_table(
-      dataset, temp_table_name:, primary_key: [:id], primary_key_types: [:integer], **opts, &
-    )
-      unless DB.table_exists?(temp_table_name)
-        UmbrellioUtils::Database.create_temp_table(
-          nil, primary_key:, primary_key_types:, temp_table_name:, &
-        )
-        populate_temp_table!(temp_table_name, dataset)
-      end
-      UmbrellioUtils::Database.with_temp_table(nil, primary_key:, temp_table_name:, **opts, &)
+    # Testing hook — clears the memoized backend so specs can flip
+    # `clickhouse_backend` mid-run. Not part of the public API.
+    def reset_backend!
+      @backend = nil
     end
 
     private
 
-    def client(host = nil)
-      cfg = ::ClickHouse.config
-      cfg.host = resolve(host) if host
-      ::ClickHouse::Connection.new(cfg)
-    end
-    memoize :client, ttl: 1.minute
-
-    def resolve(host)
-      IPSocket.getaddress(host)
-    rescue => e
-      Exceptions.notify!(e, raise_errors: false)
-      config.host
-    end
-
-    def logger
-      client.config.logger
-    end
-
-    def log_errors(sql)
-      yield
-    rescue ::ClickHouse::Error => e
-      logger.error("ClickHouse error: #{e.inspect}\nSQL: #{sql}")
-      raise e
-    end
-
-    def sql_for(dataset)
-      unless ch_dataset?(dataset)
-        raise "Non-ClickHouse dataset: #{dataset.inspect}. " \
-              "You should use `CH.from` instead of `DB`"
+    def backend_for(name)
+      case name
+      when :legacy then Backends::Legacy.instance
+      when :native then Backends::Native.instance
+      else raise "Unknown clickhouse_backend: #{name.inspect} (expected one of #{VALID_BACKENDS})"
       end
-
-      dataset.sql
-    end
-
-    def ch_dataset?(dataset)
-      case dataset
-      when Sequel::Dataset
-        dataset.opts[:ch] && Array(dataset.opts[:from]).all? { |x| ch_dataset?(x) }
-      when Sequel::SQL::AliasedExpression
-        ch_dataset?(dataset.expression)
-      when Sequel::SQL::Identifier, Sequel::SQL::QualifiedIdentifier
-        true
-      else
-        raise "Unknown dataset type: #{dataset.inspect}"
-      end
-    end
-
-    def full_table_name(table_name, db_name)
-      table_name = table_name.value if table_name.is_a?(Sequel::SQL::Identifier)
-      "#{db_name}.#{table_name}"
-    end
-
-    def select_all(sql, host: nil, **opts)
-      response = client(host).get(body: sql, query: { default_format: "JSON", **opts })
-      ::ClickHouse::Response::Factory.response(response, client(host).config)
-    end
-
-    def select_value(...)
-      select_all(...).first.to_a.dig(0, -1)
-    end
-
-    def populate_temp_table!(temp_table_name, dataset)
-      execute(<<~SQL.squish)
-        INSERT INTO TABLE FUNCTION #{DB.literal(pg_table_connection(temp_table_name))}
-        #{dataset.sql}
-      SQL
     end
   end
 end

data/lib/umbrellio_utils/control.rb

@@ -45,8 +45,8 @@ module UmbrellioUtils
       end
     end
 
-    def run_non_critical(rescue_all: false, in_transaction: false, &block)
-      in_transaction ? DB.transaction(savepoint: true, &block) : yield
+    def run_non_critical(rescue_all: false, in_transaction: false, &)
+      in_transaction ? DB.transaction(savepoint: true, &) : yield
     rescue (rescue_all ? Exception : StandardError) => e
       Exceptions.notify!(e)
       nil

data/lib/umbrellio_utils/database.rb

@@ -4,8 +4,11 @@ module UmbrellioUtils
   module Database
     extend self
 
-    HandledConstaintError = Class.new(StandardError)
-    InvalidPkError = Class.new(StandardError)
+    class HandledConstaintError < StandardError
+    end
+
+    class InvalidPkError < StandardError
+    end
 
     def handle_constraint_error(constraint_name, &)
       DB.transaction(savepoint: true, &)

data/lib/umbrellio_utils/http_client.rb

@@ -6,16 +6,16 @@ module UmbrellioUtils
   class HTTPClient
     include Singleton
 
-    def perform(*args, **kwargs)
-      client.perform(*args, **kwargs)
+    def perform(*, **)
+      client.perform(*, **)
     end
 
-    def perform!(*args, **kwargs)
-      client.perform!(*args, **kwargs)
+    def perform!(*, **)
+      client.perform!(*, **)
     end
 
-    def request(*args, **kwargs)
-      client.request(*args, **kwargs)
+    def request(*, **)
+      client.request(*, **)
     end
 
     private