activerecord-clickhouse-adapter 0.1.0

data/lib/active_record/connection_adapters/clickhouse/type_parser.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module ClickHouse
+      # Recursive-descent parser for ClickHouse type strings from
+      # JSONCompactEachRowWithNamesAndTypes. Structure only — no regex nesting.
+      class TypeParser
+        Error = Class.new(ArgumentError) # rubocop:disable Style/EmptyClassDefinition
+        Node = Data.define(:name, :args)
+
+        def self.parse(type_string) = new(type_string).parse
+
+        def initialize(source)
+          @source = source.to_s
+          @position = 0
+        end
+
+        def parse
+          raise Error, "empty type string" if @source.strip.empty?
+
+          node = parse_type
+          skip_whitespace
+          raise Error, "trailing input after type" unless eof?
+
+          node
+        end
+
+        private
+
+        def parse_type = parse_type_with_name(parse_identifier)
+
+        def parse_type_with_name(name) # rubocop:disable Metrics/MethodLength
+          return Node.new(name, []) unless consume("(")
+
+          args = case name
+                 when "Enum8", "Enum16" then parse_enum_args
+                 when "Tuple" then parse_tuple_args
+                 when "SimpleAggregateFunction", "AggregateFunction" then parse_aggregate_args
+                 when "Decimal", "Decimal32", "Decimal64", "Decimal128", "Decimal256",
+                      "FixedString", "DateTime", "DateTime64" then parse_literal_args
+                 else parse_type_args
+                 end
+          expect(")")
+          Node.new(name, args)
+        end
+
+        def parse_type_args = parse_comma_separated { parse_type }
+
+        def parse_literal_args = parse_comma_separated { parse_literal }
+
+        def parse_enum_args = parse_comma_separated { parse_enum_mapping }
+
+        def parse_aggregate_args
+          function_name = parse_aggregate_function_label
+          expect(",")
+          [function_name, *parse_type_args]
+        end
+
+        # Parametric combinators (quantile(0.95), topK(10)) carry parameters in the
+        # function name itself; the raw balanced-paren text stays in the label. String
+        # parameters containing parens would break this — none exist in practice.
+        def parse_aggregate_function_label
+          name = parse_identifier
+          skip_whitespace
+          return name unless peek("(")
+
+          "#{name}#{consume_balanced_parens}"
+        end
+
+        def consume_balanced_parens
+          start = @position
+          depth = 0
+          loop do
+            char = @source[@position] or raise Error, "unterminated aggregate function parameters"
+            depth += 1 if char == "("
+            depth -= 1 if char == ")"
+            @position += 1
+            break if depth.zero?
+          end
+          @source[start...@position]
+        end
+
+        def parse_tuple_args
+          return [] if peek(")")
+
+          parse_comma_separated { parse_tuple_element }
+        end
+
+        def parse_tuple_element
+          first = parse_identifier
+          skip_whitespace
+
+          if identifier_start?
+            [first, parse_type]
+          else
+            parse_type_with_name(first)
+          end
+        end
+
+        def parse_enum_mapping
+          label = parse_quoted_string
+          expect("=")
+          [label, parse_signed_integer]
+        end
+
+        def parse_literal
+          skip_whitespace
+          return parse_quoted_string if peek("'")
+          return parse_signed_integer if digit? || peek("-")
+
+          raise Error, "expected literal at position #{@position}"
+        end
+
+        def parse_comma_separated
+          skip_whitespace
+          return [] if peek(")")
+
+          values = [yield]
+          while consume(",")
+            skip_whitespace
+            raise Error, "trailing comma in parameters" if peek(")")
+
+            values << yield
+          end
+          values
+        end
+
+        def parse_identifier
+          skip_whitespace
+          raise Error, "expected identifier at position #{@position}" unless identifier_start?
+
+          start = @position
+          @position += 1 while @position < @source.length && identifier_char?(@source[@position])
+          @source[start...@position]
+        end
+
+        def parse_quoted_string # rubocop:disable Metrics/MethodLength
+          skip_whitespace
+          expect("'")
+          chars = +""
+          while @position < @source.length
+            char = @source[@position]
+            if char == "\\"
+              @position += 1
+              raise Error, "unterminated escape in string" if eof?
+
+              chars << @source[@position]
+              @position += 1
+            elsif char == "'"
+              @position += 1
+              return chars
+            else
+              chars << char
+              @position += 1
+            end
+          end
+          raise Error, "unterminated string"
+        end
+
+        def parse_signed_integer
+          skip_whitespace
+          start = @position
+          @position += 1 if peek("-")
+          raise Error, "expected integer at position #{@position}" unless digit?
+
+          @position += 1 while digit?
+          Integer(@source[start...@position])
+        end
+
+        def expect(token)
+          skip_whitespace
+          return if consume(token)
+
+          raise Error, "expected #{token.inspect} at position #{@position}"
+        end
+
+        def consume(token)
+          skip_whitespace
+          return false unless @source[@position, token.length] == token
+
+          @position += token.length
+          true
+        end
+
+        def peek(token)
+          skip_whitespace
+          @source[@position, token.length] == token
+        end
+
+        def skip_whitespace
+          @position += 1 while @position < @source.length && @source[@position].match?(/\s/)
+        end
+
+        def eof? = @position >= @source.length
+
+        def peek_char = eof? ? nil : @source[@position]
+
+        def digit? = peek_char&.match?(/\d/)
+
+        def identifier_start? = peek_char&.match?(/[A-Za-z_]/)
+
+        def identifier_char?(char) = char.match?(/[A-Za-z0-9_]/)
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse/types.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+require "bigdecimal"
+require "date"
+require "ipaddr"
+require "json"
+require "singleton"
+
+module ActiveRecord
+  module ConnectionAdapters
+    module ClickHouse
+      # Builds eager value casters from ClickHouse type AST nodes.
+      module Types
+        module_function
+
+        def caster_for(type_string) = build(TypeParser.parse(type_string))
+
+        # ActiveModel type for schema introspection (Column#type); wrappers unwrap.
+        def active_record_cast_type(type_string) = ar_cast_type(TypeParser.parse(type_string))
+
+        def ar_cast_type(node) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
+          case node.name
+          when "Nullable", "LowCardinality" then ar_cast_type(node.args.fetch(0))
+          when "SimpleAggregateFunction" then ar_cast_type(node.args.fetch(1))
+          when /\AU?Int(8|16|32|64|128|256)\z/ then ActiveModel::Type::Integer.new(limit: Regexp.last_match(1).to_i / 8)
+          when "Float32", "Float64" then ActiveModel::Type::Float.new
+          when "Decimal", "Decimal32", "Decimal64", "Decimal128", "Decimal256"
+            precision, scale = node.args.grep(Integer)
+            ActiveModel::Type::Decimal.new(precision: precision, scale: scale)
+          when "Date", "Date32" then ActiveModel::Type::Date.new
+          when "DateTime" then ActiveModel::Type::DateTime.new
+          when "DateTime64" then ActiveModel::Type::DateTime.new(precision: node.args.first)
+          when "Bool" then ActiveModel::Type::Boolean.new
+          when "String", "FixedString", "Enum8", "Enum16", "UUID", "IPv4", "IPv6" then ActiveModel::Type::String.new
+          when "JSON" then ActiveRecord::Type::Json.new
+          else ActiveModel::Type::Value.new
+          end
+        end
+
+        def build(node) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
+          case node.name
+          when "Nullable" then Nullable.new(build(node.args.fetch(0)))
+          when "LowCardinality" then build(node.args.fetch(0))
+          when "Array" then ArrayCaster.new(build(node.args.fetch(0)))
+          when "Map" then MapCaster.new(build(node.args.fetch(0)), build(node.args.fetch(1)))
+          when "Tuple" then tuple_caster(node.args)
+          when "SimpleAggregateFunction" then build(node.args.fetch(1))
+          when "AggregateFunction" then Passthrough.instance
+          when "Bool" then BoolCaster.instance
+          when "String", "FixedString", "UUID", "Enum8", "Enum16" then StringCaster.instance
+          when "JSON" then JsonCaster.instance
+          when "Nothing" then NullCaster.instance
+          when "Date", "Date32" then DateCaster.instance
+          when "DateTime" then DateTimeCaster.new(node.args[0])
+          when "DateTime64" then DateTimeCaster.new(node.args[1])
+          when "IPv4", "IPv6" then IpCaster.instance
+          when "Float32", "Float64" then FloatCaster.instance
+          when "Decimal", "Decimal32", "Decimal64", "Decimal128", "Decimal256" then DecimalCaster.instance
+          when /\A(?:UInt|Int)(?:8|16|32|64|128|256)\z/ then IntegerCaster.instance
+          else
+            raise ArgumentError, "unsupported ClickHouse type family: #{node.name}"
+          end
+        end
+
+        def tuple_caster(args)
+          if args.first.is_a?(Array)
+            NamedTupleCaster.new(args.to_h { |name, type_node| [name, build(type_node)] })
+          else
+            PositionalTupleCaster.new(args.map { |type_node| build(type_node) })
+          end
+        end
+        private_class_method :tuple_caster
+
+        class Passthrough
+          include Singleton
+
+          def cast(value) = value
+        end
+
+        class NullCaster
+          include Singleton
+
+          def cast(_value) = nil
+        end
+
+        class Nullable
+          def initialize(inner) = @inner = inner
+          def cast(value) = value.nil? ? nil : @inner.cast(value)
+        end
+
+        class IntegerCaster
+          include Singleton
+
+          def cast(value) = Integer(value)
+        end
+
+        class FloatCaster
+          include Singleton
+
+          # quote_denormals=1 delivers these as strings; anything else must be numeric.
+          DENORMAL_VALUES = {
+            "nan" => Float::NAN, "inf" => Float::INFINITY,
+            "+inf" => Float::INFINITY, "-inf" => -Float::INFINITY
+          }.freeze
+
+          def cast(value) = DENORMAL_VALUES.fetch(value) { Float(value) }
+        end
+
+        class DecimalCaster
+          include Singleton
+
+          def cast(value) = value.is_a?(BigDecimal) ? value : BigDecimal(value.to_s)
+        end
+
+        class StringCaster
+          include Singleton
+
+          def cast(value) = value.to_s
+        end
+
+        class BoolCaster
+          include Singleton
+
+          def cast(value) = [true, 1, "1", "true"].include?(value)
+        end
+
+        class DateCaster
+          include Singleton
+
+          def cast(value) = value.is_a?(Date) ? value : Date.parse(value.to_s)
+        end
+
+        # Returns plain Time instances (like every built-in adapter's database layer);
+        # TimeWithZone wrapping belongs to the attribute layer.
+        class DateTimeCaster
+          # The server always sends this exact shape; building the Time directly is 3.3x
+          # faster than zone.parse and allocates a fraction of the memory
+          # (benchmarks/BASELINE.md).
+          WIRE_FORMAT = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(?:\.(\d+))?\z/
+
+          def initialize(time_zone) = @time_zone = time_zone
+
+          def cast(value)
+            return value if value.is_a?(Time)
+
+            string = value.to_s
+            instant = if (match = WIRE_FORMAT.match(string))
+                        time_from_wall_clock(wall_clock_parts(match))
+                      else
+                        zone.parse(string).utc
+                      end
+            ActiveRecord.default_timezone == :local ? instant.getlocal : instant
+          end
+
+          private
+
+          def time_from_wall_clock(parts)
+            found = zone
+            found.name == "UTC" ? ::Time.utc(*parts) : found.local(*parts).utc
+          end
+
+          def wall_clock_parts(match)
+            seconds = match[6].to_i
+            seconds += Rational(match[7].to_i, 10**match[7].length) if match[7]
+            [match[1].to_i, match[2].to_i, match[3].to_i, match[4].to_i, match[5].to_i, seconds]
+          end
+
+          def zone
+            found = @time_zone ? ActiveSupport::TimeZone[@time_zone] : default_zone
+            raise ArgumentError, "unknown time zone #{@time_zone.inspect}" if @time_zone && found.nil?
+
+            found
+          end
+
+          def default_zone
+            if ActiveRecord.default_timezone == :local && ::Time.zone
+              ::Time.zone
+            else
+              ActiveSupport::TimeZone["UTC"]
+            end
+          end
+        end
+
+        class IpCaster
+          include Singleton
+
+          def cast(value) = value.is_a?(IPAddr) ? value : IPAddr.new(value.to_s)
+        end
+
+        class JsonCaster
+          include Singleton
+
+          def cast(value)
+            case value
+            when String then JSON.parse(value)
+            else value
+            end
+          end
+        end
+
+        class ArrayCaster
+          def initialize(inner) = @inner = inner
+          def cast(value) = Array(value).map { |item| @inner.cast(item) }
+        end
+
+        class MapCaster
+          def initialize(key_caster, value_caster)
+            @key_caster = key_caster
+            @value_caster = value_caster
+          end
+
+          def cast(value)
+            value.to_h { |key, item| [@key_caster.cast(key), @value_caster.cast(item)] }
+          end
+        end
+
+        class PositionalTupleCaster
+          def initialize(casters) = @casters = casters
+
+          def cast(value)
+            Array(value).each_with_index.map { |item, index| @casters.fetch(index).cast(item) }
+          end
+        end
+
+        class NamedTupleCaster
+          def initialize(casters) = @casters = casters
+
+          def cast(value)
+            value.to_h do |key, item|
+              name = key.to_s
+              [name, @casters.fetch(name).cast(item)]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "active_record/connection_adapters/abstract_adapter"
+require "active_record/connection_adapters/clickhouse/database_statements"
+require "active_record/connection_adapters/clickhouse/error_translation"
+require "active_record/connection_adapters/clickhouse/http_connection"
+require "active_record/connection_adapters/clickhouse/querying"
+require "active_record/connection_adapters/clickhouse/quoting"
+require "active_record/connection_adapters/clickhouse/row_binary"
+require "active_record/connection_adapters/clickhouse/schema_definitions"
+require "active_record/connection_adapters/clickhouse/schema_dumper"
+require "active_record/connection_adapters/clickhouse/schema_statements"
+require "active_record/connection_adapters/clickhouse/type_parser"
+require "active_record/connection_adapters/clickhouse/types"
+require "arel/visitors/clickhouse"
+
+module ActiveRecord
+  module ConnectionAdapters
+    class ClickHouseAdapter < AbstractAdapter
+      ADAPTER_NAME = "ClickHouse"
+
+      include ClickHouse::Quoting
+      include ClickHouse::DatabaseStatements
+      include ClickHouse::SchemaStatements
+      include ClickHouse::ErrorTranslation
+
+      class << self
+        def new_client(config)
+          ClickHouse::HTTPConnection.new(config)
+        end
+      end
+
+      CONNECTION_PARAMETER_KEYS = %i[
+        host port username password database ssl ssl_verify select_format
+        connect_timeout read_timeout write_timeout
+        mutations_sync compression join_use_nulls async_insert wait_for_async_insert
+      ].freeze
+
+      def initialize(...)
+        super
+
+        @connection_parameters =
+          { host: "localhost", port: 8123 }.merge(@config.slice(*CONNECTION_PARAMETER_KEYS)).compact
+      end
+
+      def active?
+        @lock.synchronize { @raw_connection&.ping } || false
+      end
+
+      # With cluster: configured, schema DDL is stamped ON CLUSTER so every replica
+      # runs it through the distributed DDL queue.
+      def cluster = @config[:cluster]
+
+      def on_cluster_clause
+        cluster ? " ON CLUSTER #{quote_table_name(cluster)}" : ""
+      end
+
+      def disconnect!
+        super
+        @raw_connection&.close
+        @raw_connection = nil
+      end
+
+      def supports_explain? = true
+
+      # DateTime64(P) is native precision; claiming it makes the DSL apply Rails'
+      # default microsecond precision (6) to datetime columns, like other adapters.
+      def supports_datetime_with_precision? = true
+
+      # Data-skipping indexes are INDEX clauses inside CREATE TABLE, not statements.
+      def supports_indexes_in_create? = true
+
+      # insert_all implies skip-duplicates; without unique constraints nothing can
+      # conflict, so the semantics hold vacuously and the INSERT goes through plain.
+      # Upsert still raises upstream (supports_insert_on_duplicate_update? is false).
+      def supports_insert_on_duplicate_skip? = true
+
+      def build_insert_sql(insert) = "INSERT #{insert.into} #{insert.values_list}" # :nodoc:
+
+      NATIVE_DATABASE_TYPES = {
+        string: { name: "String" },
+        text: { name: "String" },
+        integer: { name: "Int32", limit: 4 },
+        bigint: { name: "Int64", limit: 8 },
+        float: { name: "Float64" },
+        decimal: { name: "Decimal" },
+        datetime: { name: "DateTime64" },
+        date: { name: "Date32" },
+        boolean: { name: "Bool" },
+        uuid: { name: "UUID" },
+        json: { name: "JSON" }
+      }.freeze
+
+      def native_database_types = NATIVE_DATABASE_TYPES
+
+      # Column types the dumper can't map to AR symbols (Array, Map, Tuple, ...) are
+      # dumped verbatim, so every introspected type is valid by construction.
+      def valid_type?(_type) = true
+
+      def create_schema_dumper(options) # :nodoc:
+        ClickHouse::SchemaDumper.create(self, options)
+      end
+
+      def get_database_version # :nodoc:
+        Version.new(query_value("SELECT version()", "SCHEMA"))
+      end
+
+      private
+
+      def connect
+        @raw_connection = self.class.new_client(@connection_parameters)
+      end
+
+      def reconnect
+        @raw_connection&.close
+        connect
+      end
+
+      # No server-side prepared statements; `true` selects the Arel Bind collector so we can
+      # rewrite `?` into ClickHouse `{pN:Type}` HTTP query parameters in perform_query.
+      def default_prepared_statements = true
+
+      def arel_visitor = Arel::Visitors::ClickHouse.new(self)
+    end
+  end
+end

data/lib/active_record/tasks/clickhouse_database_tasks.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "active_record/connection_adapters/clickhouse/http_connection"
+
+module ActiveRecord
+  module Tasks
+    # db:create/db:drop/db:purge and structure dump/load for ClickHouse. Database-level
+    # DDL runs over a raw HTTP connection without a database selected, since the
+    # configured database may not exist yet.
+    class ClickHouseDatabaseTasks < AbstractTasks
+      DATABASE_ALREADY_EXISTS_CODE = 82
+      UNKNOWN_DATABASE_CODE = 81
+
+      def create
+        server_execute("CREATE DATABASE #{quoted_database}")
+      rescue ConnectionAdapters::ClickHouse::HTTPConnection::ExecutionError => e
+        raise DatabaseAlreadyExists if e.code == DATABASE_ALREADY_EXISTS_CODE
+
+        raise
+      end
+
+      def drop
+        server_execute("DROP DATABASE #{quoted_database}")
+      rescue ConnectionAdapters::ClickHouse::HTTPConnection::ExecutionError => e
+        raise NoDatabaseError, e.message if e.code == UNKNOWN_DATABASE_CODE
+
+        raise
+      end
+
+      def purge
+        drop
+      rescue NoDatabaseError
+        nil
+      ensure
+        create
+      end
+
+      def structure_dump(filename, _flags)
+        establish_connection
+        statements = dumpable_data_sources.map do |name|
+          connection.select_value("SHOW CREATE TABLE #{connection.quote_table_name(name)}")
+        end
+        File.write(filename, statements.map { |statement| "#{statement};\n\n" }.join)
+      end
+
+      # Statements are separated by ";\n\n" exactly as structure_dump writes them —
+      # ClickHouse HTTP accepts one statement per request, so the file is replayed.
+      def structure_load(filename, _flags)
+        establish_connection
+        File.read(filename).split(";\n\n").each do |statement|
+          statement = statement.strip
+          connection.execute(reinject_dictionary_credentials(statement)) unless statement.empty?
+        end
+      end
+
+      private
+
+      # SHOW CREATE masks dictionary source passwords as [HIDDEN] (and the dumped USER
+      # belongs to the dumping environment), so loading swaps in this connection's
+      # credentials — the same ones create_dictionary would inject.
+      def reinject_dictionary_credentials(statement)
+        return statement unless statement.match?(/\ACREATE DICTIONARY/i)
+
+        quote = ->(value) { "'#{value.to_s.gsub("\\", "\\\\\\\\").gsub("'", "\\\\'")}'" }
+        statement
+          .gsub(/\bUSER '(?:[^'\\]|\\.)*'/) { "USER #{quote.call(configuration_hash[:username])}" }
+          .gsub(/\bPASSWORD '(?:[^'\\]|\\.)*'/) { "PASSWORD #{quote.call(configuration_hash[:password])}" }
+      end
+
+      # ignore_tables entries may be String, Regexp, or Proc — === is the contract
+      # Rails' own SchemaDumper uses to match them.
+      def dumpable_data_sources
+        ignored = ActiveRecord::SchemaDumper.ignore_tables
+        connection.data_sources.sort.reject { |name| ignored.any? { |pattern| pattern === name } } # rubocop:disable Style/CaseEquality
+      end
+
+      def quoted_database
+        "`#{db_config.database.to_s.gsub("`", "``")}`"
+      end
+
+      def server_execute(sql)
+        client = ConnectionAdapters::ClickHouse::HTTPConnection.new(
+          configuration_hash.except(:adapter, :database)
+        )
+        client.execute(sql)
+      ensure
+        client&.close
+      end
+    end
+  end
+end

data/lib/activerecord-clickhouse-adapter.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "active_record/connection_adapters"
+require "active_record/connection_adapters/clickhouse/gem_version"
+# Eager so models can `include ...ClickHouse::Querying` at boot, before (or without)
+# any ClickHouse connection loading the adapter — e.g. app code shared across envs
+# where the sink is only wired in production.
+require "active_record/connection_adapters/clickhouse/querying"
+require "active_record/tasks/clickhouse_database_tasks"
+
+ActiveRecord::ConnectionAdapters.register(
+  "clickhouse",
+  "ActiveRecord::ConnectionAdapters::ClickHouseAdapter",
+  "active_record/connection_adapters/clickhouse_adapter"
+)
+
+ActiveRecord::Tasks::DatabaseTasks.register_task(
+  /clickhouse/,
+  "ActiveRecord::Tasks::ClickHouseDatabaseTasks"
+)