activerecord-clickhouse-adapter 0.1.0

data/lib/active_record/connection_adapters/clickhouse/http_connection.rb

@@ -0,0 +1,233 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "openssl"
+require "uri"
+
+require "active_record/connection_adapters/clickhouse/row_binary"
+
+module ActiveRecord
+  module ConnectionAdapters
+    module ClickHouse
+      # Raw connection to a ClickHouse server over its HTTP interface: one persistent
+      # keep-alive Net::HTTP socket per adapter instance (the adapter lock serializes use).
+      # Results arrive as RowBinaryWithNamesAndTypes by default — names, type strings,
+      # then packed binary rows — so every value comes back with its server type; queries
+      # whose types have no binary decoder retry transparently on the JSON wire
+      # (select_format: :json in the config forces JSON for everything).
+      class HTTPConnection
+        BINARY_FORMAT = "RowBinaryWithNamesAndTypes"
+        JSON_FORMAT = "JSONCompactEachRowWithNamesAndTypes"
+
+        class ExecutionError < StandardError
+          attr_reader :code
+
+          def initialize(message, code: nil)
+            super(message)
+            @code = code
+          end
+        end
+
+        class RawResult
+          attr_reader :columns, :types, :rows, :summary, :query_id
+
+          def initialize(columns: [], types: [], rows: [], summary: {}, query_id: nil)
+            @columns = columns
+            @types = types
+            @rows = rows
+            @summary = summary
+            @query_id = query_id
+          end
+
+          def written_rows = Integer(summary.fetch("written_rows", 0))
+
+          # Server-side execution stats, for the sql.active_record notification payload.
+          def stats
+            {
+              query_id: query_id,
+              read_rows: Integer(summary.fetch("read_rows", 0)),
+              read_bytes: Integer(summary.fetch("read_bytes", 0)),
+              written_rows: written_rows,
+              elapsed_ns: Integer(summary.fetch("elapsed_ns", 0))
+            }
+          end
+        end
+
+        def initialize(config)
+          @config = config
+          @select_format = config[:select_format].to_s == "json" ? JSON_FORMAT : BINARY_FORMAT
+          @http = build_http
+        end
+
+        # Adapts an enumerator of encoded lines to the partial-read IO contract
+        # Net::HTTP uses for chunked transfer encoding, so a streamed INSERT never
+        # holds more than one chunk of the body in memory.
+        class ChunkedBody
+          CHUNK_BYTES = 64 * 1024
+
+          def initialize(lines)
+            @lines = lines
+            @buffer = +""
+          end
+
+          def read(length = CHUNK_BYTES, out = +"")
+            buffer_lines(length)
+            return nil if @buffer.empty?
+
+            out.replace(@buffer.slice!(0, length))
+          end
+
+          private
+
+          def buffer_lines(length)
+            @buffer << @lines.next << "\n" while @buffer.bytesize < length
+          rescue StopIteration
+            nil
+          end
+        end
+
+        def execute(sql, params: {})
+          parse(perform(sql, params, @select_format), @select_format)
+        rescue RowBinary::Undecodable
+          parse(perform(sql, params, JSON_FORMAT), JSON_FORMAT)
+        end
+
+        # Streams pre-encoded body lines as one chunked POST; the statement travels
+        # in the query string because the request body is the data.
+        def execute_stream(sql, lines)
+          request = post_request(query_params({}, JSON_FORMAT).merge(query: sql))
+          request["Transfer-Encoding"] = "chunked"
+          request.body_stream = ChunkedBody.new(lines)
+          parse(raise_unless_success(@http.request(request)), JSON_FORMAT)
+        end
+
+        # Scopes extra server settings to the requests made inside the block — the
+        # write-side counterpart of the SETTINGS clause SELECTs carry in-SQL.
+        def with_request_settings(settings)
+          previous = @request_settings
+          @request_settings = (previous || {}).merge(settings)
+          yield
+        ensure
+          @request_settings = previous
+        end
+
+        def ping
+          execute("SELECT 1")
+          true
+        rescue StandardError
+          false
+        end
+
+        def close
+          @http.finish if @http.started?
+        rescue IOError
+          nil
+        end
+
+        private
+
+        def build_http
+          http = Net::HTTP.new(@config[:host], @config[:port])
+          configure_tls(http)
+          http.open_timeout = @config[:connect_timeout] if @config[:connect_timeout]
+          http.read_timeout = @config[:read_timeout] if @config[:read_timeout]
+          http.write_timeout = @config[:write_timeout] if @config[:write_timeout]
+          http
+        end
+
+        # Verification stays ON by default; ssl_verify: false is the explicit escape
+        # hatch for sinks terminating TLS with a self-signed certificate.
+        def configure_tls(http)
+          http.use_ssl = @config[:ssl] if @config.key?(:ssl)
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE if @config[:ssl_verify] == false
+        end
+
+        def perform(sql, params, format)
+          request = post_request(query_params(params, format))
+          request.body = sql
+          raise_unless_success(@http.request(request))
+        end
+
+        def post_request(params)
+          request = Net::HTTP::Post.new("/?#{URI.encode_www_form(params)}")
+          request["X-ClickHouse-User"] = @config[:username] if @config[:username]
+          request["X-ClickHouse-Key"] = @config[:password] if @config[:password]
+          request
+        end
+
+        def raise_unless_success(response)
+          raise_execution_error(response) unless response.is_a?(Net::HTTPSuccess)
+
+          response
+        end
+
+        def query_params(params, format)
+          session_settings(format).merge(async_insert_params)
+                                  .merge(@request_settings || {})
+                                  .merge(params.transform_keys { |key| "param_#{key}" }).compact
+        end
+
+        # Defaults corrupt silently: JSON Decimals float-parse lossily, NaN/Inf become
+        # null; binary JSON columns have no stable layout, so they travel as text.
+        OUTPUT_SETTINGS = {
+          output_format_json_quote_decimals: 1,
+          output_format_json_quote_denormals: 1,
+          output_format_binary_write_json_as_string: 1
+        }.freeze
+        private_constant :OUTPUT_SETTINGS
+
+        def session_settings(format)
+          OUTPUT_SETTINGS.merge(
+            database: @config[:database],
+            default_format: format,
+            wait_end_of_query: 1,
+            # 1/2 make ALTER UPDATE/DELETE mutations block until applied (spec determinism).
+            mutations_sync: @config[:mutations_sync],
+            # ClickHouse fills non-matched outer-join columns with type defaults (0, '');
+            # every other AR adapter returns SQL NULLs, so default to standard semantics.
+            join_use_nulls: @config.fetch(:join_use_nulls, 1),
+            # By default the server coerces NULL to the type default on insert into a
+            # non-Nullable column — silent data corruption by AR semantics. Off, the
+            # insert raises (code 53, probed 2026-07-14) and maps to NotNullViolation.
+            input_format_null_as_default: 0,
+            # Server gzips responses ~3.6x smaller; Net::HTTP decompresses transparently
+            # (it sends Accept-Encoding: gzip by default). Probed 2026-07-12, PLAN.md §2.
+            enable_http_compression: @config.fetch(:compression, true) ? 1 : 0
+          )
+        end
+
+        # Server-side batching for high-frequency small INSERTs. wait_for_async_insert
+        # defaults to 1 so an acked insert is durable; 0 (fire-and-forget) is an
+        # explicit opt-in because it loses acked rows on a server crash.
+        def async_insert_params
+          return {} unless @config[:async_insert]
+
+          { async_insert: 1, wait_for_async_insert: @config.fetch(:wait_for_async_insert, 1) }
+        end
+
+        def raise_execution_error(response)
+          code = response["x-clickhouse-exception-code"]&.to_i
+          raise ExecutionError.new(response.body.to_s.strip, code: code)
+        end
+
+        def parse(response, format)
+          names, types, rows = decode_body(response.body.to_s, format)
+
+          RawResult.new(
+            columns: names, types: types, rows: rows,
+            summary: JSON.parse(response["x-clickhouse-summary"] || "{}"),
+            query_id: response["x-clickhouse-query-id"]
+          )
+        end
+
+        def decode_body(body, format)
+          return RowBinary.decode(body) if format == BINARY_FORMAT
+
+          names, types, *rows = body.each_line.map { |line| JSON.parse(line) }
+          [names || [], types || [], rows]
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse/querying.rb

@@ -0,0 +1,417 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module ClickHouse
+      # Custom Arel nodes carrying ClickHouse dialect state through the AST; only
+      # Arel::Visitors::ClickHouse knows how to render them.
+      module Nodes
+        # Wraps the FROM table to append FINAL and/or SAMPLE (table modifiers).
+        class TableWithModifiers < ::Arel::Nodes::Unary
+          attr_accessor :final, :sample
+        end
+
+        # A condition rendered as PREWHERE, smuggled through SelectCore#wheres.
+        class Prewhere < ::Arel::Nodes::Unary
+        end
+
+        # Rides in SelectStatement#lock (ClickHouse has no row locks): LIMIT BY
+        # renders before LIMIT, SETTINGS renders last.
+        class DialectSuffix < ::Arel::Nodes::Node
+          attr_reader :limit_by, :settings
+
+          def initialize(limit_by: nil, settings: nil)
+            super()
+            @limit_by = limit_by
+            @settings = settings
+          end
+        end
+
+        # Wraps an ORDER BY expression to append WITH FILL [STEP n] (gap filling).
+        class OrderWithFill < ::Arel::Nodes::Unary
+          attr_accessor :step
+        end
+
+        # ARRAY JOIN unnests array columns into rows; rides the join list so it renders
+        # after FROM and any regular JOINs, before WHERE.
+        class ArrayJoin < ::Arel::Nodes::Node
+          attr_reader :columns, :left, :alias_name
+
+          def initialize(columns, left:, alias_name:)
+            super()
+            @columns = columns
+            @left = left
+            @alias_name = alias_name
+          end
+        end
+      end
+
+      # Relation methods for the ClickHouse query surface. Injected via
+      # `extending` (public Relation API) by ClickHouse::Querying — no reopens.
+      module RelationMethods
+        def final = spawn.final!
+
+        def final!
+          assign_clickhouse_dialect(final: true)
+        end
+
+        def sample(fraction) = spawn.sample!(fraction)
+
+        def sample!(fraction)
+          assign_clickhouse_dialect(sample: Float(fraction))
+        end
+
+        def prewhere(condition) = spawn.prewhere!(condition)
+
+        def prewhere!(condition)
+          sanitized = klass.send(:sanitize_sql_for_conditions, condition)
+          assign_clickhouse_dialect(prewhere: [*clickhouse_dialect[:prewhere], sanitized])
+        end
+
+        def settings(values) = spawn.settings!(values)
+
+        def settings!(values)
+          assign_clickhouse_dialect(settings: (clickhouse_dialect[:settings] || {}).merge(values))
+        end
+
+        def limit_by(count, *columns) = spawn.limit_by!(count, *columns)
+
+        def limit_by!(count, *columns)
+          assign_clickhouse_dialect(limit_by: [Integer(count), columns.flatten])
+        end
+
+        # toStartOfInterval buckets any period the server knows; grouping and ordering
+        # by the same expression makes `.count` come back as a chronological series.
+        GROUP_BY_PERIODS = %w[minute hour day week month quarter year].freeze
+
+        def group_by_period(period, column) = spawn.group_by_period!(period, column)
+
+        def group_by_period!(period, column)
+          unless GROUP_BY_PERIODS.include?(period.to_s)
+            raise ArgumentError, "unknown period #{period.inspect}; use one of #{GROUP_BY_PERIODS.join(", ")}"
+          end
+
+          bucket = ::Arel::Nodes::NamedFunction.new(
+            "toStartOfInterval", [klass.arel_table[column], ::Arel.sql("INTERVAL 1 #{period}")]
+          )
+          group!(bucket).order!(bucket)
+        end
+
+        def fill(step: nil) = spawn.fill!(step: step)
+
+        def fill!(step: nil)
+          assign_clickhouse_dialect(fill: { step: fill_step_sql(step) })
+        end
+
+        # Unnests array columns into one row per element; as: names the element so the
+        # original array column stays addressable. left: true keeps empty-array rows.
+        def array_join(*columns, left: false, as: nil) = spawn.array_join!(*columns, left: left, as: as)
+
+        def array_join!(*columns, left: false, as: nil)
+          raise ArgumentError, "as: names a single element, so array_join takes one column with it" if
+            as && columns.many?
+
+          assign_clickhouse_dialect(array_join: { columns: columns.flatten, left: left, as: as })
+        end
+
+        # ClickHouse renders WITH TOTALS out-of-band, which the row-stream wire format
+        # drops (probed 2026-07-13) — ROLLUP delivers totals as ordinary rows instead,
+        # keyed nil thanks to group_by_use_nulls.
+        def rollup = spawn.rollup!
+
+        def rollup!
+          settings!(group_by_use_nulls: 1)
+          assign_clickhouse_dialect(rollup: true)
+        end
+
+        def clickhouse_dialect
+          @clickhouse_dialect ||= {}
+        end
+
+        private
+
+        def assign_clickhouse_dialect(changes)
+          @clickhouse_dialect = clickhouse_dialect.merge(changes)
+          self
+        end
+
+        def fill_step_sql(step)
+          case step
+          when nil then nil
+          when ActiveSupport::Duration then "INTERVAL #{Integer(step.to_i)} SECOND"
+          else Integer(step).to_s
+          end
+        end
+
+        def build_arel(...)
+          manager = super
+          apply_clickhouse_dialect(manager.ast) unless clickhouse_dialect.empty?
+          manager
+        end
+      end
+
+      # Projects one window expression per call, e.g.
+      # window(:sum, :battery, as: :drained, partition_by: :device_id, order_by: :observed_at).
+      # Arel's own Window/Over nodes do the rendering; the function name, alias and
+      # frame are the only free-form strings, so they are validated here.
+      module RelationWindowing
+        WINDOW_IDENTIFIER = /\A[A-Za-z_]\w*\z/
+        WINDOW_FRAME = /\A(?:ROWS|RANGE|GROUPS)\b[\w\s]+\z/i
+
+        def window(function, *columns, as:, partition_by: nil, order_by: nil, frame: nil)
+          spawn.window!(function, *columns, as: as, partition_by: partition_by, order_by: order_by, frame: frame)
+        end
+
+        def window!(function, *columns, as:, partition_by: nil, order_by: nil, frame: nil)
+          projection = window_projection(function, columns, window_definition(partition_by, order_by, frame))
+          self.select_values += [klass.arel_table[::Arel.star]] if select_values.empty?
+          self.select_values += [::Arel::Nodes::As.new(projection, ::Arel.sql(window_identifier(as, "alias")))]
+          self
+        end
+
+        private
+
+        def window_projection(function, columns, definition)
+          arguments = columns.map { |column| klass.arel_table[column] }
+          ::Arel::Nodes::NamedFunction.new(window_identifier(function, "function"), arguments).over(definition)
+        end
+
+        def window_identifier(value, role)
+          raise ArgumentError, "window #{role} must be an identifier" unless WINDOW_IDENTIFIER.match?(value.to_s)
+
+          value.to_s
+        end
+
+        def window_definition(partition_by, order_by, frame)
+          raise ArgumentError, "window frame must start with ROWS, RANGE or GROUPS" if
+            frame && !WINDOW_FRAME.match?(frame)
+
+          definition = ::Arel::Nodes::Window.new
+          Array(partition_by).each { |column| definition.partition(klass.arel_table[column]) }
+          window_orderings(order_by).each { |ordering| definition.order(ordering) }
+          definition.frame(::Arel.sql(frame)) if frame
+          definition
+        end
+
+        def window_orderings(order_by)
+          return order_by.map { |column, direction| window_ordering(column, direction) } if order_by.is_a?(Hash)
+
+          Array(order_by).map { |column| window_ordering(column, :asc) }
+        end
+
+        def window_ordering(column, direction)
+          direction.to_s == "desc" ? klass.arel_table[column].desc : klass.arel_table[column].asc
+        end
+      end
+
+      # Projects a dictionary lookup alongside the row — dictGet replaces the
+      # dimension JOIN of a star schema. Dictionary/attribute names travel as quoted
+      # string literals; only the alias needs identifier validation.
+      module RelationDictionaries
+        def dict_get(dictionary, attribute, key:, as: nil, default: nil)
+          spawn.dict_get!(dictionary, attribute, key: key, as: as, default: default)
+        end
+
+        def dict_get!(dictionary, attribute, key:, as: nil, default: nil)
+          alias_name = (as || attribute).to_s
+          raise ArgumentError, "dict_get alias must be an identifier" unless
+            RelationWindowing::WINDOW_IDENTIFIER.match?(alias_name)
+
+          self.select_values += [klass.arel_table[::Arel.star]] if select_values.empty?
+          self.select_values += [::Arel::Nodes::As.new(dict_get_node(dictionary, attribute, key, default),
+                                                       ::Arel.sql(alias_name))]
+          self
+        end
+
+        private
+
+        def dict_get_node(dictionary, attribute, key, default)
+          arguments = [::Arel::Nodes.build_quoted(dictionary.to_s), ::Arel::Nodes.build_quoted(attribute.to_s),
+                       klass.arel_table[key]]
+          return ::Arel::Nodes::NamedFunction.new("dictGet", arguments) if default.nil?
+
+          ::Arel::Nodes::NamedFunction.new("dictGetOrDefault", arguments + [::Arel::Nodes.build_quoted(default)])
+        end
+      end
+
+      # Compiles the dialect state RelationMethods accumulated into the Arel AST right
+      # before SQL generation; only RelationMethods#build_arel calls in here.
+      module RelationDialectCompilation
+        private
+
+        def apply_clickhouse_dialect(ast)
+          dialect = clickhouse_dialect
+          core = ast.cores.last
+          apply_table_modifiers(core, dialect)
+          apply_array_join(core, dialect)
+          apply_prewheres(core, dialect)
+          apply_rollup(core, dialect)
+          apply_fill(ast, dialect)
+          return unless dialect[:limit_by] || dialect[:settings]
+
+          ast.lock = Nodes::DialectSuffix.new(limit_by: dialect[:limit_by], settings: dialect[:settings])
+        end
+
+        def apply_array_join(core, dialect)
+          spec = dialect[:array_join] or return
+
+          columns = spec[:columns].map { |column| klass.arel_table[column] }
+          core.source.right << Nodes::ArrayJoin.new(columns, left: spec[:left], alias_name: spec[:as])
+        end
+
+        def apply_rollup(core, dialect)
+          return unless dialect[:rollup]
+          raise ArgumentError, "rollup requires a grouped relation" if core.groups.empty?
+
+          core.groups = [::Arel::Nodes::Group.new(::Arel::Nodes::RollUp.new(core.groups.map(&:expr)))]
+        end
+
+        def apply_fill(ast, dialect)
+          fill = dialect[:fill] or return
+          raise ArgumentError, "fill requires an ordered relation (WITH FILL rides on ORDER BY)" if ast.orders.empty?
+
+          filled = Nodes::OrderWithFill.new(ast.orders.pop)
+          filled.step = fill[:step]
+          ast.orders.push(filled)
+        end
+
+        def apply_prewheres(core, dialect)
+          Array(dialect[:prewhere]).each do |condition|
+            core.wheres.unshift(Nodes::Prewhere.new(::Arel.sql(condition)))
+          end
+        end
+
+        def apply_table_modifiers(core, dialect)
+          return unless dialect[:final] || dialect[:sample]
+
+          modified = Nodes::TableWithModifiers.new(core.source.left)
+          modified.final = dialect[:final]
+          modified.sample = dialect[:sample]
+          core.source.left = modified
+        end
+      end
+
+      # Writes can't carry an in-SQL SETTINGS clause the way SELECTs do (ALTER and
+      # INSERT grammars each differ), so a relation's settings travel as per-request
+      # HTTP query parameters instead.
+      module RelationWrites
+        def update_all(...) = with_write_settings { super }
+
+        def delete_all(...) = with_write_settings { super }
+
+        def insert_all(...) = with_write_settings { super }
+
+        def insert_all!(...) = with_write_settings { super }
+
+        private
+
+        def with_write_settings(&block)
+          settings = clickhouse_dialect[:settings]
+          return yield if settings.blank?
+
+          klass.with_connection { |connection| connection.with_request_settings(settings, &block) }
+        end
+      end
+
+      # Terminal calculations over ClickHouse's aggregate-function library; separate
+      # from RelationMethods because these execute immediately rather than build state.
+      # All accept merge: true (-Merge, finishing AggregateFunction state columns) and
+      # if: condition (-If, per-row conditional aggregation in a single scan); `if` is
+      # a Ruby keyword, so it travels in **options.
+      module RelationCalculations
+        def uniq_count(column, exact: false, merge: false, **options)
+          aggregate(exact ? "uniqExact" : "uniq", [klass.arel_table[column]],
+                    merge: merge, condition: options[:if])
+        end
+
+        # Parametric aggregates render as name(param)(args); Float/Integer coercion is
+        # the injection guard for the parameter.
+        def quantile(fraction, column, merge: false, **options)
+          aggregate("quantile", [klass.arel_table[column]],
+                    parameter: Float(fraction), merge: merge, condition: options[:if])
+        end
+
+        def top_k(count, column, merge: false, **options)
+          aggregate("topK", [klass.arel_table[column]],
+                    parameter: Integer(count), merge: merge, condition: options[:if])
+        end
+
+        def arg_max(column, criterion, **options)
+          aggregate("argMax", [klass.arel_table[column], klass.arel_table[criterion]], condition: options[:if])
+        end
+
+        def arg_min(column, criterion, **options)
+          aggregate("argMin", [klass.arel_table[column], klass.arel_table[criterion]], condition: options[:if])
+        end
+
+        # Table-level row estimate from metadata — O(1), ignores relation scopes.
+        def estimated_count
+          klass.with_connection do |connection|
+            connection.select_value(<<~SQL.squish, "#{klass.name} Estimated Count").to_i
+              SELECT total_rows FROM system.tables
+              WHERE database = currentDatabase() AND name = #{connection.quote(klass.table_name)}
+            SQL
+          end
+        end
+
+        private
+
+        def aggregate(name, columns, parameter: nil, merge: false, condition: nil)
+          raise ArgumentError, "merge: and if: cannot combine (ClickHouse has no -MergeIf)" if merge && condition
+
+          function = +name
+          function << "Merge" if merge
+          function << "If" if condition
+          function << "(#{parameter})" unless parameter.nil?
+          columns << condition_node(condition) if condition
+          pick_aggregate(function, columns)
+        end
+
+        # Hashes go through the relation's own predicate builder (ranges, arrays and
+        # casting come free); strings/arrays through sanitize_sql like prewhere.
+        def condition_node(condition)
+          return klass.unscoped.where(condition).where_clause.ast if condition.is_a?(Hash)
+
+          ::Arel.sql(klass.send(:sanitize_sql_for_conditions, condition))
+        end
+
+        # Ungrouped: one value. Grouped: a hash keyed like Rails' grouped calculations
+        # (scalar for one group column, array for several).
+        def pick_aggregate(function, columns)
+          node = ::Arel::Nodes::NamedFunction.new(function, columns)
+          return unscope(:order, :select).pick(node) if group_values.empty?
+
+          pluck(*group_values, node)
+            .to_h { |row| [group_values.many? ? row[0..-2] : row.first, row.last] }
+        end
+      end
+
+      # Opt-in model concern: `include ActiveRecord::ConnectionAdapters::ClickHouse::Querying`
+      # gives every relation of the model the ClickHouse dialect surface
+      # (.final/.sample/.prewhere/.settings/.limit_by/.group_by_period/.fill/.rollup)
+      # and the OLAP calculations (.uniq_count/.quantile/.top_k/.arg_max/.arg_min).
+      module Querying
+        extend ActiveSupport::Concern
+
+        included do
+          default_scope do
+            extending(RelationMethods, RelationWindowing, RelationDictionaries,
+                      RelationDialectCompilation, RelationWrites, RelationCalculations)
+          end
+        end
+
+        class_methods do
+          delegate :final, :sample, :prewhere, :settings, :limit_by, :array_join,
+                   :group_by_period, :fill, :rollup, :window, :dict_get, :uniq_count,
+                   :quantile, :top_k, :arg_max, :arg_min, :estimated_count, to: :all
+
+          # insert_all's streaming sibling: the batch goes over the wire as one
+          # chunked INSERT instead of being rendered into a SQL string first.
+          def insert_stream(rows, columns: nil)
+            with_connection { |connection| connection.insert_stream(table_name, rows, columns: columns) }
+          end
+        end
+      end
+    end
+  end
+end