activerecord-clickhouse-adapter 0.1.0

data/lib/active_record/connection_adapters/clickhouse/database_statements.rb

@@ -0,0 +1,464 @@
+# frozen_string_literal: true
+
+require "date"
+require "json"
+require "securerandom"
+require "strscan"
+
+module ActiveRecord
+  module ConnectionAdapters
+    module ClickHouse
+      module DatabaseStatements
+        READ_QUERY = ActiveRecord::ConnectionAdapters::AbstractAdapter.build_read_query_regexp(
+          :select, :show, :describe, :desc, :exists, :explain, :check, :with
+        )
+        private_constant :READ_QUERY
+
+        def write_query?(sql) # :nodoc:
+          !READ_QUERY.match?(sql)
+        rescue ArgumentError # non-UTF8 SQL, mirror the built-in adapters
+          !READ_QUERY.match?(sql.b)
+        end
+
+        EXPLAIN_VARIANTS = {
+          plan: "EXPLAIN", pipeline: "EXPLAIN PIPELINE",
+          estimate: "EXPLAIN ESTIMATE", indexes: "EXPLAIN indexes = 1"
+        }.freeze
+        private_constant :EXPLAIN_VARIANTS
+
+        # Scopes extra ClickHouse settings to every request made inside the block —
+        # the write-path counterpart of a relation's in-SQL SETTINGS clause. Validated
+        # here, before with_raw_connection wraps errors into StatementInvalid.
+        def with_request_settings(settings, &block)
+          settings.each_key do |name|
+            unless /\A[a-zA-Z_][a-zA-Z0-9_]*\z/.match?(name.to_s)
+              raise ArgumentError, "invalid ClickHouse setting name: #{name.inspect}"
+            end
+          end
+          with_raw_connection { |raw_connection| raw_connection.with_request_settings(settings, &block) }
+        end
+
+        # Bulk ingestion without materializing the batch: rows (any Enumerable of
+        # Hashes, lazy included) stream to the server as one chunked
+        # JSONCompactEachRow INSERT. Returns the server-reported written row count.
+        def insert_stream(table_name, rows, columns: nil)
+          column_names = columns || stream_column_names(rows)
+          sql = insert_stream_sql(table_name, column_names)
+          lines = Enumerator.new do |yielder|
+            rows.each { |row| yielder << encoded_stream_row(row, column_names) }
+          end
+
+          stream_lines(sql, "#{table_name} Stream Insert", lines)
+        end
+
+        def explain(arel, binds = [], options = []) # :nodoc:
+          sql = "#{build_explain_clause(options)} #{to_sql(arel, binds)}"
+          result = select_all(sql, "EXPLAIN", binds)
+          ([result.columns.join("\t")] + result.rows.map { |row| row.join("\t") }).join("\n")
+        end
+
+        def build_explain_clause(options = [])
+          variant = options.first || :plan
+          EXPLAIN_VARIANTS.fetch(variant.to_sym) do
+            raise ArgumentError, "unknown EXPLAIN variant #{variant.inspect}; use #{EXPLAIN_VARIANTS.keys.inspect}"
+          end
+        end
+
+        # No autoincrement and no INSERT ... RETURNING: primary keys are generated
+        # client-side before the INSERT (the Oracle-adapter prefetch seam), but only
+        # for tables whose sorting key is a single generatable column — Rails'
+        # prefetch path cannot handle composite primary keys.
+        def prefetch_primary_key?(table_name = nil)
+          !table_name.nil? && !generatable_primary_key(table_name).nil?
+        end
+
+        # The "sequence" is the pk column itself; encode table.column so
+        # next_sequence_value can check it against the sorting key. Composite keys
+        # have no single column to generate.
+        def default_sequence_name(table_name, column_name)
+          return nil if column_name.is_a?(Array)
+
+          "#{table_name}.#{column_name}"
+        end
+
+        # Sequence names are usually the "table.column" this adapter encodes, but models
+        # may override sequence_name with a free-form label (Oracle legacy); those get
+        # integer ids on trust — the prefetch gate already vetted the table. nil means a
+        # composite primary key, which Rails' prefetch path cannot populate.
+        def next_sequence_value(sequence_name)
+          raise_ungeneratable_primary_key(sequence_name) if sequence_name.nil?
+
+          table_name, column_name = sequence_name.to_s.split(".", 2)
+          return generate_time_ordered_id unless column_name
+
+          generatable_column, sql_type = generatable_primary_key(table_name)
+          raise_ungeneratable_primary_key(sequence_name) unless generatable_column == column_name
+
+          sql_type == "UUID" ? generate_uuid_v7 : generate_time_ordered_id
+        end
+
+        # Without RETURNING the server hands nothing back after an insert, whatever a
+        # column's default expression says; only the prefetched pk is knowable.
+        def return_value_after_insert?(_column) = false
+
+        # No INSERT ... RETURNING in ClickHouse: the prefetched client-side id is the
+        # only post-insert-knowable value, so surface it aligned to the requested
+        # returning columns for _create_record's write-back. The signature is Rails'
+        # DatabaseStatements#insert contract.
+        def insert(arel, name = nil, pk = nil, id_value = nil, sequence_name = nil, binds = [], returning: nil) # rubocop:disable Metrics/ParameterLists
+          inserted_id = super(arel, name, pk, id_value, sequence_name, binds, returning: nil)
+          return inserted_id if returning.nil?
+
+          returning.map { |column| column.to_s == pk.to_s ? inserted_id : nil }
+        end
+
+        # Mutations report no affected-row count (X-ClickHouse-Summary stays all zeros —
+        # probed 2026-07-13), so matching rows are counted just before mutating. Rows
+        # touched by concurrent writes between the two statements are not reflected.
+        def update(arel, name = nil, binds = [])
+          matched_rows = mutation_target_count(arel, name)
+          result = super
+          matched_rows || result
+        end
+
+        def delete(arel, name = nil, binds = [])
+          matched_rows = mutation_target_count(arel, name)
+          result = super
+          matched_rows || result
+        end
+
+        # The abstract default (CURRENT_TIMESTAMP) is not a ClickHouse identifier.
+        # Rails stamps date (created_on) and datetime (created_at) attributes with this
+        # one expression, and only a plain DateTime coerces into both Date32 and
+        # DateTime64 in a VALUES section — now64(6) raises TYPE_MISMATCH on dates, so
+        # bulk-insert timestamps carry second precision (probed 2026-07-13, PLAN.md §2).
+        HIGH_PRECISION_CURRENT_TIMESTAMP = Arel.sql("now()").freeze
+        private_constant :HIGH_PRECISION_CURRENT_TIMESTAMP
+
+        def high_precision_current_timestamp
+          HIGH_PRECISION_CURRENT_TIMESTAMP
+        end
+
+        # Rails nests a SavepointTransaction inside any dirty transaction (e.g. the
+        # retry in create_or_find_by) even though supports_savepoints? is false. Like
+        # begin/commit/rollback (PLAN.md §5 decision 4), the savepoint verbs are honest
+        # no-ops — nothing transactional exists to restore.
+        def create_savepoint(_name = nil); end
+        def exec_rollback_to_savepoint(_name = nil); end
+        def release_savepoint(_name = nil); end
+
+        # The abstract version wraps bare DELETEs in a transaction; ClickHouse has
+        # neither, so fixtures load as TRUNCATE + batched INSERTs.
+        def insert_fixtures_set(fixture_set, tables_to_delete = [])
+          statements = tables_to_delete.map { |table| build_truncate_statement(table) }
+          statements += fixture_set.filter_map do |table_name, fixtures|
+            build_fixture_sql(fixtures, table_name) unless fixtures.empty?
+          end
+          statements.each { |statement| execute(statement, "Fixtures Load") }
+        end
+
+        private
+
+        # nil for raw-SQL mutations; callers then fall back to the server's report of
+        # zero. ORDER alone never changes how many rows match, and a LIMIT caps the
+        # count through a subquery — SELECT count() FROM (matching rows LIMIT n).
+        def mutation_target_count(arel, name)
+          ast = arel.respond_to?(:ast) ? arel.ast : arel
+          return nil unless countable_mutation?(ast)
+
+          select_value(mutation_target_count_select(ast), "#{name} Count").to_i
+        end
+
+        def mutation_target_count_select(ast)
+          matching = Arel::SelectManager.new(ast.relation)
+          ast.wheres.each { |where| matching.where(where) }
+          ast.limit.nil? ? matching.project(Arel.star.count) : capped_count_select(matching, ast.limit)
+        end
+
+        def capped_count_select(matching, limit)
+          capped = matching.project(Arel.sql("1")).take(limit.expr).as("mutation_target")
+          Arel::SelectManager.new(capped).project(Arel.star.count)
+        end
+
+        def countable_mutation?(ast)
+          ast.is_a?(Arel::Nodes::UpdateStatement) || ast.is_a?(Arel::Nodes::DeleteStatement)
+        end
+
+        GENERATABLE_ID_TYPES = /\A(?:U?Int(?:64|128|256)|UUID)\z/
+        private_constant :GENERATABLE_ID_TYPES
+
+        SORTING_KEY_COLUMN_SQL = <<~SQL.squish
+          SELECT columns.name AS name, columns.type AS type
+          FROM system.tables AS tables
+          INNER JOIN system.columns AS columns
+            ON columns.database = tables.database
+            AND columns.table = tables.name
+            AND columns.name = tables.sorting_key
+          WHERE tables.database = currentDatabase() AND tables.name = %s
+        SQL
+        private_constant :SORTING_KEY_COLUMN_SQL
+
+        # [column_name, sql_type] when the table's sorting key is one column typed
+        # widely enough to hold a generated id; nil otherwise (composite keys,
+        # expression keys, narrow integers, strings). Cached per connection because
+        # Rails re-checks prefetch_primary_key? on every create; the migration-API
+        # DDL methods (create/drop/rename_table) invalidate it.
+        def generatable_primary_key(table_name)
+          cache = @generatable_primary_keys ||= {}
+          cache.fetch(table_name.to_s) do |key|
+            cache[key] = fetch_generatable_primary_key(key)
+          end
+        end
+
+        def clear_generatable_primary_key_cache
+          @generatable_primary_keys = nil
+        end
+
+        def fetch_generatable_primary_key(table_name)
+          row = select_one(format(SORTING_KEY_COLUMN_SQL, quote(table_name)), "SCHEMA")
+          return nil unless row && GENERATABLE_ID_TYPES.match?(row["type"])
+
+          [row["name"], row["type"]]
+        end
+
+        RANDOM_ID_BITS = 22
+        private_constant :RANDOM_ID_BITS
+
+        # 41 bits of Unix milliseconds + 22 random bits = 63 bits: time-ordered like
+        # UUIDv7 and safely inside signed Int64 until the year 4707.
+        def generate_time_ordered_id
+          milliseconds = Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond)
+          (milliseconds << RANDOM_ID_BITS) | SecureRandom.random_number(1 << RANDOM_ID_BITS)
+        end
+
+        # SecureRandom.uuid_v7 needs Ruby >= 3.3; build the same layout on 3.2:
+        # 48-bit millisecond timestamp, version nibble 7, variant bits 10.
+        def generate_uuid_v7
+          return SecureRandom.uuid_v7 if SecureRandom.respond_to?(:uuid_v7)
+
+          milliseconds = Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond)
+          hex = format("%<ms>012x", ms: milliseconds) + SecureRandom.hex(10)
+          hex[12] = "7"
+          hex[16] = %w[8 9 a b].fetch(hex[16].to_i(16) & 0x3)
+          hex.insert(20, "-").insert(16, "-").insert(12, "-").insert(8, "-")
+        end
+
+        def raise_ungeneratable_primary_key(sequence_name)
+          raise ActiveRecordError, "cannot generate a primary key for #{sequence_name.inspect}: " \
+                                   "the table's sorting key must be that single column, typed " \
+                                   "at least 64 bits wide or UUID; assign ids explicitly instead"
+        end
+
+        def stream_lines(sql, name, lines)
+          log(sql, name) do |notification_payload|
+            with_raw_connection do |raw_connection|
+              result = raw_connection.execute_stream(sql, lines)
+              verified!
+              notification_payload[:clickhouse] = result.stats
+              result.written_rows
+            end
+          end
+        end
+
+        def stream_column_names(rows)
+          first_row = rows.first
+          raise ArgumentError, "insert_stream got no rows" unless first_row
+
+          first_row.keys
+        end
+
+        def insert_stream_sql(table_name, column_names)
+          quoted = column_names.map { |name| quote_column_name(name) }.join(", ")
+          "INSERT INTO #{quote_table_name(table_name)} (#{quoted}) FORMAT JSONCompactEachRow"
+        end
+
+        def encoded_stream_row(row, column_names)
+          JSON.generate(column_names.map { |name| encoded_stream_value(row[name]) })
+        end
+
+        # JSONCompactEachRow input casts strings server-side; only the types whose
+        # default Ruby JSON form the server misreads need explicit encoding.
+        def encoded_stream_value(value)
+          case value
+          when Time, DateTime then quoted_date(value)
+          when Date then value.to_s
+          when BigDecimal then value.to_s("F")
+          else value
+          end
+        end
+
+        TRAILING_COMMENTS = %r{(?:\s*/\*(?:[^*]|\*(?!/))*\*/)+\s*\z}
+        INSERT_STATEMENT = /\A\s*INSERT\b/i
+        private_constant :TRAILING_COMMENTS, :INSERT_STATEMENT
+
+        # ClickHouse reads everything after VALUES with the Values input format, which
+        # rejects trailing comments (probed 2026-07-13), so sqlcommenter tags appended
+        # by Rails' QueryLogs are hoisted to the front of INSERT statements.
+        def hoist_trailing_comments(sql)
+          return sql unless INSERT_STATEMENT.match?(sql)
+
+          comments = sql[TRAILING_COMMENTS]
+          return sql unless comments
+
+          "#{comments.strip} #{sql[0...-comments.length]}"
+        end
+
+        # Rails main (8.2) funnels execution through a QueryIntent object; released
+        # 8.1 passes the pieces positionally. One wire body serves both contracts.
+        if ActiveRecord::ConnectionAdapters.const_defined?(:QueryIntent)
+          def perform_query(raw_connection, intent)
+            execute_wire_query(raw_connection, intent.processed_sql, intent.binds, intent.type_casted_binds,
+                               notification_payload: intent.notification_payload)
+          end
+        else
+          def perform_query(raw_connection, sql, binds, type_casted_binds, prepare:, notification_payload:, batch:) # rubocop:disable Lint/UnusedMethodArgument
+            execute_wire_query(raw_connection, sql, binds, type_casted_binds,
+                               notification_payload: notification_payload)
+          end
+        end
+
+        def execute_wire_query(raw_connection, sql, binds, type_casted_binds, notification_payload:)
+          sql, params = materialize_query_params(hoist_trailing_comments(sql), binds, type_casted_binds)
+          result = raw_connection.execute(sql, params: params)
+          verified!
+          if notification_payload
+            notification_payload[:row_count] = result.rows.size
+            notification_payload[:clickhouse] = result.stats
+          end
+          result
+        end
+
+        def cast_result(raw_result)
+          if raw_result.columns.empty?
+            ActiveRecord::Result.empty(affected_rows: raw_result.written_rows)
+          else
+            ActiveRecord::Result.new(bare_column_names(raw_result.columns), cast_rows(raw_result))
+          end
+        end
+
+        def cast_rows(raw_result)
+          casters = raw_result.types.map { |type_string| Types.caster_for(type_string) }
+          raw_result.rows.map do |row|
+            row.each_with_index.map { |value, index| casters.fetch(index).cast(value) }
+          end
+        end
+
+        # With 2+ JOINs the analyzer renames a qualified star's colliding columns to
+        # "table.column" on the wire (probed 2026-07-13, PLAN.md §2); Rails maps
+        # attributes by bare name, so the qualifier goes wherever stripping it leaves
+        # the name unambiguous.
+        def bare_column_names(columns)
+          return columns if columns.none? { |name| name.include?(".") }
+
+          stripped = columns.map { |name| name.rpartition(".").last }
+          collisions = stripped.tally
+          columns.zip(stripped).map { |original, bare| collisions[bare] == 1 ? bare : original }
+        end
+
+        def affected_rows(raw_result) = raw_result.written_rows
+
+        STRING_LITERAL = /'(?:\\.|''|[^'])*'/m
+        BACKTICK_IDENTIFIER = /`(?:\\.|``|[^`])*`/m
+        private_constant :STRING_LITERAL, :BACKTICK_IDENTIFIER
+
+        # Replace `?` placeholders with ClickHouse `{pN:Type}` and collect HTTP param
+        # values, skipping `?` inside string literals and backtick identifiers.
+        def materialize_query_params(sql, binds, type_casted_binds) # rubocop:disable Metrics/MethodLength
+          return [sql, {}] if binds.blank?
+
+          scanner = StringScanner.new(sql)
+          rewritten = +""
+          params = {}
+          until scanner.eos?
+            rewritten << if scanner.skip("?")
+                           bind_placeholder(params, binds, type_casted_binds)
+                         else
+                           scan_non_placeholder_fragment(scanner)
+                         end
+          end
+          verify_bind_count(params, binds)
+          [rewritten, params]
+        end
+
+        def verify_bind_count(params, binds)
+          return if params.length == binds.length
+
+          raise ArgumentError, "wrong number of binds (#{params.length} for #{binds.length})"
+        end
+
+        def scan_non_placeholder_fragment(scanner)
+          scanner.scan(STRING_LITERAL) || scanner.scan(BACKTICK_IDENTIFIER) || scanner.scan(/[^?'`]+/) || scanner.getch
+        end
+
+        def bind_placeholder(params, binds, type_casted_binds)
+          index = params.length
+          raise ArgumentError, "more ? placeholders than binds (#{binds.length})" if index >= binds.length
+
+          name = "p#{index}"
+          params[name] = format_query_param(type_casted_binds.fetch(index))
+          "{#{name}:#{clickhouse_bind_type(binds.fetch(index), type_casted_binds.fetch(index))}}"
+        end
+
+        # The server reads String params in its escaped format: raw newlines/tabs raise
+        # BAD_QUERY_PARAMETER and a literal backslash-n would silently become a newline.
+        PARAM_ESCAPES = { "\\" => "\\\\", "\n" => "\\n", "\t" => "\\t", "\r" => "\\r", "\0" => "\\0" }.freeze
+        PARAM_ESCAPE_PATTERN = /[\\\n\t\r\0]/
+        private_constant :PARAM_ESCAPES, :PARAM_ESCAPE_PATTERN
+
+        # Adapter-level type_cast already stringifies Date/Time (with subseconds) and
+        # BigDecimal before values reach here.
+        def format_query_param(value)
+          case value
+          when nil then "\\N"
+          when true, false then value
+          else
+            string = value.to_s
+            string.match?(PARAM_ESCAPE_PATTERN) ? string.gsub(PARAM_ESCAPE_PATTERN, PARAM_ESCAPES) : string
+          end
+        end
+
+        # Date/Time binds arrive as pre-formatted strings (adapter type_cast), so only the
+        # declared AR type can recover them; everything else is inferable from the value.
+        def clickhouse_bind_type(bind, casted_value)
+          # Nullable(Nothing) is the only param type that both carries NULL and compares
+          # against any column type (probed 2026-07-13, PLAN.md §2).
+          return "Nullable(Nothing)" if casted_value.nil?
+
+          type = bind.type if bind.respond_to?(:type)
+          case type
+          when ActiveRecord::Type::Date then "Date"
+          when ActiveRecord::Type::DateTime, ActiveRecord::Type::Time then "DateTime64(6)"
+          else inferred_bind_type(casted_value)
+          end
+        end
+
+        def inferred_bind_type(casted_value)
+          case casted_value
+          when Integer then clickhouse_integer_type(casted_value)
+          when Float then "Float64"
+          when true, false then "Bool"
+          when Date then "Date"
+          when Time, DateTime then "DateTime64(6)"
+          else "String"
+          end
+        end
+
+        INTEGER_TYPE_BY_RANGE = {
+          "Int64" => -(2**63)...(2**63),
+          "UInt64" => 0...(2**64),
+          "Int128" => -(2**127)...(2**127),
+          "UInt128" => 0...(2**128),
+          "Int256" => -(2**255)...(2**255),
+          "UInt256" => 0...(2**256)
+        }.freeze
+        private_constant :INTEGER_TYPE_BY_RANGE
+
+        # Sized by magnitude — a too-small type makes the server wrap the value mod 2^N.
+        def clickhouse_integer_type(value)
+          type, = INTEGER_TYPE_BY_RANGE.find { |_, range| range.cover?(value) }
+          type || raise(ActiveModel::RangeError, "#{value} is out of range for ClickHouse integer types")
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse/error_translation.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module ClickHouse
+      class UnknownTable < ActiveRecord::StatementInvalid; end
+      class UnknownDatabase < ActiveRecord::StatementInvalid; end
+      class MemoryLimitExceeded < ActiveRecord::StatementInvalid; end
+      class QueryTimeout < ActiveRecord::StatementInvalid; end
+      class AuthenticationError < ActiveRecord::StatementInvalid; end
+
+      module ErrorTranslation
+        EXCEPTION_CLASS_BY_CODE = {
+          60 => UnknownTable,
+          81 => UnknownDatabase,
+          241 => MemoryLimitExceeded,
+          159 => QueryTimeout,
+          160 => QueryTimeout,
+          516 => AuthenticationError
+        }.freeze
+
+        # Code 53 is TYPE_MISMATCH at large; only its NULL-insert shape (surfaced by
+        # input_format_null_as_default = 0) is a Rails not-null violation.
+        NULL_INSERT_MESSAGE = /Cannot insert NULL value into a column of type/
+
+        private
+
+        def translate_exception(exception, message:, sql:, binds:)
+          server_error = server_exception_class(exception)
+          return server_error.new(message, sql: sql, binds: binds) if server_error
+
+          case exception
+          when Errno::ECONNREFUSED, SocketError, Net::OpenTimeout, Net::ReadTimeout
+            ActiveRecord::ConnectionNotEstablished.new(exception)
+          else
+            super
+          end
+        end
+
+        def server_exception_class(exception)
+          return nil unless exception.is_a?(HTTPConnection::ExecutionError)
+
+          if exception.code == 53 && exception.message.match?(NULL_INSERT_MESSAGE)
+            ActiveRecord::NotNullViolation
+          else
+            EXCEPTION_CLASS_BY_CODE[exception.code]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse/gem_version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module ClickHouse
+      VERSION = "0.1.0"
+    end
+  end
+end