dwh 0.4.4 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 236ec4ef010d0c8645ed1199e35b5dfab533cc959ab5e6ca6a4e8bf402a1457c
-  data.tar.gz: 0cdb42b1bcd885ecaae880672efe96d8894c95cc06ee7552bb29dca69f396900
+  metadata.gz: d91ce7dc2866f584744c0636918a6e4d5a2d189d3356b08fcee2b60ee40b5317
+  data.tar.gz: 51e0c508c1a77e5302493868a3e493547ec00fc022195a1b31f581b49abb843b
 SHA512:
-  metadata.gz: 377c2e70b3bf535e5ca8b4803b73bbfd3088219173f76a3ca56806e156d2426f4d8b606c8df96ddad3839cf07c447c2814b46f110b18d680a8755e80480e67e9
-  data.tar.gz: 600bfd6233f11efb33f9b9071e05a4c8876421a7c9964c27da3e99d99c21bc39cf18d3ac60843a04b97938594c0c55bd4dc21cbf5e6b4a871e642da0b58708be
+  metadata.gz: 1ac97b11067710faab814ff16f4e17e46ffa70acc7281e3c45c86e32861feb5eb7207d9f372f29849bc7453f891241c5715624024fc82f5d660d098d812990e5
+  data.tar.gz: 2aa0820ef26779facc52dc41797403930ebb39186fc0eb81146b310395e8a0052c2d23c1468ee3991ec901204cd9d4f1ce079cf1d86d2c69c7a6616a07b2093e

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## [Unreleased]
 
+## [0.5.0] - 2026-06-19
+
+### Added
+
+- ClickHouse adapter with dedicated settings and test coverage for system and adapter behavior.
+
+### Changed
+
+- Added dialect-specific reserved keywords and aggregate functions for ClickHouse, Snowflake, and Databricks expression parsing.
+
 ## [0.4.2] - 2026-05-22
 
 ### Fixed

data/lib/dwh/adapters/click_house.rb

@@ -0,0 +1,172 @@
+module DWH
+  module Adapters
+    # ClickHouse adapter for executing analytical queries against ClickHouse databases.
+    # Uses the ClickHouse HTTP interface (default port 8123) via Faraday.
+    #
+    # @example Basic local connection
+    #   DWH.create(:clickhouse, { host: 'localhost', database: 'default' })
+    #
+    # @example With authentication
+    #   DWH.create(:clickhouse, {
+    #     host: 'my-clickhouse.example.com',
+    #     port: 8443,
+    #     protocol: 'https',
+    #     database: 'analytics',
+    #     username: 'analyst',
+    #     password: 'secret'
+    #   })
+    class ClickHouse < Adapter
+      QUERY_FORMAT = 'JSONCompact'.freeze
+
+      config :protocol, String, required: false, default: 'http', message: 'http or https'
+      config :host, String, required: true, message: 'server host ip address or domain name'
+      config :port, Integer, required: false, default: 8123, message: 'ClickHouse HTTP interface port (default 8123)'
+      config :database, String, required: false, default: 'default', message: 'database to connect to'
+      config :username, String, required: false, default: 'default', message: 'username (default: default)'
+      config :password, String, required: false, default: nil, message: 'password'
+      config :query_timeout, Integer, required: false, default: 300, message: 'query execution timeout in seconds'
+
+      def connection
+        return @connection if @connection
+
+        headers = {
+          'Content-Type' => 'text/plain',
+          'X-ClickHouse-User' => config[:username],
+          'X-ClickHouse-Database' => database
+        }
+        headers['X-ClickHouse-Key'] = config[:password] if config[:password]
+
+        @connection = Faraday.new(
+          url: "#{config[:protocol]}://#{config[:host]}:#{config[:port]}",
+          headers: headers,
+          request: { timeout: config[:query_timeout] }
+        )
+
+        @connection
+      end
+
+      def test_connection(raise_exception: false)
+        res = connection.get('/ping')
+        unless res.success? && res.body.strip == 'Ok.'
+          raise ConnectionError, "ClickHouse ping returned: #{res.body}" if raise_exception
+
+          return false
+        end
+        true
+      rescue Faraday::ConnectionFailed => e
+        raise ConnectionError, e.message if raise_exception
+
+        false
+      end
+
+      def tables(**qualifiers)
+        db = qualifiers[:schema] || database
+        sql = "SELECT name FROM system.tables WHERE database = '#{db}' AND engine NOT IN ('View', 'MaterializedView')"
+        execute_raw(sql)['data'].flatten
+      end
+
+      def metadata(table, **qualifiers)
+        db = qualifiers[:schema] || database
+        full_table = db ? "#{db}.#{table}" : table
+        # DESCRIBE returns: name, type, default_type, default_expression, comment, codec_expression, ttl_expression
+        res = execute_raw("DESCRIBE TABLE #{full_table}")
+        db_table = Table.new(table, schema: db)
+        res['data'].each do |row|
+          db_table << Column.new(name: row[0], data_type: row[1])
+        end
+        db_table
+      end
+
+      def stats(table, date_column: nil, **qualifiers)
+        db = qualifiers[:schema] || database
+        full_table = db ? "#{db}.#{table}" : table
+        sql = +'SELECT count() AS row_count'
+        sql << ", min(#{date_column}) AS date_start, max(#{date_column}) AS date_end" if date_column
+        sql << " FROM #{full_table}"
+
+        row = execute_raw(sql)['data'][0]
+        TableStats.new(
+          row_count: row[0].to_i,
+          date_start: date_column ? safe_parse_date(row[1]) : nil,
+          date_end: date_column ? safe_parse_date(row[2]) : nil
+        )
+      end
+
+      def execute(sql, format: :array, retries: 0)
+        raw = with_debug(sql) { with_retry(retries) { execute_raw(sql) } }
+        format_result(raw, format)
+      rescue ExecutionError
+        raise
+      rescue StandardError => e
+        raise ExecutionError, e.message
+      end
+
+      def execute_stream(sql, io, stats: nil, retries: 0)
+        with_debug(sql) do
+          with_retry(retries) do
+            raw = execute_raw(sql)
+            cols = raw['meta'].map { it['name'] }
+            io.write(CSV.generate_line(cols))
+            raw['data'].each do |row|
+              stats << row unless stats.nil?
+              io.write(CSV.generate_line(row))
+            end
+          end
+        end
+        io.rewind
+        io
+      rescue ExecutionError
+        raise
+      rescue StandardError => e
+        raise ExecutionError, e.message
+      end
+
+      # (see Adapter#stream)
+      def stream(sql, &block)
+        with_debug(sql) do
+          execute_raw(sql)['data'].each(&block)
+        end
+      end
+
+      private
+
+      def execute_raw(sql)
+        resp = connection.post('/') do |req|
+          req.body = "#{sql} FORMAT #{QUERY_FORMAT}"
+        end
+        raise ExecutionError, "ClickHouse error: #{resp.body}" unless resp.success?
+
+        JSON.parse(resp.body)
+      rescue Faraday::Error => e
+        raise ExecutionError, e.message
+      end
+
+      def format_result(raw, format)
+        case format.to_sym
+        when :array
+          raw['data']
+        when :object
+          cols = raw['meta'].map { it['name'] }
+          raw['data'].map { |row| cols.zip(row).to_h }
+        when :csv
+          CSV.generate do |csv|
+            csv << raw['meta'].map { it['name'] }
+            raw['data'].each { |row| csv << row }
+          end
+        when :native
+          raw
+        else
+          raise ArgumentError, "Unsupported format: #{format}"
+        end
+      end
+
+      def safe_parse_date(val)
+        return nil if val.nil? || val.to_s.empty?
+
+        Date.parse(val.to_s)
+      rescue Date::Error
+        nil
+      end
+    end
+  end
+end

data/lib/dwh/column.rb

@@ -31,23 +31,32 @@ module DWH
     end
 
     def normalized_data_type
-      case data_type
+      # Strip ClickHouse type wrappers (Nullable(T), LowCardinality(T), Array(T))
+      # so the inner type is matched by the rules below.
+      inner = unwrap_type(data_type)
+
+      case inner
       when /binary/, 'image'
         'binary'
-      when /varchar/, 'string', /text/, /char/
+      when /varchar/, 'string', /text/, /char/, /fixedstring/
         'string'
-      when 'date'
+      when 'date', 'date32'
         'date'
       when /date_time/, /datetime/, 'time', /timestamp/
         'date_time'
-      when 'int', 'integer', 'smallint', 'tinyint'
+      when 'int', 'integer', 'smallint', 'tinyint', /^int8$/, /^int16$/, /^int32$/,
+           /^uint8$/, /^uint16$/, /^uint32$/
         'integer'
-      when 'bigint', 'bit_int', 'big_integer'
+      when 'bigint', 'bit_int', 'big_integer', /^int64$/, /^int128$/, /^int256$/,
+           /^uint64$/, /^uint128$/, /^uint256$/
         'bigint'
-      when 'decimal', 'double', 'float', 'real', 'dec', 'numeric', 'money'
+      when 'decimal', 'double', 'float', 'real', 'dec', 'numeric', 'money',
+           /^float32$/, /^float64$/, /^decimal/
         'decimal'
-      when 'boolean', 'bit'
+      when 'boolean', 'bit', 'bool'
         'boolean'
+      when 'uuid'
+        'string'
       when 'number'
         if precision >= 38 && scale.zero?
           'bigint'
@@ -76,6 +85,16 @@ module DWH
       "<Column:#{name}:#{data_type}>"
     end
 
+    # Strips ClickHouse parameterized wrappers like Nullable(T), LowCardinality(T),
+    # Array(T) so the inner type can be normalised by the standard rules above.
+    # Safe to call on any type string; returns the input unchanged if no wrapper matches.
+    def unwrap_type(type)
+      inner = type.to_s.downcase
+      inner = inner.sub(/\Anullable\((.+)\)\z/, '\1')
+      inner = inner.sub(/\Alowcardinality\((.+)\)\z/, '\1')
+      inner.sub(/\Aarray\((.+)\)\z/, '\1')
+    end
+
     def titleize(name)
       # Handle underscores, dashes, and multiple spaces
       # Also preserves existing spacing patterns better

data/lib/dwh/settings/base.yml

@@ -79,3 +79,153 @@ final_pass_measure_join_type: "full" # inner left right etc
 apply_advanced_filtering_on_array_projections: false # druid needs a having clause or un-nesting
 greedy_apply_date_filters: true
 cross_universe_measure_filtering_strategy: "both" # both, final, intermediate
+
+# Standard SQL reserved keywords. Bare tokens matching these are treated as SQL
+# syntax during expression parsing, not column names.
+# Per-adapter YAMLs extend this baseline with extra_reserved_keywords.
+reserved_keywords:
+  - __STR__
+  - as
+  - cast
+  - case
+  - when
+  - then
+  - else
+  - 'null'
+  - end
+  - and
+  - or
+  - current_date
+  - current_time
+  - current_timestamp
+  - is
+  - not
+  - between
+  - like
+  - in
+  - exists
+  - any
+  - some
+  - over
+  - partition
+  - by
+  - order
+  - asc
+  - desc
+  - row
+  - rows
+  - range
+  - preceding
+  - following
+  - unbounded
+  - current
+  - interval
+  - int
+  - integer
+  - bigint
+  - smallint
+  - tinyint
+  - boolean
+  - date
+  - time
+  - timestamp
+  - varchar
+  - string
+  - decimal
+  - float
+  - real
+  - numeric
+  - double
+  - array
+  - varbinary
+  - json
+  - map
+  - from
+  - where
+  - filter
+  - distinct
+  - select
+  - having
+  - limit
+  - offset
+  - group
+  - join
+  - left
+  - right
+  - inner
+  - outer
+  - full
+  - cross
+  - on
+  - using
+  - union
+  - intersect
+  - except
+  - all
+  - with
+  - recursive
+  - lateral
+  - 'true'
+  - 'false'
+  - value
+  - epoch
+  - year
+  - month
+  - day
+  - hour
+  - minute
+  - second
+  - week
+  - quarter
+  - millisecond
+  - microsecond
+  - dow
+  - doy
+  - timezone
+  - leading
+  - trailing
+  - both
+  - nulls
+  - first
+  - last
+  - within
+  - ties
+  - at
+  - zone
+  - to
+  - ilike
+  - similar
+
+# Standard SQL aggregate functions. Presence of any of these in a formula marks
+# the expression as a measure (re-aggregating). Per-adapter YAMLs extend with
+# extra_aggregate_functions.
+aggregate_functions:
+  - sum
+  - count
+  - avg
+  - min
+  - max
+  - median
+  - mode
+  - stddev
+  - stddev_pop
+  - stddev_samp
+  - variance
+  - var_pop
+  - var_samp
+  - array_agg
+  - string_agg
+  - listagg
+  - bool_and
+  - bool_or
+  - every
+  - bit_and
+  - bit_or
+  - count_if
+  - percentile_cont
+  - percentile_disc
+  - corr
+  - covar_pop
+  - covar_samp
+  - approx_distinct
+  - approx_count_distinct

data/lib/dwh/settings/clickhouse.yml

@@ -0,0 +1,129 @@
+# ClickHouse adapter settings
+# Only overrides that differ from base.yml are listed here.
+
+# ClickHouse uses backticks for identifier quoting (same as MySQL).
+quote: "`@exp`"
+
+# Date literals
+date_literal: "toDate('@val')"
+date_time_literal: "toDateTime('@val')"
+
+# ClickHouse date arithmetic uses interval syntax compatible with SQL standard.
+date_add: "(@exp + INTERVAL @val @unit)"
+# dateDiff(unit, start, end) — note the argument order (start before end)
+date_diff: "dateDiff('@unit', @start_exp, @end_exp)"
+
+# DATE_TRUNC is supported in modern ClickHouse (v22+) as date_trunc.
+truncate_date: "date_trunc('@unit', @exp)"
+
+# ClickHouse date format function
+date_format_sql: "formatDateTime(@exp, '@format')"
+
+# ClickHouse date-part extraction
+extract_year:         'toYear(@exp)'
+extract_month:        'toMonth(@exp)'
+extract_quarter:      'toQuarter(@exp)'
+extract_day_of_year:  'toDayOfYear(@exp)'
+extract_day_of_month: 'toDayOfMonth(@exp)'
+extract_day_of_week:  'toDayOfWeek(@exp)'
+extract_week_of_year: 'toWeek(@exp)'
+extract_hour:         'toHour(@exp)'
+extract_minute:       'toMinute(@exp)'
+extract_year_month:   'CAST((toString(toYear(@exp)) || lpad(toString(toMonth(@exp)), 2, ''0'')) AS Int32)'
+
+# ClickHouse week-start helpers
+sunday_week_start_day: "toMonday((@exp + INTERVAL 1 DAY)) - INTERVAL 1 DAY"
+monday_week_start_day: "toMonday(@exp)"
+
+# Current time functions
+current_date:      "today()"
+current_timestamp: "now()"
+# current_time is less commonly used; now() returns a DateTime.
+current_time:      "now()"
+
+# Null handling — ClickHouse-native forms
+if_null:     "ifNull(@exp, @when_null)"
+null_if:     "nullIf(@exp, @target)"
+null_if_zero: "nullIf(@exp, 0)"
+
+# Array operations — ClickHouse uses hasAny / NOT hasAny
+array_in_list:     "hasAny(@exp, [@list])"
+array_exclude_list: "NOT hasAny(@exp, [@list])"
+array_unnest_join:  "ARRAY JOIN @exp AS @alias"
+
+# Capabilities
+# ClickHouse supports CTEs (WITH clause) but not CREATE TEMPORARY TABLE.
+supports_temp_tables: false
+temp_table_type: "cte"
+# Window functions supported since v21.x
+supports_window_functions: true
+supports_array_functions: true
+supports_common_table_expressions: true
+supports_full_join: true
+
+# ClickHouse-specific reserved keywords not in the standard baseline.
+extra_reserved_keywords:
+  - prewhere
+  - final
+  - sample
+  - totals
+  - rollup
+  - cube
+  - tuple
+  - array
+  - fixedstring
+  - lowcardinality
+  - nullable
+  - enum
+  - uuid
+  - ipv4
+  - ipv6
+
+# ClickHouse-specific aggregate functions not in the standard baseline.
+extra_aggregate_functions:
+  - uniq
+  - uniqexact
+  - uniqhll12
+  - uniqcombined
+  - uniqthetasketch
+  - quantile
+  - quantileexact
+  - quantileexactweighted
+  - quantiles
+  - quantilestdigest
+  - quantiletiming
+  - quantilebfloat16
+  - argmax
+  - argmin
+  - sumif
+  - countif
+  - avgif
+  - maxif
+  - minif
+  - anyif
+  - anylazyif
+  - sumarray
+  - grouparray
+  - grouparrayinsertat
+  - grouparraysample
+  - groupuniqarray
+  - skewpop
+  - skewsamp
+  - kurtpop
+  - kurtsamp
+  - entropy
+  - simplelinearregression
+  - stochasticlinearregression
+  - stochasticlogisticregression
+  - categoricaldistancefunctions
+  - any
+  - anylast
+  - anyheavy
+  - first_value
+  - last_value
+  - topk
+  - topkweighted
+  - singlevalueorempty
+  - deltasum
+  - deltasumtimestamp
+  - exponentialmovingaverage

data/lib/dwh/settings/databricks.yml

@@ -48,3 +48,14 @@ array_in_list: "EXISTS(@exp, x -> x IN (@list))"
 array_exclude_list: "NOT EXISTS(@exp, x -> x IN (@list))"
 array_unnest_join: "LATERAL VIEW EXPLODE(@exp) AS @alias"
 
+# Databricks-specific reserved keywords not in the standard baseline.
+extra_reserved_keywords:
+  - qualify
+
+# Databricks-specific aggregate functions not in the standard baseline.
+extra_aggregate_functions:
+  - collect_list
+  - collect_set
+  - percentile_approx
+  - approx_percentile
+

data/lib/dwh/settings/snowflake.yml

@@ -43,3 +43,18 @@ array_unnest_join: "LATERAL FLATTEN(INPUT => @exp) @alias"
 # null handling
 if_null: "NVL(@exp, @when_null)"
 null_if_zero: "NULLIFZERO(@exp)"
+
+# Snowflake-specific reserved keywords not in the standard baseline.
+extra_reserved_keywords:
+  - qualify
+  - pivot
+  - unpivot
+  - iff
+  - nvl
+  - decode
+
+# Snowflake-specific aggregate functions not in the standard baseline.
+extra_aggregate_functions:
+  - boolor_agg
+  - booland_agg
+  - approx_percentile

data/lib/dwh/settings.rb

@@ -79,5 +79,31 @@ module DWH
     def using_base_settings?
       @using_base
     end
+
+    # Returns the full reserved-keyword list for this adapter class
+    # (baseline from base.yml merged with any extra_reserved_keywords from the
+    # adapter's own settings file). Safe to call at class level without an instance.
+    def reserved_keywords
+      base  = Array(adapter_settings[:reserved_keywords])
+      extra = Array(adapter_settings[:extra_reserved_keywords])
+      (base + extra).map { |k| k.to_s.downcase }.uniq.freeze
+    end
+
+    # Returns the full aggregate-function list for this adapter class
+    # (baseline from base.yml merged with any extra_aggregate_functions from the
+    # adapter's own settings file). Safe to call at class level without an instance.
+    def aggregate_functions
+      base  = Array(adapter_settings[:aggregate_functions])
+      extra = Array(adapter_settings[:extra_aggregate_functions])
+      (base + extra).map { |k| k.to_s.downcase }.uniq.freeze
+    end
+
+    def reserved?(name)
+      reserved_keywords.include?(name.to_s.downcase)
+    end
+
+    def aggregate_function?(name)
+      aggregate_functions.include?(name.to_s.downcase)
+    end
   end
 end

data/lib/dwh/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DWH
-  VERSION = '0.4.4'
+  VERSION = '0.5.0'
 end

data/lib/dwh.rb

@@ -20,6 +20,7 @@ require_relative 'dwh/adapters/sqlite'
 require_relative 'dwh/adapters/athena'
 require_relative 'dwh/adapters/redshift'
 require_relative 'dwh/adapters/databricks'
+require_relative 'dwh/adapters/click_house'
 
 # DWH encapsulates the full functionality of this gem.
 #
@@ -52,6 +53,25 @@ module DWH
   register(:athena, Adapters::Athena)
   register(:redshift, Adapters::Redshift)
   register(:databricks, Adapters::Databricks)
+  register(:clickhouse, Adapters::ClickHouse)
+
+  # The raw base.yml settings, loaded once. This is the single source of
+  # truth for the standard, warehouse-agnostic dialect baseline.
+  def self.base_settings
+    @base_settings ||= YAML.load_file(Settings::BASE_SETTINGS_FILE).transform_keys(&:to_sym)
+  end
+
+  # Baseline reserved keywords shared by all adapters (from base.yml).
+  # Per-adapter sets (adapter.reserved_keywords) extend this.
+  def self.reserved_keywords
+    @reserved_keywords ||= Array(base_settings[:reserved_keywords]).map { |k| k.to_s.downcase }.uniq.freeze
+  end
+
+  # Baseline aggregate functions shared by all adapters (from base.yml).
+  # Per-adapter sets (adapter.aggregate_functions) extend this.
+  def self.aggregate_functions
+    @aggregate_functions ||= Array(base_settings[:aggregate_functions]).map { |k| k.to_s.downcase }.uniq.freeze
+  end
 
   # start_reaper
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dwh
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.5.0
 platform: ruby
 authors:
 - Ajo Abraham
@@ -154,6 +154,7 @@ files:
 - lib/dwh.rb
 - lib/dwh/adapters.rb
 - lib/dwh/adapters/athena.rb
+- lib/dwh/adapters/click_house.rb
 - lib/dwh/adapters/databricks.rb
 - lib/dwh/adapters/druid.rb
 - lib/dwh/adapters/duck_db.rb
@@ -180,6 +181,7 @@ files:
 - lib/dwh/settings.rb
 - lib/dwh/settings/athena.yml
 - lib/dwh/settings/base.yml
+- lib/dwh/settings/clickhouse.yml
 - lib/dwh/settings/databricks.yml
 - lib/dwh/settings/druid.yml
 - lib/dwh/settings/duckdb.yml