dwh 0.5.1 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd8f7098719fc4a93610c3295a6bcbbf14e8487f07cb8dec86c9b95df7805b4e
-  data.tar.gz: a18967ab069c582d12bd3eaaa7d97f5242e3380c9f348e1fd2809da4c82ace88
+  metadata.gz: 1bb280be5e0be9e979828b1e35d8271dbfd35f6926cedcac3c9c134140ca8b30
+  data.tar.gz: 407efc9f63ce81f8bb166d970d5ee2d43e8cd31e1fe41f62802362ddc02653e7
 SHA512:
-  metadata.gz: 27599ae5ac8ddd1db8fddf7d9714318eb36f98cb66ff73ed71026dddb0ab4f97e8f2f060dde216e6217485b1627cdc5ce7031254aca92da20395334a61d9bb57
-  data.tar.gz: a560342b601b038bd8e1d6d30708822433b09d83bef6094a3a1839e5cd0b09dae4bdbffce45e945f43ec1b6fc6e894e1def9d1c3d2a4c76d38a6a2c7cac3f3b6
+  metadata.gz: 8773bba3799deb57bd3e3e4738287ded1f0bd8f32fc1225490f346076676f3c962dc58ad43a697e87dc86b41ce10f563af6ac0a1eb3f1ca84836ea9d468cf29d
+  data.tar.gz: af66e29a860e89af3c5f8794df099a622d3baa4f7ea5576076c1856399cc91838e3ecdf85ed70c038a8f8061b8da2e65bffc2f49ceca0596ac0457f5b35dfb46

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 ## [Unreleased]
 
+## [0.6.1] - 2026-09-30
+
+### Fixed
+
+- **BigQuery**: `quote` replaces characters BigQuery rejects in column names (parentheses and similar) so planner-generated aliases such as `Month(Post Date)` execute.
+
+## [0.6.0] - 2026-09-28
+
+### Added
+
+- Google BigQuery adapter (`:bigquery`) with dedicated settings and unit/system test coverage. Authenticates with a service-account keyfile or Application Default Credentials; requires the `google-cloud-bigquery` gem.
+
 ## [0.5.1] - 2026-08-03
 
 ### Fixed

data/README.md

@@ -35,6 +35,7 @@ The adapter only has 5 core methods (6 including the connection method).  A YAML
 - **SQL Server** - Microsoft's enterprise database
 - **ClickHouse** - High performance analytical database
 - **Databricks** - Lakehouse SQL warehouse
+- **Google BigQuery** - Google Cloud serverless warehouse
 
 ## Integrations Coming Soon
 
@@ -121,6 +122,21 @@ Run tests on  druid:
 bundle exec rake test:system:druid 
 ```
 
+Cloud warehouse tests (`test/system/cloud_*_test.rb`) run against real accounts and skip
+unless configured through environment variables, so no credentials live in the repo:
+
+| Adapter | Variables |
+|---|---|
+| Athena | `ATHENA_S3_OUTPUT`, optional `ATHENA_REGION`, `ATHENA_DATABASE`; AWS credentials from the environment |
+| BigQuery | `BIGQUERY_PROJECT`, optional `BIGQUERY_DATASET`; `GOOGLE_APPLICATION_CREDENTIALS` or gcloud ADC. Creates its own fixture tables |
+| Databricks | `DATABRICKS_HOST`, `DATABRICKS_WAREHOUSE`, `DATABRICKS_CLIENT_ID`, `DATABRICKS_CLIENT_SECRET` |
+| Redshift | `REDSHIFT_HOST`, `REDSHIFT_PASSWORD`, optional `REDSHIFT_USER`, `REDSHIFT_DATABASE`, `REDSHIFT_PORT` |
+| Snowflake | `SNOWFLAKE_ACCOUNT`, `SNOWFLAKE_PAT`, optional `SNOWFLAKE_DATABASE`; key-pair test needs `SNOWFLAKE_USER`, `SNOWFLAKE_PRIVATE_KEY` |
+
+```bash
+BIGQUERY_PROJECT=my-project BUNDLE_WITH=development bundle exec ruby -Itest test/system/cloud_bigquery_test.rb
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt.
@@ -137,4 +153,4 @@ This project is available as open source under the terms of the MIT License.
 
 ## Version
 
-Current version: 0.5.0
+Current version: 0.6.1

data/docs/guides/adapters.md

@@ -697,6 +697,22 @@ athena = DWH.create(:athena, {
 
 See full list of config options here: [athena-api](https://docs.aws.amazon.com/sdk-for-ruby/v2/api/Aws/Athena/Client.html#initialize-instance_method)
 
+## Google BigQuery Adapter
+
+Requires the `google-cloud-bigquery` gem (`gem install google-cloud-bigquery`, pure Ruby).
+
+```ruby
+bq = DWH.create(:bigquery, {
+    project_id: 'my-gcp-project',
+    dataset: 'analytics',                        # default dataset for unqualified table names
+    keyfile: '/path/to/service-account.json',   # optional; omit to use Application Default Credentials
+    query_timeout: 300                           # optional, seconds
+})
+```
+
+Without `keyfile`, credentials resolve via `GOOGLE_APPLICATION_CREDENTIALS` or `gcloud auth application-default login`.
+Extra client options can be passed with `extra_connection_params` (see `Google::Cloud::Bigquery.new`).
+
 ## Configuration Validation
 
 DWH validates configuration parameters at creation time:

data/lib/dwh/adapters/big_query.rb

@@ -0,0 +1,184 @@
+require 'csv'
+require 'timeout'
+
+module DWH
+  module Adapters
+    # Google BigQuery adapter. Requires the google-cloud-bigquery gem, which is
+    # loaded lazily on first connection so the adapter can be created without it.
+    #
+    # @example Service account keyfile
+    #   DWH.create(:bigquery, {
+    #     project_id: 'my-gcp-project',
+    #     dataset: 'analytics',
+    #     keyfile: '/path/to/service-account.json'
+    #   })
+    #
+    # @example Application Default Credentials (gcloud auth application-default login)
+    #   DWH.create(:bigquery, { project_id: 'my-gcp-project', dataset: 'analytics' })
+    class BigQuery < Adapter
+      config :project_id, String, required: true, message: 'GCP project id'
+      config :dataset, String, required: true, message: 'default dataset for unqualified table names'
+      config :keyfile, String, required: false, default: nil,
+                               message: 'path to service-account JSON; omit to use Application Default Credentials'
+      config :query_timeout, Integer, required: false, default: 300, message: 'query timeout in seconds'
+
+      # The schema API reports legacy type names; map the ones whose
+      # normalized type would otherwise be wrong (INTEGER is 64-bit in BigQuery).
+      TYPE_ALIASES = { 'INTEGER' => 'INT64', 'FLOAT' => 'FLOAT64', 'BOOLEAN' => 'BOOL', 'RECORD' => 'STRUCT' }.freeze
+
+      # BigQuery column names, and therefore SELECT aliases, may not contain these
+      # characters (https://cloud.google.com/bigquery/docs/schemas#column_names).
+      # Dots are left alone so backtick-quoted `dataset.table` paths still work.
+      INVALID_NAME_CHARS = %r{[!"$()*,/;?@\[\\\]^{}~]}
+
+      # (see Functions#quote) Replaces characters BigQuery rejects in a column
+      # name so planner-generated aliases like "Month(Post Date)" execute.
+      def quote(exp)
+        super(exp.to_s.gsub(INVALID_NAME_CHARS, '_'))
+      end
+
+      # (see Adapter#connection)
+      def connection
+        return @connection if @connection
+
+        require 'google/cloud/bigquery'
+        opts = { project_id: config[:project_id] }
+        opts[:credentials] = config[:keyfile] unless config[:keyfile].to_s.empty?
+        @connection = Google::Cloud::Bigquery.new(**opts, **extra_connection_params)
+      rescue LoadError
+        raise ConfigError, <<~MSG
+          BigQuery adapter requires the 'google-cloud-bigquery' gem.
+
+          Install with: gem install google-cloud-bigquery
+
+          No system libraries required (pure Ruby).
+        MSG
+      rescue StandardError => e
+        raise ConfigError, "Failed to connect to BigQuery: #{e.message}"
+      end
+
+      # (see Adapter#test_connection)
+      def test_connection(raise_exception: false)
+        raise ConnectionError, "Dataset '#{config[:dataset]}' not found" unless connection.dataset(config[:dataset])
+
+        true
+      rescue StandardError => e
+        raise ConnectionError, "BigQuery connection test failed: #{e.message}" if raise_exception
+
+        false
+      end
+
+      # (see Adapter#tables)
+      def tables(**qualifiers)
+        dataset_for(qualifiers).tables.all.map(&:table_id)
+      end
+
+      # (see Adapter#stats)
+      def stats(table, date_column: nil, **qualifiers)
+        sql = 'SELECT COUNT(*) AS row_count'
+        sql += ", MIN(#{date_column}) AS date_start, MAX(#{date_column}) AS date_end" if date_column
+        row = execute("#{sql} FROM `#{dataset_name(qualifiers)}.#{table}`", format: :object).first || {}
+
+        TableStats.new(row_count: row['row_count'], date_start: row['date_start'], date_end: row['date_end'])
+      end
+
+      # (see Adapter#metadata)
+      # Uses the table schema API instead of INFORMATION_SCHEMA: one call, and
+      # precision/scale/length come back as attributes rather than parsed from strings.
+      def metadata(table, **qualifiers)
+        db_table = Table.new table, schema: dataset_name(qualifiers)
+        bq_table = dataset_for(qualifiers).table(db_table.physical_name)
+        raise ExecutionError, "Table '#{db_table.schema}.#{db_table.physical_name}' not found" unless bq_table
+
+        bq_table.schema.fields.each do |field|
+          db_table << Column.new(
+            name: field.name,
+            data_type: TYPE_ALIASES.fetch(field.type, field.type),
+            precision: field.precision || 0,
+            scale: field.scale || 0,
+            max_char_length: field.max_length
+          )
+        end
+
+        db_table
+      end
+
+      # (see Adapter#execute)
+      def execute(sql, format: :array, retries: 0)
+        result = with_debug(sql) { with_retry(retries) { run_query(sql) } }
+
+        format = format.downcase if format.is_a?(String)
+        case format.to_sym
+        when :array then result[:rows]
+        when :object then result[:rows].map { |row| Hash[result[:headers].zip(row)] }
+        when :csv then rows_to_csv(result[:headers], result[:rows])
+        when :native then result
+        else raise UnsupportedCapability, "Unsupported format: #{format} for BigQuery adapter"
+        end
+      end
+
+      # (see Adapter#execute_stream)
+      def execute_stream(sql, io, stats: nil, retries: 0)
+        with_debug(sql) do
+          with_retry(retries) do
+            data = query_data(sql)
+            io.write(CSV.generate_line(headers_of(data)))
+            data.all.each do |row|
+              values = row.values
+              stats << values unless stats.nil?
+              io.write(CSV.generate_line(values))
+            end
+          end
+        end
+
+        io.rewind
+        io
+      end
+
+      # (see Adapter#stream)
+      def stream(sql, &block)
+        with_debug(sql) { query_data(sql).all.each { |row| block.call(row.values) } }
+      end
+
+      private
+
+      def dataset_name(qualifiers)
+        qualifiers[:dataset] || qualifiers[:schema] || config[:dataset]
+      end
+
+      def dataset_for(qualifiers)
+        name = dataset_name(qualifiers)
+        connection.dataset(name) || raise(ExecutionError, "Dataset '#{name}' not found")
+      end
+
+      # Runs the query and returns the gem's paged Data object. Rows are hashes
+      # keyed by column name in column order, so row.values matches data.fields.
+      def query_data(sql)
+        Timeout.timeout(config[:query_timeout]) do
+          connection.query(sql, dataset: config[:dataset], project: config[:project_id])
+        end
+      rescue DWHError
+        raise
+      rescue StandardError => e
+        raise ExecutionError, "BigQuery query failed: #{e.message}"
+      end
+
+      def run_query(sql)
+        data = query_data(sql)
+        { headers: headers_of(data), rows: data.all.map(&:values) }
+      end
+
+      # DDL/DML statements return no schema.
+      def headers_of(data)
+        data.schema&.fields&.map(&:name) || []
+      end
+
+      def rows_to_csv(headers, rows)
+        CSV.generate do |csv|
+          csv << headers
+          rows.each { |row| csv << row }
+        end
+      end
+    end
+  end
+end

data/lib/dwh/column.rb

@@ -36,7 +36,7 @@ module DWH
       inner = unwrap_type(data_type)
 
       case inner
-      when /binary/, 'image'
+      when /binary/, 'image', 'bytes'
         'binary'
       when /varchar/, 'string', /text/, /char/, /fixedstring/
         'string'
@@ -50,7 +50,7 @@ module DWH
       when 'bigint', 'bit_int', 'big_integer', /^int64$/, /^int128$/, /^int256$/,
            /^uint64$/, /^uint128$/, /^uint256$/
         'bigint'
-      when 'decimal', 'double', 'float', 'real', 'dec', 'numeric', 'money',
+      when 'decimal', 'double', 'float', 'real', 'dec', 'numeric', 'bignumeric', 'money',
            /^float32$/, /^float64$/, /^decimal/
         'decimal'
       when 'boolean', 'bit', 'bool'

data/lib/dwh/settings/bigquery.yml

@@ -0,0 +1,92 @@
+# Google BigQuery adapter settings
+# Only overrides that differ from base.yml are listed here.
+
+# BigQuery uses backticks for identifier quoting.
+quote: "`@exp`"
+
+# FORMAT_DATE uses strftime elements. base.yml's %W/%M mean
+# week-number/minute in BigQuery, so use the standard name codes.
+day_name_format: "%A"
+abbreviated_day_name_format: "%a"
+month_name_format: "%B"
+abbreviated_month_name_format: "%b"
+
+# Current time functions require parentheses.
+current_date:      "CURRENT_DATE()"
+current_time:      "CURRENT_TIME()"
+current_timestamp: "CURRENT_TIMESTAMP()"
+
+# Date literals
+date_literal:      "DATE '@val'"
+date_time_literal: "TIMESTAMP '@val'"
+
+# BigQuery puts the expression first and takes the unit as a bare keyword.
+truncate_date:   "DATE_TRUNC(@exp, @unit)"
+date_add:        "DATE_ADD(@exp, INTERVAL @val @unit)"
+date_diff:       "DATE_DIFF(@end_exp, @start_exp, @unit)"
+date_format_sql: "FORMAT_DATE('@format', @exp)"
+
+extract_year:         "EXTRACT(YEAR FROM @exp)"
+extract_month:        "EXTRACT(MONTH FROM @exp)"
+extract_quarter:      "EXTRACT(QUARTER FROM @exp)"
+extract_day_of_year:  "EXTRACT(DAYOFYEAR FROM @exp)"
+extract_day_of_month: "EXTRACT(DAY FROM @exp)"
+extract_day_of_week:  "EXTRACT(DAYOFWEEK FROM @exp)"
+extract_week_of_year: "EXTRACT(WEEK FROM @exp)"
+extract_hour:         "EXTRACT(HOUR FROM @exp)"
+extract_minute:       "EXTRACT(MINUTE FROM @exp)"
+extract_year_month:   "CAST(FORMAT_DATE('%Y%m', @exp) AS INT64)"
+
+# DATE_TRUNC(x, WEEK) is Sunday-based in BigQuery; the explicit forms
+# below are used whenever the requested week_start_day differs.
+default_week_start_day: "sunday"
+sunday_week_start_day:  "DATE_TRUNC(@exp, WEEK(SUNDAY))"
+monday_week_start_day:  "DATE_TRUNC(@exp, WEEK(MONDAY))"
+
+# Null handling
+if_null: "IFNULL(@exp, @when_null)"
+
+# Array operations via UNNEST
+array_in_list:      "EXISTS(SELECT 1 FROM UNNEST(@exp) AS x WHERE x IN (@list))"
+array_exclude_list: "NOT EXISTS(SELECT 1 FROM UNNEST(@exp) AS x WHERE x IN (@list))"
+array_unnest_join:  "CROSS JOIN UNNEST(@exp) AS @alias"
+
+# Capabilities
+# Temp tables only live inside a script/session; use CTEs instead.
+supports_temp_tables: false
+temp_table_type: "cte"
+
+# BigQuery-specific reserved keywords not in the standard baseline.
+extra_reserved_keywords:
+  - struct
+  - unnest
+  - qualify
+  - tablesample
+  - window
+  - ignore
+  - respect
+  - int64
+  - float64
+  - bytes
+  - bignumeric
+  - bool
+  - datetime
+  - geography
+  - dayofweek
+  - dayofyear
+  - isoweek
+  - isoyear
+
+# BigQuery-specific aggregate functions not in the standard baseline.
+extra_aggregate_functions:
+  - any_value
+  - countif
+  - logical_and
+  - logical_or
+  - array_concat_agg
+  - approx_quantiles
+  - approx_top_count
+  - approx_top_sum
+  - bit_xor
+  - max_by
+  - min_by

data/lib/dwh/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DWH
-  VERSION = '0.5.1'
+  VERSION = '0.6.1'
 end

data/lib/dwh.rb

@@ -21,6 +21,7 @@ require_relative 'dwh/adapters/athena'
 require_relative 'dwh/adapters/redshift'
 require_relative 'dwh/adapters/databricks'
 require_relative 'dwh/adapters/click_house'
+require_relative 'dwh/adapters/big_query'
 
 # DWH encapsulates the full functionality of this gem.
 #
@@ -54,6 +55,7 @@ module DWH
   register(:redshift, Adapters::Redshift)
   register(:databricks, Adapters::Databricks)
   register(:clickhouse, Adapters::ClickHouse)
+  register(:bigquery, Adapters::BigQuery)
 
   # The raw base.yml settings, loaded once. This is the single source of
   # truth for the standard, warehouse-agnostic dialect baseline.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dwh
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.6.1
 platform: ruby
 authors:
 - Ajo Abraham
@@ -154,6 +154,7 @@ files:
 - lib/dwh.rb
 - lib/dwh/adapters.rb
 - lib/dwh/adapters/athena.rb
+- lib/dwh/adapters/big_query.rb
 - lib/dwh/adapters/click_house.rb
 - lib/dwh/adapters/databricks.rb
 - lib/dwh/adapters/druid.rb
@@ -181,6 +182,7 @@ files:
 - lib/dwh/settings.rb
 - lib/dwh/settings/athena.yml
 - lib/dwh/settings/base.yml
+- lib/dwh/settings/bigquery.yml
 - lib/dwh/settings/clickhouse.yml
 - lib/dwh/settings/databricks.yml
 - lib/dwh/settings/druid.yml