dwh 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 899fb3e403f0362cb21132d9af44f622e870b339776aac7bd4d3e7ed333d59e4
-  data.tar.gz: a3d536eb6da5885d4817be013d80e6b5a5ed422bce1466ac582c2b478724e278
+  metadata.gz: e70f914cc994c4be7a9d76b0d72d170ae6bf4d895427ec90adcdf9e3099774fe
+  data.tar.gz: 3ef66bc3d9a326bbae4b51d2bb3ec6af45425971617aee3a3131c7d496cf9127
 SHA512:
-  metadata.gz: 5703c4ee14bf336b92f15740bfd0a6bce086ac9b636d67bcf269ea337a550a80bdd6a2f9d4265dd0a476d8d693a313dd2e6c8fa2fc587ea0f4653f6e0b528860
-  data.tar.gz: fa72a56f3eac7c5b4eae524a70b002f93b23e54513209d182de92e119aa43ea57a08d31d0f183b19d7df36601472045d9d2a001b26dda8bd56ee886556d4b21e
+  metadata.gz: d05640e86dc5a6df2135173dd7a513df0805b4035fa5c4c5fa182659b1286fb6fd78f67ff2e259899f3d4ecab19b44bf5e7bfb427acfa5daa6065d71fb2fa18d
+  data.tar.gz: 28e5c623c8401dea1d222a1b318325543d4c209e083944d2fd43367ae6721c2ebe1c2b7e69190462a19887770d722edb7f2d1d851f90e5cc53b4c51cf2b65953

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 ## [Unreleased]
 
+## [0.3.0] - 2026-04-22
+
+### Changed
+
+- Added Databricks Adapter
+
+## [0.2.1] - 2025-01-27
+
+### Changed
+
+- **Adapter missing-gem error messages** (Athena, DuckDB, MySQL, PostgreSQL, SQL Server, Trino): replace platform-specific system library install instructions with links to official documentation. Messages now include `gem install` and a single link for system libraries.
+
 ## [0.2.0] - 2025-10-12
 
 ### Added

data/lib/dwh/adapters/athena.rb

@@ -202,8 +202,15 @@ module DWH
       def valid_config?
         super
         require 'aws-sdk-athena'
+        require 'aws-sdk-s3'
       rescue LoadError
-        raise ConfigError, "Required 'aws-sdk-athena' and 'aws-sdk-s3' gems missing. Please add them to your Gemfile."
+        raise ConfigError, <<~MSG
+          Athena adapter requires the 'aws-sdk-athena' and 'aws-sdk-s3' gems.
+
+          Install with: gem install aws-sdk-athena aws-sdk-s3
+
+          No system libraries required (pure Ruby).
+        MSG
       end
 
       private

data/lib/dwh/adapters/databricks.rb

@@ -0,0 +1,328 @@
+require 'csv'
+require 'base64'
+
+module DWH
+  module Adapters
+    # Databricks adapter for executing SQL queries against Databricks SQL warehouses.
+    #
+    # Supports OAuth M2M (service principal) authentication only.
+    #
+    # @example Connection with OAuth (service principal)
+    #   DWH.create(:databricks, {
+    #     host: 'adb-1234567890123456.7.azuredatabricks.net',
+    #     warehouse: 'abc123def456',
+    #     oauth_client_id: 'service-principal-app-id',
+    #     oauth_client_secret: 'your-oauth-secret-here',
+    #     catalog: 'main',
+    #     schema: 'default'
+    #   })
+    class Databricks < Adapter
+      config :host, String, required: true, message: 'Databricks workspace host (e.g., adb-xxx.databricks.cloud.com)'
+      config :oauth_client_id, String, required: true, message: 'OAuth client ID (service principal application ID)'
+      config :oauth_client_secret, String, required: true, message: 'OAuth client secret'
+      config :client_name, String, required: false, default: 'Ruby DWH Gem', message: 'Client name sent to Databricks'
+      config :query_timeout, Integer, required: false, default: 3600, message: 'Query execution timeout in seconds'
+      config :warehouse, String, required: true, message: 'Databricks SQL warehouse ID to use for query execution'
+      config :catalog, String, required: false, message: 'Default catalog (Unity Catalog)'
+      config :schema, String, required: false, message: 'Default schema'
+
+      DEFAULT_POLL_INTERVAL = 0.25
+      MAX_POLL_INTERVAL = 30
+
+      STATEMENTS_API = '/api/2.0/sql/statements'.freeze
+
+      def initialize(config)
+        super
+        validate_auth_config
+      end
+
+      def connection
+        return @connection if @connection && !token_expired?
+
+        reset_connection if token_expired?
+        @connection = Faraday.new(
+          url: "https://#{workspace_host}",
+          headers: {
+            'Content-Type' => 'application/json',
+            'Authorization' => "Bearer #{auth_token}",
+            'User-Agent' => config[:client_name]
+          },
+          request: {
+            timeout: config[:query_timeout]
+          }.merge(extra_connection_params)
+        )
+      end
+
+      def test_connection(raise_exception: false)
+        execute('SELECT 1')
+        true
+      rescue StandardError => e
+        raise ConnectionError, "Failed to connect to Databricks: #{e.message}" if raise_exception
+
+        logger.error "Connection test failed: #{e.message}"
+        false
+      end
+
+      # (see Adapter#execute)
+      def execute(sql, format: :array, retries: 0)
+        result = with_retry(retries + 1) do
+          with_debug(sql) do
+            response = submit_query(sql)
+            fetch_data(handle_query_response(response))
+          end
+        end
+
+        format_result(result, format)
+      end
+
+      def execute_stream(sql, io, stats: nil, retries: 0)
+        with_retry(retries) do
+          with_debug(sql) do
+            response = submit_query(sql)
+            fetch_data(handle_query_response(response), io: io, stats: stats)
+          end
+        end
+
+        io.rewind
+        io
+      end
+
+      # Execute SQL query and yield streamed results
+      # @param sql [String] SQL query to execute
+      # @yield [chunk] yields each chunk of data as it's processed
+      def stream(sql, &block)
+        with_debug(sql) do
+          response = submit_query(sql)
+          fetch_data(handle_query_response(response), proc: block)
+        end
+      end
+
+      def tables(**qualifiers)
+        catalog = qualifiers[:catalog] || config[:catalog]
+        schema = qualifiers[:schema] || config[:schema]
+
+        raise ConfigError, 'catalog is required for Databricks tables query' unless catalog
+
+        sql = "SELECT table_name FROM #{catalog}.information_schema.tables"
+        sql += " WHERE table_schema = '#{schema}'" if schema
+
+        result = execute(sql)
+        result.flatten
+      end
+
+      def metadata(table, **qualifiers)
+        catalog = qualifiers[:catalog] || config[:catalog]
+        schema = qualifiers[:schema] || config[:schema]
+
+        raise ConfigError, 'catalog is required for Databricks metadata query' unless catalog
+
+        db_table = Table.new(table, schema: schema, catalog: catalog)
+
+        sql = <<~SQL
+          SELECT column_name, data_type, numeric_precision, numeric_scale, character_maximum_length
+          FROM #{catalog}.information_schema.columns
+          WHERE table_name = '#{db_table.physical_name}'
+        SQL
+        sql += " AND table_schema = '#{db_table.schema}'" if db_table.schema
+
+        columns = execute(sql)
+
+        columns.each do |col|
+          db_table << Column.new(
+            name: col[0]&.downcase,
+            data_type: col[1]&.downcase,
+            precision: col[2],
+            scale: col[3],
+            max_char_length: col[4]
+          )
+        end
+
+        db_table
+      end
+
+      def stats(table, date_column: nil)
+        date_fields = if date_column
+                        ", MIN(#{date_column}) AS date_start, MAX(#{date_column}) AS date_end"
+                      else
+                        ', NULL AS date_start, NULL AS date_end'
+                      end
+
+        data = execute("SELECT COUNT(*) AS row_count#{date_fields} FROM #{table}")
+        cols = data.first
+
+        TableStats.new(
+          row_count: cols[0],
+          date_start: cols[1],
+          date_end: cols[2]
+        )
+      end
+
+      private
+
+      def validate_auth_config
+        raise ConfigError, 'oauth_client_id is required' unless config[:oauth_client_id]
+        raise ConfigError, 'oauth_client_secret is required' unless config[:oauth_client_secret]
+      end
+
+      def auth_token
+        return @oauth_access_token if @oauth_access_token && !token_expired?
+
+        request_oauth_access_token!
+        @oauth_access_token
+      end
+
+      def request_oauth_access_token!
+        credentials = Base64.strict_encode64("#{config[:oauth_client_id]}:#{config[:oauth_client_secret]}")
+        response = Faraday.post(
+          "https://#{workspace_host}/oidc/v1/token",
+          'grant_type=client_credentials&scope=all-apis',
+          'Authorization' => "Basic #{credentials}",
+          'Content-Type' => 'application/x-www-form-urlencoded'
+        )
+
+        raise AuthenticationError, "OAuth M2M token request failed (#{response.status}): #{response.body}" unless response.status == 200
+
+        data = JSON.parse(response.body)
+        @oauth_access_token = data['access_token']
+        expires_in = data['expires_in'] || 3600
+        @token_expires_at = Time.now + [expires_in - 60, 60].max
+      end
+
+      def reset_connection
+        @oauth_access_token = nil
+        @token_expires_at = nil
+        close
+      end
+
+      def submit_query(sql)
+        connection.post(STATEMENTS_API) do |req|
+          req.body = {
+            statement: sql,
+            warehouse_id: config[:warehouse],
+            catalog: config[:catalog],
+            schema: config[:schema],
+            wait_timeout: '30s',
+            on_wait_timeout: 'CONTINUE',
+            format: 'JSON_ARRAY',
+            disposition: 'INLINE'
+          }.compact.merge(extra_query_params).to_json
+        end
+      end
+
+      def handle_query_response(response)
+        body = JSON.parse(response.body)
+
+        case response.status
+        when 200
+          state = body.dig('status', 'state')
+          state == 'SUCCEEDED' ? body : poll(body['statement_id'])
+        when 202
+          poll(body['statement_id'])
+        else
+          error_message = body['message'] || body['error_code'] || response.body
+          raise ExecutionError, "Databricks query failed (#{response.status}): #{error_message}"
+        end
+      end
+
+      def poll(statement_id)
+        sleep_interval = DEFAULT_POLL_INTERVAL
+
+        logger.debug "Polling for query completion: #{statement_id}"
+
+        loop do
+          response = connection.get("#{STATEMENTS_API}/#{statement_id}")
+          body = JSON.parse(response.body)
+          state = body.dig('status', 'state')
+
+          case state
+          when 'SUCCEEDED'
+            return body
+          when 'FAILED', 'CANCELED', 'CLOSED'
+            error_msg = body.dig('status', 'error', 'message') || state
+            raise ExecutionError, "Databricks query #{state}: #{error_msg}"
+          else
+            logger.debug "Query still running (state: #{state}). Sleeping #{sleep_interval}s..."
+            sleep(sleep_interval)
+            sleep_interval = sleep_interval == MAX_POLL_INTERVAL ? DEFAULT_POLL_INTERVAL : sleep_interval
+            sleep_interval = [sleep_interval * 2, MAX_POLL_INTERVAL].min
+          end
+        end
+      end
+
+      def fetch_data(result, io: nil, stats: nil, proc: nil)
+        columns = result.dig('manifest', 'schema', 'columns')&.map { |col| col['name'] } || []
+        chunks = result.dig('manifest', 'chunks') || []
+        collector = {
+          columns: columns,
+          data: [],
+          io: io,
+          stats: stats,
+          wrote_header: false
+        }
+
+        write_data(result.dig('result', 'data_array') || [], collector, io, stats, proc)
+
+        return collector unless chunks.size > 1
+
+        statement_id = result['statement_id']
+        chunks[1..].each do |chunk|
+          chunk_index = chunk['chunk_index']
+          logger.debug "Fetching chunk #{chunk_index} of #{chunks.size} for statement: #{statement_id}"
+
+          resp = connection.get("#{STATEMENTS_API}/#{statement_id}/result/chunks/#{chunk_index}")
+          raise ExecutionError, "Failed to fetch chunk #{chunk_index}: #{resp.body}" unless resp.status == 200
+
+          chunk_data = JSON.parse(resp.body)
+          write_data(chunk_data['data_array'] || [], collector, io, stats, proc)
+        end
+
+        collector
+      end
+
+      def write_data(data, collector, io = nil, stats = nil, proc = nil)
+        if io
+          unless collector[:wrote_header]
+            io << CSV.generate_line(collector[:columns])
+            collector[:wrote_header] = true
+          end
+
+          data.each do |row|
+            stats << row if stats
+            io << CSV.generate_line(row)
+          end
+        elsif proc
+          data.each { proc.call(it) }
+        else
+          data.each { collector[:data] << it }
+        end
+
+        collector
+      end
+
+      def format_result(result, format)
+        data = result[:data]
+        columns = result[:columns]
+
+        case format
+        when :array
+          data
+        when :object
+          data.map { |row| columns.zip(row).to_h }
+        when :csv
+          CSV.generate do |csv|
+            csv << columns
+            data.each { |row| csv << row }
+          end
+        when :native
+          result
+        else
+          raise UnsupportedCapability, "Unknown result format: #{format}"
+        end
+      end
+
+      def workspace_host
+        config[:host].to_s.gsub(%r{\Ahttps?://}, '').gsub(%r{/+\z}, '')
+      end
+    end
+  end
+end

data/lib/dwh/adapters/duck_db.rb

@@ -209,7 +209,13 @@ module DWH
         super
         require 'duckdb'
       rescue LoadError
-        raise ConfigError, "Required 'duckdb' gem missing. Please add it to your Gemfile."
+        raise ConfigError, <<~MSG
+          DuckDB adapter requires the 'duckdb' gem.
+
+          Install with: gem install duckdb
+
+          See https://github.com/suketa/ruby-duckdb for installation details.
+        MSG
       end
 
       private

data/lib/dwh/adapters/my_sql.rb

@@ -219,7 +219,13 @@ module DWH
         super
         require 'mysql2'
       rescue LoadError
-        raise ConfigError, "Required 'MySql2' gem missing. Please add it to your Gemfile."
+        raise ConfigError, <<~MSG
+          MySQL adapter requires the 'mysql2' gem.
+
+          Install with: gem install mysql2
+
+          System libraries: https://dev.mysql.com/downloads/
+        MSG
       end
 
       def result_to_csv(result)

data/lib/dwh/adapters/postgres.rb

@@ -221,7 +221,13 @@ module DWH
         super
         require 'pg'
       rescue LoadError
-        raise ConfigError, "Required 'pg' gem missing. Please add it to your Gemfile."
+        raise ConfigError, <<~MSG
+          PostgreSQL adapter requires the 'pg' gem.
+
+          Install with: gem install pg
+
+          System libraries: https://www.postgresql.org/download/
+        MSG
       end
 
       private

data/lib/dwh/adapters/sql_server.rb

@@ -234,7 +234,13 @@ module DWH
         super
         require 'tiny_tds'
       rescue LoadError
-        raise ConfigError, "Required 'tiny_tds' gem missing. Please add it to your Gemfile."
+        raise ConfigError, <<~MSG
+          SQL Server adapter requires the 'tiny_tds' gem.
+
+          Install with: gem install tiny_tds
+
+          System libraries (FreeTDS): https://www.freetds.org/
+        MSG
       end
 
       private

data/lib/dwh/adapters/trino.rb

@@ -194,7 +194,13 @@ module DWH
         super
         require 'trino-client'
       rescue LoadError
-        raise ConfigError, "Required 'trino-client' gem missing. Please add it to your Gemfile."
+        raise ConfigError, <<~MSG
+          Trino adapter requires the 'trino-client' gem.
+
+          Install with: gem install trino-client
+
+          No system libraries required (pure Ruby).
+        MSG
       end
 
       private

data/lib/dwh/settings/databricks.yml

@@ -1,6 +1,5 @@
-
 # quotes and string lit
-quote: "\"@exp\""
+quote: "`@exp`"
 string_literal: "'@exp'"
 
 # Date Literal Formats

data/lib/dwh/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DWH
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/lib/dwh.rb

@@ -18,6 +18,7 @@ require_relative 'dwh/adapters/duck_db'
 require_relative 'dwh/adapters/sqlite'
 require_relative 'dwh/adapters/athena'
 require_relative 'dwh/adapters/redshift'
+require_relative 'dwh/adapters/databricks'
 
 # DWH encapsulates the full functionality of this gem.
 #
@@ -49,6 +50,7 @@ module DWH
   register(:sqlite, Adapters::Sqlite)
   register(:athena, Adapters::Athena)
   register(:redshift, Adapters::Redshift)
+  register(:databricks, Adapters::Databricks)
 
   # start_reaper
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dwh
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Ajo Abraham
@@ -154,6 +154,7 @@ files:
 - lib/dwh.rb
 - lib/dwh/adapters.rb
 - lib/dwh/adapters/athena.rb
+- lib/dwh/adapters/databricks.rb
 - lib/dwh/adapters/druid.rb
 - lib/dwh/adapters/duck_db.rb
 - lib/dwh/adapters/my_sql.rb