dwh 0.1.0

data/lib/dwh/adapters/my_sql.rb

@@ -0,0 +1,235 @@
+module DWH
+  module Adapters
+    # MySql Adapter. To use this adapter make sure you have the
+    # {https://github.com/brianmario/mysql2 MySql2 Gem} installed. You
+    # can also pass additional connection properties via {Adapter#extra_connection_params}
+    # config property.
+    #
+    # MySql concept of database maps to schema in this adapter. This is only important
+    # for the metadata methods where you want to pull up tables from a different
+    # database   (aka schema).
+    #
+    # @example Connecting to Localhost
+    #   Please use 127.0.0.1 when using a local docker instance to run MySQl.
+    #   Otherwise the Gem will try to connect over unix socket.
+    #
+    #   DWH.create(:mysql, { host: '127.0.0.1', databse: 'mydb', username: 'me', password: 'mypwd', client_name: 'Strata CLI'})
+    #
+    # @example Connecting with SSL
+    #   DWH.create(:mysql, { host: '127.0.0.1', databse: 'mydb',
+    #       username: 'me', password: 'mypwd', ssl: true}) # this will default ssl_mode to required
+    #
+    # @example Modify the SSL mode. All extra ssl config can be passed this way.
+    #   DWH.create(:mysql, { host: '127.0.0.1', databse: 'mydb',
+    #       username: 'me', password: 'mypwd', ssl: true,
+    #       extra_connection_params: {ssl_mode: "verify"})
+    class MySql < Adapter
+      config :host, String, required: true, message: 'server host ip address or domain name'
+      config :port, Integer, required: false, default: 3306, message: 'port to connect to'
+      config :database, String, required: true, message: 'name of database to connect to'
+      config :username, String, required: true, message: 'connection username'
+      config :password, String, required: false, default: nil, message: 'connection password'
+      config :query_timeout, Integer, required: false, default: 3600, message: 'query execution timeout in seconds'
+      config :ssl, Boolean, required: false, default: false, message: 'use ssl'
+      config :client_name, String, required: false, default: 'DWH Ruby Gem', message: 'The name of the connecting app'
+
+      # (see Adapter#connection)
+      def connection
+        return @connection if @connection
+
+        set_default_ssl_mode_if_needed
+
+        properties = {
+          # Connection Settings
+          host: config[:host],
+          username: config[:username],
+          password: config[:password],
+          port: 3306,
+          database: config[:database],
+
+          # Timeout Settings
+          connect_timeout: 10,
+          read_timeout: config[:query_timeout],
+          connect_attrs: {
+            program: config[:client_name]
+          }
+        }.merge(extra_connection_params)
+
+        @connection = Mysql2::Client.new(properties)
+      rescue StandardError => e
+        raise ConfigError, e.message
+      end
+
+      # (see Adapter#test_connection)
+      def test_connection(raise_exception: false)
+        connection
+        true
+      rescue StandardError => e
+        raise ConnectionError, e.message if raise_exception
+
+        false
+      end
+
+      # (see Adapter#tables)
+      def tables(**qualifiers)
+        schema = qualifiers[:schema] || config[:database]
+        query = "
+                  SELECT
+                    t.table_name
+                  FROM information_schema.tables t
+                  WHERE t.table_schema = '#{schema}'
+                  ORDER BY t.table_name
+        "
+
+        res = connection.query(query, as: :array)
+        res.to_a.flatten
+      end
+
+      # (see Adapter#stats)
+      def stats(table, date_column: nil, **qualifiers)
+        table = "#{qualifiers[:schema]}.#{table}" if qualifiers[:schema]
+        sql = <<-SQL
+                    SELECT count(*) row_count
+                        #{date_column.nil? ? nil : ", min(#{date_column}) date_start"}
+                        #{date_column.nil? ? nil : ", max(#{date_column}) date_end"}
+                    FROM #{table}
+        SQL
+
+        result = connection.query(sql)
+
+        TableStats.new(
+          row_count: result.first['row_count'],
+          date_start: result.first['date_start'],
+          date_end: result.first['date_end']
+        )
+      end
+
+      # (see Adapter#metadata)
+      def metadata(table, **qualifiers)
+        db_table = Table.new table, schema: qualifiers[:schema]
+        schema_where = db_table.schema ? " AND table_schema = '#{db_table.schema}'" : ''
+
+        sql = <<-SQL
+        SELECT column_name, data_type, character_maximum_length, numeric_precision,numeric_scale
+        FROM information_schema.columns
+        WHERE lower(table_name) = lower('#{db_table.physical_name}')
+        #{schema_where}
+        SQL
+
+        cols = execute(sql, format: :object)
+        cols.each do |col|
+          db_table << Column.new(
+            name: col['COLUMN_NAME'],
+            data_type: col['DATA_TYPE'],
+            precision: col['NUMERIC_PRECISION'],
+            scale: col['NUMERIC_SCALE'],
+            max_char_length: col['CHARACTER_MAXIMUM_LENGTH']
+          )
+        end
+
+        db_table
+      end
+
+      # (see Adapter#execute)
+      def execute(sql, format: :array, retries: 0)
+        begin
+          as_param = %i[array object].include?(format) ? format : :array
+          result = with_debug(sql) { with_retry(retries) { connection.query(sql, as: as_param) } }
+        rescue StandardError => e
+          raise ExecutionError, e.message
+        end
+
+        format = format.downcase if format.is_a?(String)
+        case format.to_sym
+        when :array, :object
+          result.to_a
+        when :csv
+          result_to_csv(result)
+        when :native
+          result
+        else
+          raise UnsupportedCapability, "Unsupported format: #{format} for this #{name}"
+        end
+      end
+
+      # (see Adapter#execute_stream)
+      def execute_stream(sql, io, stats: nil, retries: 0)
+        with_debug(sql) do
+          with_retry(retries) do
+            result = connection.query(sql, stream: true, as: :array, cache_rows: false)
+            io.write(CSV.generate_line(result.fields))
+            result.each do |row|
+              io.write(CSV.generate_line(row))
+              stats << row if stats
+            end
+          end
+        end
+
+        io.rewind
+        io
+      rescue StandardError => e
+        raise ExecutionError, e.message
+      end
+
+      # (see Adapter#stream)
+      def stream(sql, &block)
+        with_debug(sql) do
+          result = connection.query(sql, as: :array, cache_rows: false)
+          result.each do |row|
+            block.call(row)
+          end
+        end
+      end
+
+      # Custom date truncation implementation. MySql doesn't offer
+      # a native function. We basially have to format it and convert back
+      # to date object.
+      # @see Dates#truncate_date
+      def truncate_date(unit, exp)
+        unit = unit.strip.downcase
+
+        case unit
+        when 'year'
+          "DATE(DATE_FORMAT(#{exp}, '%Y-01-01'))"
+        when 'quarter'
+          "DATE(DATE_ADD(DATE_FORMAT(#{exp}, '%Y-01-01'), INTERVAL (QUARTER(#{exp}) - 1) * 3 MONTH))"
+        when 'month'
+          "DATE(DATE_FORMAT(#{exp}, '%Y-%m-01'))"
+        when 'week'
+          gsk("#{settings[:week_start_day].downcase}_week_start_day")
+            .gsub(/@exp/i, exp)
+        when 'day', 'date'
+          "DATE(#{exp})"
+        when 'hour'
+          "TIMESTAMP(DATE_FORMAT(#{exp}, '%Y-%m-%d %H:00:00'))"
+        else
+          raise UnsupportedCapability, "Currently not supporting truncation at #{unit} level"
+        end
+      end
+
+      private
+
+      def set_default_ssl_mode_if_needed
+        return unless config[:ssl] && !extra_connection_params[:ssl_mode]
+
+        extra_connection_params[:sslmode] = 'required'
+      end
+
+      def valid_config?
+        super
+        require 'mysql2'
+      rescue LoadError
+        raise ConfigError, "Required 'MySql2' gem missing. Please add it to your Gemfile."
+      end
+
+      def result_to_csv(result)
+        CSV.generate do |csv|
+          csv << result.fields
+          result.each do |row|
+            csv << row
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dwh/adapters/open_authorizable.rb

@@ -0,0 +1,215 @@
+require 'base64'
+require 'securerandom'
+
+module DWH
+  module Adapters
+    # OpenAuthorizable aka OAuth module will add functionality
+    # to get and refresh access tokens for databases that supported
+    # OAuth.
+    #
+    # To use this module include it in your adapter and call the oauth_with
+    # class method.
+    #
+    # @example Endpoint that needs to use instance to generate
+    #   oauth_with authorize: ->(adapter) { "url#{config[:val]}"}, tokenize: "http://blue.com"
+    #
+    # @example Get authorization_url
+    #   adapter.authorization_url
+    #   Then capture the code and gen tokens
+    #
+    # @example Generate acess tokens
+    #   adapter.generate_oauth_tokens(code_from_authorization)
+    #   # this will also apply the tokens
+    #
+    # @example Reuse cached tokens
+    #   adapter.apply_oauth_tokens(access_token: 'myaccesstoken', refresh_token: 'rtoken', expires_at: Time.now)
+    module OpenAuthorizable
+      # rubcop:disable Style/DocumentationModule
+      module ClassMethods
+        def oauth_with(authorize:, tokenize:, default_scope: 'refresh_token')
+          @oauth_settings = { authorize: authorize, tokenize: tokenize, default_scope: default_scope }
+        end
+
+        def oauth_settings
+          raise OAuthError, 'Please configure oauth settings by calling oauth_with class method.' unless @oauth_settings
+
+          @oauth_settings
+        end
+      end
+
+      def self.included(base)
+        base.extend(ClassMethods)
+        base.config :oauth_client_id, String, required: false, message: 'OAuth client_id'
+        base.config :oauth_client_secret, String, required: false, message: 'OAuth client_secret'
+        base.config :oauth_redirect_uri, String, required: false, message: 'OAuth redirect_uri'
+        base.config :oauth_scope, String, required: false, message: 'OAuth redirect_url'
+      end
+
+      # Generate authorization URL for user to visit
+      def authorization_url(state: SecureRandom.hex(16), scope: nil)
+        params = {
+          'response_type' => 'code',
+          'client_id' => oauth_client_id,
+          'redirect_uri' => oauth_redirect_uri,
+          'state' => state,
+          'scope' => scope || oauth_scope || oauth_settings[:default_scope]
+        }.compact
+
+        uri = URI(oauth_settings[:authorize])
+        uri.query = URI.encode_www_form(params)
+        uri.to_s
+      end
+
+      # You can reuse existing tokens that were saved outside
+      # of this app by passing it here.  This could be tokens
+      # cached from a previous call to @see #generate_oauth_tokens
+      #
+      # param access_token [String] the access token
+      # @param refresh_token [String] optional refresh token
+      def apply_oauth_tokens(access_token:, refresh_token: nil, expires_at: nil)
+        @oauth_access_token = access_token
+        @oauth_refresh_token = refresh_token
+        @token_expires_at = expires_at
+      end
+
+      # Takes the given authorization code and generates new
+      # access and refresh tokens. It will also apply them.
+      # @param authorization_code [String] this code should come from
+      #   the redirect that is captured from the #authorization_url
+      def generate_oauth_tokens(authorization_code)
+        params = {
+          grant_type: 'authorization_code',
+          code: authorization_code,
+          redirect_uri: oauth_redirect_uri
+        }
+
+        response = oauth_http_client.post(oauth_tokenization_url) do |req|
+          req.headers['Content-Type'] = 'application/x-www-form-urlencoded'
+          req.headers['Authorization'] = basic_auth_header
+          req.body = URI.encode_www_form(params)
+        end
+        oauth_token_response(response)
+      end
+
+      # Refresh access token using refresh token
+      def refresh_access_token
+        raise AuthenticationError, 'No refresh token available' unless @oauth_refresh_token
+
+        params = {
+          grant_type: 'refresh_token',
+          refresh_token: @oauth_refresh_token
+        }
+
+        response = oauth_http_client.post(oauth_tokenization_url) do |req|
+          req.headers['Content-Type'] = 'application/x-www-form-urlencoded'
+          req.headers['Authorization'] = basic_auth_header
+          req.body = URI.encode_www_form(params)
+        end
+
+        oauth_token_response(response)
+      end
+
+      # This will return the current access_token or
+      # if it expired and refresh_token token is available
+      # it will generate a new token.
+      #
+      # @return [String] access token
+      # @raise [AuthenticationError]
+      def oauth_access_token
+        if token_expired? && @oauth_refresh_token
+          refresh_access_token
+
+          # return token unless exception was raised
+          @oauth_access_token
+        elsif @oauth_access_token
+          @oauth_access_token
+        else
+          raise AuthenticationError,
+                'Access token was never set. Either run the auth flow or set the tokens via apply_oauth_tokens method.'
+        end
+      end
+
+      # Check if we have a valid access token
+      def oauth_authenticated?
+        @oauth_access_token && !token_expired?
+      end
+
+      # Get current state of tokens
+      def oauth_token_info
+        {
+          access_token: @oauth_access_token,
+          refresh_token: @oauth_refresh_token,
+          expires_at: @token_expires_at,
+          expired: token_expired?,
+          authenticated: oauth_authenticated?
+        }
+      end
+
+      def validate_oauth_config
+        raise ConfigError, 'Missing config: oauth_client_id. Required for OAuth.' unless config[:oauth_client_id]
+        raise ConfigError, 'Missing config: oauth_client_secret. Required for OAuth.' unless config[:oauth_client_secret]
+        raise ConfigError, 'Missing config: oauth_redirect_url. Required for OAuth.' unless config[:oauth_redirect_uri]
+
+        oauth_settings
+      end
+
+      def oauth_settings
+        @oauth_settings ||= self.class.oauth_settings.transform_values do
+          it.is_a?(Proc) ? it.call(self) : it
+        end
+      end
+
+      def oauth_tokenization_url
+        oauth_settings[:tokenize]
+      end
+
+      protected
+
+      def basic_auth_header
+        credentials = Base64.strict_encode64("#{config[:oauth_client_id]}:#{config[:oauth_client_secret]}")
+        "Basic #{credentials}"
+      end
+
+      def oauth_http_client
+        @oauth_http_client ||= Faraday.new(
+          headers: {
+            'Content-Type' => 'application/json',
+            'User-Agent' => config[:client_name]
+          }
+        )
+      end
+
+      # Override this method to handle provider-specific token response formats
+      def oauth_token_response(response)
+        case response.status
+        when 200..299
+          data = JSON.parse(response.body)
+
+          apply_oauth_tokens(access_token: data['access_token'],
+                             refresh_token: data['refresh_token'] || @oauth_refresh_token)
+
+          # Calculate expiration time
+          expires_in = data['expires_in'] || 3600
+          @token_expires_at = Time.now + expires_in
+
+          { success: true, data: data }
+        else
+          error_data = parse_error_response(response)
+          if error_data['error'] == 'invalid_grant' && @oauth_refresh_token
+            raise TokenExpiredError, "Potentially expired refresh token. #{error_data['message']}"
+          end
+
+          raise AuthenticationError, "Token request failed: #{error_data['error']} - #{error_data['message']}"
+        end
+      end
+
+      private
+
+      def parse_error_response(response)
+        JSON.parse(response.body)
+      rescue JSON::ParserError
+        { 'error' => 'unknown', 'message' => response.body }
+      end
+    end
+  end
+end

data/lib/dwh/adapters/postgres.rb

@@ -0,0 +1,250 @@
+module DWH
+  module Adapters
+    # Postgres adapter. Please ensure the pg gem is available before using this adapter.
+    # Generally, adapters should be created using {DWH::Factory#create DWH.create}. Where a configuration
+    # is passed in as options hash or argument list.
+    #
+    # @example Basic connection with required only options
+    #   DWH.create(:postgres, {host: 'localhost', database: 'postgres',
+    #     username: 'postgres'})
+    #
+    # @example Connection with cert based SSL connection
+    #   DWH.create(:postgres, {host: 'localhost', database: 'postgres',
+    #     username: 'postgres', ssl: true,
+    #     extra_connection_params: { sslmode: 'require' })
+    #
+    #   valid sslmodes: disable, prefer, require, verify-ca, verify-full
+    #   For modes requiring Certs make sure you add the appropirate params
+    #   to extra_connection_params. (ie sslrootcert, sslcert etc.)
+    #
+    # @example Connection sending custom application name
+    #   DWH.create(:postgres, {host: 'localhost', database: 'postgres',
+    #     username: 'postgres', application_name: "Strata CLI" })
+    class Postgres < Adapter
+      config :host, String, required: true, message: 'server host ip address or domain name'
+      config :port, Integer, required: false, default: 5432, message: 'port to connect to'
+      config :database, String, required: true, message: 'name of database to connect to'
+      config :schema, String, default: 'public', message: 'schema name. defaults to "public"'
+      config :username, String, required: true, message: 'connection username'
+      config :password, String, required: false, default: nil, message: 'connection password'
+      config :query_timeout, String, required: false, default: 3600, message: 'query execution timeout in seconds'
+      config :ssl, Boolean, required: false, default: false, message: 'use ssl'
+      config :client_name, String, required: false, default: 'DWH Ruby Gem', message: 'The name of the connecting app'
+
+      # (see Adapter#connection)
+      def connection
+        return @connection if @connection
+
+        set_default_ssl_mode_if_needed
+
+        properties = {
+          host: config[:host],
+          port: config[:port],
+          dbname: config[:database],
+          user: config[:username],
+          password: config[:password],
+          application_name: config[:client_name]
+        }.merge(extra_connection_params)
+        properties[:options] = "#{properties[:options]} -c statement_timeout=#{config[:query_timeout]}s"
+
+        @connection = PG.connect(properties)
+
+        # this could be comma separated list
+        @connection.exec("SET search_path TO #{config[:schema]}") if schema?
+
+        @connection
+      rescue StandardError => e
+        raise ConfigError, e.message
+      end
+
+      # (see Adapter#test_connection)
+      def test_connection(raise_exception: false)
+        connection
+        true
+      rescue StandardError => e
+        raise ConnectionError, e.message if raise_exception
+
+        false
+      end
+
+      # (see Adapter#tables)
+      def tables(**qualifiers)
+        sql = if schema? || qualifiers[:schema]
+                <<-SQL
+                        SELECT table_name#{' '}
+                        FROM information_schema.tables
+                        WHERE table_schema in (#{qualified_schema_name(qualifiers)})
+                SQL
+              else
+                <<-SQL
+                        SELECT table_name
+                        FROM information_schema.tables
+                SQL
+              end
+
+        result = connection.exec(sql)
+        result.values.flatten
+      end
+
+      # (see Adapter#table?)
+      def table?(table_name)
+        tables.include?(table_name)
+      end
+
+      # (see Adapter#stats)
+      def stats(table, date_column: nil, **qualifiers)
+        table_name = qualifiers[:schema] ? "#{qualifiers[:schema]}.#{table}" : table
+        sql = <<-SQL
+                    SELECT count(*) ROW_COUNT
+                        #{date_column.nil? ? nil : ", min(#{date_column}) DATE_START"}
+                        #{date_column.nil? ? nil : ", max(#{date_column}) DATE_END"}
+                    FROM "#{table_name}"
+        SQL
+
+        result = connection.exec(sql)
+        TableStats.new(
+          row_count: result.first['row_count'],
+          date_start: result.first['date_start'],
+          date_end: result.first['date_end']
+        )
+      end
+
+      # (see Adapter#metadata)
+      def metadata(table, **qualifiers)
+        db_table = Table.new table, schema: qualifiers[:schema]
+
+        schema_where = ''
+        if db_table.schema.present?
+          schema_where = "AND table_schema = '#{db_table.schema}'"
+        elsif schema?
+          schema_where = "AND table_schema in (#{qualified_schema_name})"
+        end
+
+        sql = <<-SQL
+                    SELECT column_name, data_type, character_maximum_length, numeric_precision,numeric_scale
+                    FROM information_schema.columns
+                    WHERE table_name = '#{db_table.physical_name}'
+                    #{schema_where}
+        SQL
+
+        cols = execute(sql, format: 'object')
+        cols.each do |col|
+          db_table << Column.new(
+            name: col['column_name'],
+            data_type: col['data_type'],
+            precision: col['numeric_precision'],
+            scale: col['numeric_scale'],
+            max_char_length: col['character_maximum_length']
+          )
+        end
+
+        db_table
+      end
+
+      # True if the configuration was setup with a schema.
+      def schema?
+        config[:schema].present?
+      end
+
+      # (see Adapter#execute)
+      def execute(sql, format: :array, retries: 0)
+        begin
+          result = with_debug(sql) { with_retry(retries) { connection.exec(sql) } }
+        rescue StandardError => e
+          raise ExecutionError, e.message
+        end
+
+        format = format.downcase if format.is_a?(String)
+        case format.to_sym
+        when :array
+          result.values
+        when :object
+          result.to_a
+        when :csv
+          result_to_csv(result)
+        when :native
+          result
+        else
+          raise UnsupportedCapability, "Unsupported format: #{format} for this #{name}"
+        end
+      end
+
+      # (see Adapter#execute_stream)
+      def execute_stream(sql, io, stats: nil, retries: 0)
+        with_debug(sql) do
+          with_retry(retries) do
+            connection.exec(sql) do |result|
+              io.write(CSV.generate_line(result.fields))
+              result.each_row do |row|
+                stats << row unless stats.nil?
+                io.write(CSV.generate_line(row))
+              end
+            end
+          end
+        end
+
+        io.rewind
+        io
+      rescue StandardError => e
+        raise ExecutionError, e.message
+      end
+
+      # (see Adapter#stream)
+      def stream(sql, &block)
+        with_debug(sql) do
+          connection.exec(sql) do |result|
+            result.each_row do |row|
+              block.call(row)
+            end
+          end
+        end
+      end
+
+      # Need to override default add method
+      # since postgres doesn't support quarter as an
+      # interval.
+      # @param unit [String] Should be one of day, month, quarter etc
+      # @param val [String, Integer] The number of days to add
+      # @param exp [String] The sql expresssion to modify
+      def date_add(unit, val, exp)
+        if unit.downcase.strip == 'quarter'
+          unit = 'months'
+          val = val.to_i * 3
+        end
+        gsk(:date_add)
+          .gsub(/@unit/i, unit)
+          .gsub(/@val/i, val.to_s)
+          .gsub(/@exp/i, exp)
+      end
+
+      def valid_config?
+        super
+        require 'pg'
+      rescue LoadError
+        raise ConfigError, "Required 'pg' gem missing. Please add it to your Gemfile."
+      end
+
+      private
+
+      def set_default_ssl_mode_if_needed
+        return unless config[:ssl] && !extra_connection_params[:sslmode]
+
+        extra_connection_params[:sslmode] = 'require'
+      end
+
+      def qualified_schema_name(qualifiers = {})
+        qs = qualifiers[:schema] || config[:schema]
+        @qualified_schema_name ||= qs.split(',').map { |s| "'#{s}'" }.join(',')
+      end
+
+      def result_to_csv(result)
+        CSV.generate do |csv|
+          csv << result.fields
+          result.each do |row|
+            csv << row.values # default is hash
+          end
+        end
+      end
+    end
+  end
+end