dwh 0.1.0

data/lib/dwh/adapters.rb

@@ -0,0 +1,363 @@
+require 'csv'
+require_relative 'settings'
+require_relative 'capabilities'
+require_relative 'functions'
+require_relative 'behaviors'
+require_relative 'logger'
+
+module DWH
+  # Encapsulates functionality related to all Adapter implementations
+  module Adapters
+    class Boolean; end
+
+    # @abstract Adapters base class. All adapters should inherit from this class
+    #   and implement these core required methods:
+    #
+    #   * {#stats} - get table statistics
+    #   * {#tables} - list tables
+    #   * {#metadata} - get metadata for a specific table
+    #   * {#execute} - run query and return result
+    #   * {#execute_stream} - run query and stream results to provided IO
+    #   * {#stream} - run query and yeild streaming results
+    #
+    # Adapter implementations can declare configuration options,
+    # defaults, and whether it is required. This is a class
+    # level method. They will be validated and a {ConfigError} will be raised
+    # if there is an issue. Methods not implemented will raise {NotImplementedError}
+    #
+    # Additionally, if certain setting need to be overridden you can add a settings
+    # file in a relative directory like so: *settings/my_adapter.yml*. Or, you can specify
+    # an exact settings file location at the class level:
+    #
+    #   class MyAdapter < DWH::Adapters::Adapter
+    #     settings_file_path "my_dir/my_settings.yml"
+    #   end
+    #
+    # @example
+    #   class MyAdapter < DWH::Adapters::Adapter
+    #     config :username, String, required: true, message: "login id of the current user"
+    #     config :port, Integer, required: true, default: 5432
+    #   end
+    class Adapter
+      extend Settings
+      include Capabilities
+      include Functions
+      include Behaviors
+      include Logger
+
+      # Define the configurations required for the adapter to
+      # connect and query target database.
+      #
+      # @param name [String, Symbol] name of the configuration
+      # @param type [Constant] ruby type of the configuration
+      # @param options [Hash] options for the config
+      # @option options [Boolean] :required Whether option is required
+      # @option options [*] :default The default value
+      # @option options [String] :message The error message or info displayed
+      # @return [Hash]
+      def self.config(name, type, options = {})
+        configuration[name.to_sym] = {
+          type: type,
+          required: options[:required] || false,
+          default: options[:default],
+          message: options[:message] || "Invalid or missing parameter: #{name}",
+          allowed: options[:allowed] || []
+        }
+
+        define_method(name.to_sym) do
+          config[name.to_sym]
+        end
+      end
+
+      # Get the adapter class level configuration settings
+      # @return [Hash]
+      def self.configuration
+        @configuration ||= {}
+      end
+
+      # Instance level configurations
+      # @return [Hash] the actual instance configuration
+      attr_reader :config
+
+      def initialize(config)
+        @config = config.symbolize_keys
+        # Per instance customization of general settings
+        # So you can have multiple connections to Trino
+        # but exhibit diff behavior
+        @settings = self.class.adapter_settings.merge(
+          (config[:settings] || {}).symbolize_keys
+        )
+
+        valid_config?
+      end
+
+      # This is the actual runtime settings used by the adapter
+      # once initialized. During intialization settings could be
+      # overridden. Settings are different from configuration in that
+      # settings control behaviour and syntax while configuration
+      # determines how we connect.
+      # @return [Hash] symbolized hash of settings
+      attr_reader :settings
+
+      # Allows an already instantiated adapter to change its current settings.
+      # this might be useful in situations where behavior needs to be modified
+      # on runtime basis.
+      # @return [Hash] the complete settings with changes merged
+      def alter_settings(changes = {})
+        reset_settings unless @original_settings.nil?
+        @original_settings = @settings
+        @settings.merge!(changes)
+      end
+
+      # This returns settings back to its original state prior to
+      # running alter_settings.
+      # @return [Hash] with original settings
+      def reset_settings
+        @settings = @original_settings if @original_settings
+      end
+
+      # Creates a connection to the target database and returns the
+      # connection object or self
+      def connection
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      # Tests the connection to the target database and returns true
+      # if successful, or raise Exception or false
+      # connection object or self
+      # @return [Boolean]
+      # @raise [ConnectionError] when a connection cannot be made
+      def test_connection(raise_exception: false)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      # Test connection and raise exception if connection
+      # fails.
+      # @return [Boolean]
+      # @raise [ConnectionError]
+      def connect!
+        test_connection(raise_exception: true)
+      end
+
+      # Tests whether the dtabase can be connected
+      # @return [Boolean]
+      def connect?
+        test_connection(raise_exception: false)
+      end
+
+      # Close the connection if it was created.
+      def close
+        @connection&.close
+        @connection = nil
+      end
+
+      # Execute sql on the target database.
+      #
+      # @param sql [String] actual sql
+      # @param format [Symbol, String] return format type
+      #   - array returns array of array
+      #   - object returns array of Hashes
+      #   - csv returns as csv
+      #   - native returns the native result from any clients used
+      #     - For example: Postgres using pg client will return PG::Result
+      #     - Http clients will returns the HTTP response object
+      # @param retries [Integer] number of retries in case of failure. Default is 0
+      # @return [Array<Array>,Hash, CSV, Native]
+      # @raise [ConnectionError, ExecutionError]
+      def execute(sql, format: :array, retries: 0)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      # Execute sql and stream responses back. Data is writtent out in CSV format
+      # to the provided IO object.
+      #
+      # @param sql [String] actual sql
+      # @param io [IO] IO object to write records to
+      # @param stats [StreamingStats] collect stats and preview data this is optional
+      # @param retries [Integer] number of retries in case of failure
+      # @return [IO]
+      # @raise [ConnectionError, ExecutionError]
+      def execute_stream(sql, io, stats: nil, retries: 0)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      # Executes the given sql and yields the streamed results
+      # to the given block.
+      #
+      # @param sql [String] actual sql
+      # @yield [chunk] Yields a streamed chunk as it streams in. The chunk type
+      #   might vary depending on the target db and settings
+      # @raise [ConnectionError, ExecutionError]
+      def stream(sql, &block)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      # Returns basic stats of a given table.  Will typically include row_count,
+      # date_start, and date_end.
+      #
+      # @param table [String] table name
+      # @param date_column [String] optional date column to use to find range
+      # @option qualifiers [String] :catalog optional catalog or equivalent name space.
+      #   will be ignored if the adapter doesn't support
+      # @option qualifiers [String] :schema optional schema to scope to.
+      #   will be ignored if the adapter doesn't support
+      # @return [DWH::Table]
+      # @raise [ConnectionError, ExecutionError]
+      #
+      # @example
+      #   stats("public.big_table", date_column: "fact_date")
+      #   stats("big_table")
+      #   stats("big_table",schema: "public")
+      def stats(table, date_column: nil, **qualifiers)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      # Get all tables available in the
+      # target db.  It will use the default catalog and schema
+      # config only specified here.
+      #
+      # @option qualifiers [String] :catalog optional catalog or equivalent name space.
+      #   will be ignored if the adapter doesn't support
+      # @option qualifiers [String] :schema optional schema to scope to.
+      #   will be ignored if the adapter doesn't support
+      # @return [Array<String>]
+      def tables(**qualifiers)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      # Check if table exists in remote db.
+      #
+      # @option qualifiers [String] :catalog optional catalog or equivalent name space.
+      #   will be ignored if the adapter doesn't support
+      # @option qualifiers [String] :schema optional schema to scope to.
+      #   will be ignored if the adapter doesn't support
+      # @return [Boolean]
+      def table?(table, **qualifiers)
+        tables(**qualifiers).include?(table)
+      end
+
+      # Get the schema structure of a given a given table_name.
+      # Pass in optional catalog and schema info.
+      #
+      # Example:
+      #   metadata("public.big_table")
+      #   metadata("big_table")
+      #   metadata("big_table",schema: "public")
+      #
+      # @param table [String] - table name
+      # @option qualifiers [String] :catalog optional catalog or equivalent name space.
+      #   will be ignored if the adapter doesn't support
+      # @option qualifiers [String] :schema optional schema to scope to.
+      #   will be ignored if the adapter doesn't support
+      # @return [DWH::Table]
+      def metadata(table, **qualifiers)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      # Will call the block with retries given by the
+      # max attempts param. If max attempts is 0, it
+      # will just return the block.call
+      #
+      # @param max_attempts [Integer] max number of retries
+      def with_retry(max_attempts = 2, &block)
+        return block.call if max_attempts.zero?
+
+        attempts = 0
+
+        begin
+          attempts += 1
+          block.call
+        rescue StandardError => e
+          if attempts < max_attempts
+            logger.warn "Attempt #{attempts} failed with error: #{e.message}. Retrying..."
+            retry
+          else
+            logger.error "Failed after #{attempts} attempts with error: #{e.message}"
+            raise
+          end
+        end
+      end
+
+      # Wraps an SQL execution with debug logging.
+      # It sill include execution time.
+      #
+      # @param sql [String] actual sql being executed
+      # @return execution results (see #execute)
+      def with_debug(sql, &block)
+        starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+        logger.debug("=== SQL === \n#{sql}")
+
+        result = block.call
+
+        ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        elapsed = ending - starting
+        logger.debug("=== FINISHED SQL (#{elapsed.round(1)} secs) ===")
+
+        result
+      end
+
+      # Adapter name from the class name
+      # @return [String]
+      def adapter_name
+        self.class.name.demodulize
+      end
+
+      # If any extra connection params were passed in the config
+      # object, this will return it.
+      #
+      # @return [Hash] default empty hash
+      def extra_connection_params
+        config[:extra_connection_params] || {}
+      end
+
+      # If the adapter supports it, will pass on extra query params
+      # from the config to the executor.
+      #
+      # @return [Hash] default empty hash
+      def extra_query_params
+        config[:extra_query_params] || {}
+      end
+
+      # For adapters that uses access tokens via
+      # jwt or oauth this will return wether a tokens is
+      # expired
+      def token_expired?
+        @token_expires_at.nil? || Time.now >= @token_expires_at
+      end
+
+      protected
+
+      # Checks if the required configurations and type is passed
+      # when the adapter is initialized.
+      def valid_config?
+        definitions = self.class.configuration
+
+        # Check for missing required parameters
+        missing_params = definitions.select do |name, options|
+          options[:required] && !config.key?(name) && options[:default].nil?
+        end
+
+        if missing_params.any?
+          error_messages = missing_params.map { |name, options| "Missing #{name} param - #{options[:message]}" }
+          raise ConfigError, "#{adapter_name} Adapter: #{error_messages.join(', ')}"
+        end
+
+        definitions.each do |name, opts|
+          unless opts[:type].is_a?(Class)
+            raise ConfigError,
+                  "Adapter is not defined properly. Uknown configuration type #{opts[:type]} for #{name}. Should be a class like String, Integer etc."
+          end
+
+          raise ConfigError, "Invalid value. Only allowed: #{opts[:allowed]}." if opts[:allowed].any? && !opts[:allowed].include?(config[name])
+
+          config[name] = opts[:default] if opts[:default] && !config.key?(name)
+
+          if opts[:required] && !config[name].is_a?(opts[:type]) && !opts[:type].is_a?(Boolean)
+            raise ConfigError, "#{name} should be a #{opts[:type]}. Got #{opts[name.to_sym].class.name}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dwh/behaviors.rb

@@ -0,0 +1,67 @@
+module DWH
+  # The Behaviors module will help us write SQL queries that are
+  # optimized for the target database.  These are setup primarily for
+  # the purposes of Strata.  However, any sql writer can use this to
+  # write better sql.
+  #
+  # For exmaple temp_table_type will tell us what the preferred temporary
+  # table strategy should be.  intermediate_measure_filter? will let us know
+  # if an aggregation should be filtered in a CTE, final pass, or both.
+  module Behaviors
+    # In druid when you do specific time range you need
+    # to apply the last hour of the day to the date value
+    # to get all inclusive data for that date.
+    def extend_ending_date_to_last_hour_of_day?
+      settings[:extend_ending_date_to_last_hour_of_day]
+    end
+
+    # Defines how intermediate queries should be handled.
+    # Could be one off cte, subquery, temp (future view, permanent)
+    def temp_table_type
+      settings[:temp_table_type]
+    end
+
+    def temp_table_prefix
+      settings[:temp_table_prefix]
+    end
+
+    # When an array dimension is projected and it is filtered
+    # in the where clause, some db's like Druid needs a
+    # having clause to ensure the projected set matches
+    # the filtered set.
+    def apply_advanced_filtering_on_array_projections?
+      settings[:apply_advanced_filtering_on_array_projections]
+    end
+
+    # When measures from multiple fact universes are combined
+    # they need to be merged in a Merge stage SQL statement.
+    # The components is typically combined by Full Outer Join
+    # but could be modified as needed.
+    def final_pass_measure_join_type
+      settings[:final_pass_measure_join_type]
+    end
+
+    # When a filter on a Date time field can be applied to multiple
+    # tables in the join tree should we apply to all of them or
+    # just the first one.
+    def greedy_apply_date_filters
+      settings[:greedy_apply_date_filters]
+    end
+
+    # Whether to apply a measure filter to intermediate stages and
+    # final pass when appropriate.  This is the case of multi universe
+    # query with measure filter. Default behavior is to apply the filter
+    # in the intermediate stage and final pass.
+    def cross_universe_measure_filtering_strategy
+      settings[:cross_universe_measure_filtering_strategy]
+    end
+
+    def intermediate_measure_filter?
+      %w[both intermediate].include?(settings[:cross_universe_measure_filtering_strategy])
+    end
+
+    def final_measure_filter?
+      %w[both final].include?(settings[:cross_universe_measure_filtering_strategy])
+    end
+  end
+end

data/lib/dwh/capabilities.rb

@@ -0,0 +1,39 @@
+module DWH
+  # This module will handle database features that one might be
+  # interested in. Not every database necessarily supports
+  # all ANSI SQL features.  This minimal at this point and focused
+  # on analytical Query use cases for Strata [https://www.strata.site]
+  module Capabilities
+    def supports_table_join?
+      settings[:supports_table_join]
+    end
+
+    def supports_full_join?
+      settings[:supports_full_join]
+    end
+
+    def supports_cross_join?
+      settings[:supports_cross_join]
+    end
+
+    def supports_sub_queries?
+      settings[:supports_sub_queries]
+    end
+
+    def supports_common_table_expressions?
+      settings[:supports_common_table_expressions]
+    end
+
+    def supports_temp_tables?
+      settings[:supports_temp_tables]
+    end
+
+    def supports_window_functions?
+      settings[:supports_window_functions]
+    end
+
+    def supports_array_functions?
+      settings[:supports_array_functions]
+    end
+  end
+end

data/lib/dwh/column.rb

@@ -0,0 +1,79 @@
+module DWH
+  # Captures column metadata for a target table.
+  class Column
+    attr_reader :schema_type, :data_type, :name, :precision, :scale, :max_char_length
+
+    def initialize(name:, data_type:, precision: 0, scale: 0, schema_type: nil, max_char_length: nil)
+      @name = name.downcase
+      @precision = precision.is_a?(String) ? precision.to_i : precision
+      @scale = scale.is_a?(String) ? scale.to_i : scale
+      @data_type = data_type&.downcase
+      @schema_type = schema_type&.downcase
+      @max_char_length = max_char_length
+    end
+
+    def dim?
+      schema_type == 'dimension'
+    end
+
+    def measure?
+      schema_type == 'measure'
+    end
+
+    DEFAULT_RULES = { /[_+]+/ => ' ', /\s+id$/i => ' ID', /desc/i => 'Description' }.freeze
+    def namify(rules = DEFAULT_RULES)
+      named = name.titleize keep_id_suffix: true
+      rules.each do |k, v|
+        named = named.gsub(Regexp.new(k), v)
+      end
+
+      named
+    end
+
+    def normalized_data_type
+      case data_type
+      when /binary/, 'image'
+        'binary'
+      when /varchar/, 'string', /text/, /char/
+        'string'
+      when 'date'
+        'date'
+      when /date_time/, /datetime/, 'time', /timestamp/
+        'date_time'
+      when 'int', 'integer', 'smallint', 'tinyint'
+        'integer'
+      when 'bigint', 'bit_int', 'big_integer'
+        'bigint'
+      when 'decimal', 'double', 'float', 'real', 'dec', 'numeric', 'money'
+        'decimal'
+      when 'boolean', 'bit'
+        'boolean'
+      when 'number'
+        if precision >= 38 && scale.zero?
+          'bigint'
+        elsif scale.positive?
+          'decimal'
+        else
+          'integer'
+        end
+      else
+        'string'
+      end
+    end
+
+    def to_h
+      {
+        name: name,
+        data_type: data_type,
+        precision: precision,
+        scale: scale,
+        schema_type: schema_type,
+        max_char_length: max_char_length
+      }
+    end
+
+    def to_s
+      "<Column:#{name}:#{data_type}>"
+    end
+  end
+end

data/lib/dwh/errors.rb

@@ -0,0 +1,29 @@
+module DWH
+  # Top level Error class for lib.
+  class DWHError < StandardError; end
+
+  # ConfigError catches issues related to how an
+  # adapter was configured and instantiated.
+  class ConfigError < DWHError; end
+
+  # ExecutionError are thrown when there is a failuire
+  # to execute calls against the remote db server.
+  class ExecutionError < DWHError; end
+
+  # Connection erros are thrown when we fail to
+  # obtain a connection for the target database.
+  class ConnectionError < DWHError; end
+
+  # UnspportedCapability are thrown when calling a function
+  # that the target database does not support.
+  class UnsupportedCapability < StandardError; end
+
+  # Handle errors related to OAuth
+  class OAuthError < StandardError; end
+
+  # Handle Token Expirattion
+  class TokenExpiredError < OAuthError; end
+
+  # Hangle auth errors
+  class AuthenticationError < OAuthError; end
+end

data/lib/dwh/factory.rb

@@ -0,0 +1,125 @@
+require 'connection_pool'
+
+module DWH
+  # Manages adapters. This should be the means by which adapters
+  # are created, loaded, and pooled.
+  module Factory
+    include Logger
+
+    # Register your new adapter.
+    # @param adapter_name [String, Symbol] your adapter name. Could be different from
+    #   the class name.
+    # @param adapter_class [Class] actual class of the adapter.
+    def register(adapter_name, adapter_class)
+      raise ConfigError, 'adapter_class should be a class' unless adapter_class.is_a?(Class)
+
+      adapter_class.load_settings
+      adapters[adapter_name.to_sym] = adapter_class
+    end
+
+    # Remove the given adapter from the registry.
+    def unregister(adapter_name)
+      adapters.delete adapter_name.to_sym
+    end
+
+    # Get the adapter.
+    # @param adapter_name [String, Symbol]
+    def get_adapter(adapter_name)
+      raise "Adapter '#{adapter_name}' not found. Did you forget to register it: DWH.register(MyAdapterClass)" unless adapter?(adapter_name)
+
+      adapters[adapter_name.to_sym]
+    end
+
+    # Check if the given adapter is registered
+    # @param adapter_name [String, Symbol]
+    def adapter?(adapter_name)
+      adapters.key?(adapter_name.to_sym)
+    end
+
+    # Get the list of registed adapters
+    def adapters
+      @adapters ||= {}
+    end
+
+    # Current active pools
+    def pools
+      @pools ||= {}
+    end
+
+    # The canonical way of creating an adapter instance
+    # in DWH.
+    # @param adapter_name [String, Symbol]
+    # @param config [Hash] options hash for the target database
+    #
+    # @example connect to MySQL
+    #   DWH.create(:mysql, { host: '127.0.0.1', databse: 'mydb', username: 'me', password: 'mypwd', client_name: 'Strata CLI'})
+    # @example connect Trino
+    #   DWH.create(:trino, {host: 'localhost', catalog: 'native', username: 'Ajo'})
+    # @example connect to Druid
+    #   DWH.create(:druid, {host: 'localhost',port: 8080, protocol: 'http'})
+    def create(adapter_name, config)
+      get_adapter(adapter_name).new(config)
+    end
+
+    # Create a pool of connections for a given name and adapter.
+    # Returns existing pool if it was already created.
+    #
+    # @param name [String] custom name for your pool
+    # @param adapter_name [String, Symbol]
+    # @param config [Hash] connection options
+    # @param timeout [Integer] pool checkout time out
+    # @param size [Integer] size of the pool
+    def pool(name, adapter_name, config, timeout: 5, size: 10)
+      if pools.key?(name)
+        pools[name]
+      else
+        pools[name] = ConnectionPool.new(size: size, timeout: timeout) do
+          create(adapter_name, config)
+        end
+      end
+    end
+
+    # Shutdown a specific pool or all pools
+    # @param pool [String, ConnectionPool, nil] pool or name of pool
+    #   or nil to shut everything down
+    def shutdown(pool = nil)
+      case pool.class
+      when String
+        pools[pool].shutdown { it.close }
+        pools.delete(pool)
+      when Symbol
+        pools[pool.to_s].shutdown { it.close }
+        pools[pool.to_s].delete
+      when ConnectionPool
+        pool.shutdown { it.close }
+        pools.delete(pools.key(pool))
+      else
+        pools.each_value do |val|
+          val.shutdown { c.close }
+        end
+        @pools = {}
+      end
+    end
+
+    # Start reaper that will periodically clean up
+    # unused or idle connections.
+    # @param frequency [Integer] defaults to 300 seconds
+    def start_reaper(frequency = 300)
+      logger.info 'Starting DB Adapter reaper process'
+      Thread.new do
+        loop do
+          pools.each do |name, pool|
+            logger.info "DB POOL FOR #{name} STATS:"
+            pool.with do
+              logger.info "\tSize:      #{pool.size}"
+              logger.info "\tIdle:      #{pool.available}"
+              logger.info "\tAvailable: #{pool.available}"
+            end
+            pool.reap(frequency) { it.close }
+          end
+          sleep frequency
+        end
+      end
+    end
+  end
+end