dwh 0.1.0

data/lib/dwh/adapters/sql_server.rb

@@ -0,0 +1,257 @@
+module DWH
+  module Adapters
+    # Microsoft SQL Server adapter.  This adapter requires the {https://github.com/rails-sqlserver/tiny_tds tiny_tds}
+    # gem. For Mac OS and Linux, you will need to instal FreeTDS and most likely needs OpenSSL. Please follow the
+    # the instructions there before using the adapter.
+    #
+    # Create and adatper instance using {DWH::Factory#create DWH.create}.
+    #
+    # @example Basic connection with required only options
+    #   DWH.create(:sqlserver, {host: 'localhost', database: 'my_db', username: 'sa'})
+    #
+    # @example Connect to Azuer SQL Server
+    #   DWH.create(:sqlserver, {host: 'localhost', database: 'my_db', username: 'sa', azure: true})
+    # @example Connection sending custom application name
+    #   DWH.create(:sqlserver, {host: 'localhost', database: 'my_db', username: 'sa', client_name: 'Strata CLI'})
+    #
+    # @example Pass extra connection params
+    #   DWH.create(:sqlserver, {host: 'localhost', database: 'my_db',
+    #   username: 'sa', client_name: 'Strata CLI',
+    #   extra_connection_params: {
+    #     container: true,
+    #     use_utf16: false
+    #   })
+    #
+    # @example fetch tables in database
+    #   adapter.tables
+    #
+    # @example fetch tables from another database
+    #   adapter.tables(catalog: 'other_db')
+    #
+    # @example get table metadata for table in another db
+    #   adapter.metadata('other_db.dbo.my_table') or adapter.metadata('my_table', catalog: 'other_db')
+    class SqlServer < Adapter
+      config :host, String, required: true, message: 'server host ip address or domain name'
+      config :port, Integer, required: false, default: 1433, message: 'port to connect to'
+      config :database, String, required: true, message: 'name of database to connect to'
+      config :username, String, required: true, message: 'connection username'
+      config :password, String, required: false, default: nil, message: 'connection password'
+      config :query_timeout, String, required: false, default: 3600, message: 'query execution timeout in seconds'
+      config :client_name, String, required: false, default: 'DWH Ruby Gem', message: 'The name of the connecting app'
+      config :azure, Boolean, required: false, default: false, message: 'signal whether this is an azure connection'
+
+      # (see Adapter#connection)
+      def connection
+        return @connection if @connection
+
+        properties = {
+          host: config[:host],
+          port: config[:port],
+          database: config[:database],
+          username: config[:username],
+          password: config[:password],
+          appname: config[:client_name],
+          timeout: config[:query_timeout],
+          azure: config[:azure]
+        }.merge(extra_connection_params)
+
+        @connection = TinyTds::Client.new(**properties)
+
+        @connection
+      rescue StandardError => e
+        raise ConfigError, e.message
+      end
+
+      # (see Adapter#test_connection)
+      def test_connection(raise_exception: false)
+        connection
+        true
+      rescue StandardError => e
+        raise ConnectionError, e.message if raise_exception
+
+        false
+      end
+
+      # (see Adapter#tables)
+      def tables(**qualifiers)
+        change_current_database(qualifiers[:catalog])
+        table_catalog = qualifiers[:catalog] || config[:database]
+        table_schema_where = qualifiers[:schema] ? " AND table_schema = '#{qualifiers[:schema]}'" : ''
+
+        sql = <<~SQL
+          SELECT table_name
+          FROM information_schema.tables
+          WHERE table_catalog = '#{table_catalog}'
+          #{table_schema_where}
+        SQL
+
+        res = execute(sql)
+        res.flatten
+      ensure
+        reset_current_database
+      end
+
+      # Changes the default database to the one specified here.
+      # will store the default db in @current_database attr.
+      # @param catalog [String] new database to use
+      def change_current_database(catalog = nil)
+        return unless catalog && catalog != config[:database]
+
+        @current_database = catalog
+        use(catalog)
+      end
+
+      # Resets the default database to the configured one
+      # if it was changed
+      def reset_current_database
+        return if @current_database.nil? || @current_database == config[:database]
+
+        use(config[:database])
+      end
+
+      # (see Adapter#table?)
+      def table?(table_name)
+        tables.include?(table_name)
+      end
+
+      # (see Adapter#stats)
+      def stats(table, date_column: nil, **qualifiers)
+        change_current_database(qualifiers[:catalog])
+        table_name = qualifiers[:schema] ? "#{qualifiers[:schema]}.#{table}" : table
+        sql = <<-SQL
+                    SELECT count(*) ROW_COUNT
+                        #{date_column.nil? ? nil : ", min(#{date_column}) DATE_START"}
+                        #{date_column.nil? ? nil : ", max(#{date_column}) DATE_END"}
+                    FROM #{quote(table_name)}
+        SQL
+
+        result = connection.execute(sql)
+        row = result.to_a(empty_sets: true).first
+        TableStats.new(
+          row_count: row['ROW_COUNT'],
+          date_start: row['DATE_START'],
+          date_end: row['DATE_END']
+        )
+      ensure
+        reset_current_database
+      end
+
+      # (see Adapter#metadata)
+      def metadata(table, **qualifiers)
+        db_table = Table.new table, **qualifiers
+        change_current_database(db_table.catalog)
+
+        schema_where = ''
+        schema_where = "AND table_schema = '#{db_table.schema}'" if db_table.schema.present?
+
+        sql = <<-SQL
+                    SELECT column_name, data_type, character_maximum_length, numeric_precision,numeric_scale
+                    FROM information_schema.columns
+                    WHERE table_name = '#{db_table.physical_name}'
+                    #{schema_where}
+        SQL
+        cols = execute(sql, format: 'object')
+        cols.each do |col|
+          db_table << Column.new(
+            name: col['column_name'],
+            data_type: col['data_type'],
+            precision: col['numeric_precision'],
+            scale: col['numeric_scale'],
+            max_char_length: col['character_maximum_length']
+          )
+        end
+
+        db_table
+      ensure
+        reset_current_database
+      end
+
+      # (see Adapter#execute)
+      def execute(sql, format: :array, retries: 0)
+        result = with_debug(sql) { with_retry(retries) { connection.execute(sql) } }
+
+        format = format.downcase if format.is_a?(String)
+        case format.to_sym
+        when :array
+          result.to_a(as: :array, empty_sets: true, timezone: :utc)
+        when :object
+          result.to_a(as: :hash, empty_sets: true, timezone: :utc)
+        when :csv
+          result_to_csv(result)
+        when :native
+          result
+        else
+          raise UnsupportedCapability, "Unsupported format: #{format} for this #{name}"
+        end
+      rescue TinyTds::Error => e
+        raise ExecutionError, e.message
+      end
+
+      # (see Adapter#execute_stream)
+      def execute_stream(sql, io, stats: nil, retries: 0)
+        with_debug(sql) do
+          with_retry(retries) do
+            result = connection.execute(sql)
+            io.write(CSV.generate_line(result.fields))
+            result.each(as: :array, empty_sets: true, cache_rows: false, timezone: :utc) do |row|
+              stats << row unless stats.nil?
+              io.write(CSV.generate_line(row))
+            end
+          end
+        end
+
+        io.rewind
+        io
+      rescue StandardError => e
+        raise ExecutionError, e.message
+      end
+
+      # (see Adapter#stream)
+      def stream(sql, &block)
+        with_debug(sql) do
+          result = connection.execute(sql)
+          result.each(empty_sets: true, cache_rows: false, timezone: :utc) do |row|
+            block.call(row)
+          end
+        end
+      end
+
+      def extract_day_name(exp, abbreviate: false)
+        exp = cast(exp, 'date') unless exp =~ /cast/i
+        super(exp, abbreviate: abbreviate).downcase
+      end
+
+      def extract_month_name(exp, abbreviate: false)
+        exp = cast(exp, 'date') unless exp =~ /cast/i
+        if abbreviate
+          "UPPER(LEFT(DATENAME(month, #{exp}), 3))"
+        else
+          "UPPER(DATENAME(month, #{exp}))"
+        end
+      end
+
+      def valid_config?
+        super
+        require 'tiny_tds'
+      rescue LoadError
+        raise ConfigError, "Required 'tiny_tds' gem missing. Please add it to your Gemfile."
+      end
+
+      private
+
+      def use(new_database)
+        res = connection.execute("use #{quote(new_database)}")
+        res.do
+      end
+
+      def result_to_csv(result)
+        CSV.generate do |csv|
+          csv << result.fields
+          result.each(as: :array, empty_sets: true) do |row|
+            csv << row
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dwh/adapters/trino.rb

@@ -0,0 +1,213 @@
+module DWH
+  module Adapters
+    # Trino adapter. This should work for Presto as well.
+    # This adapter requires the {https://github.com/treasure-data/trino-client-ruby trino-client-ruby} gem.
+    #
+    # Create adatper instances using {DWH::Factory#create DWH.create}.
+    #
+    # @example Basic connection with required only options
+    #   DWH.create(:trino, {host: 'localhost', catalog: 'native', username: 'Ajo'})
+    #
+    # @example Connect with extra http headers
+    #   DWH.create(:trino, {host: 'localhost', port: 8080,
+    #     catalog: 'native', username: 'Ajo',
+    #     extra_connection_params: {
+    #       http_headers: {
+    #         'X-Trino-User' => 'True User Name',
+    #         'X-Forwarded-Request' => '<request passed down from client'
+    #       }
+    #     }
+    #     })
+    class Trino < Adapter
+      config :host, String, required: true, message: 'server host ip address or domain name'
+      config :port, Integer, required: true, default: 8080, message: 'port to connect to'
+      config :ssl, Boolean, required: false, default: false, message: 'use ssl?'
+      config :catalog, String, required: true, message: 'catalog to connect to'
+      config :schema, String, required: false, message: 'default schema'
+      config :username, String, required: true, message: 'connection username'
+      config :password, String, required: false, default: nil, message: 'connection password'
+      config :query_timeout, Integer, required: false, default: 3600, message: 'query execution timeout in seconds'
+      config :client_name, String, required: false, default: 'DWH Ruby Gem', message: 'client name for tracking'
+
+      def connection
+        return @connection if @connection
+
+        ssl_setup = config[:ssl] ? { verify: false } : config[:ssl]
+
+        properties = {
+          server: "#{config[:host]}:#{config[:port]}",
+          ssl: ssl_setup,
+          schema: config[:schema],
+          catalog: config[:catalog],
+          user: config[:username],
+          password: config[:password],
+          query_timeout: config[:query_timeout],
+          source: config[:client_name]
+        }.merge(extra_connection_params)
+
+        @connection = ::Trino::Client.new(properties)
+      rescue StandardError => e
+        raise ConfigError, e.message
+      end
+
+      # (see Adapter#test_conection)
+      def test_connection(raise_exception: false)
+        connection.run('select 1')
+        true
+      rescue ::Trino::Client::TrinoHttpError, Faraday::ConnectionFailed => e
+        raise ConnectionError, e.message if raise_exception
+
+        false
+      end
+
+      # (see Adapter#tables)
+      def tables(**qualifiers)
+        catalog, schema = qualifiers.values_at(:catalog, :schema)
+        query = ['SHOW TABLES']
+        query << 'FROM' if catalog || schema
+
+        if catalog && schema
+          query << "#{catalog}.#{schema}"
+        else
+          query << catalog
+          query << schema
+        end
+
+        rows = execute(query.compact.join(' '), retries: 1)
+        rows.flatten
+      end
+
+      # (see Adapter#table?)
+      def table?(table, **qualifiers)
+        db_table = Table.new(table, **qualifiers)
+
+        query = ['SHOW TABLES']
+
+        if db_table.catalog_or_schema?
+          query << 'FROM'
+          query << db_table.fully_qualified_schema_name
+        end
+        query << "LIKE '#{db_table.physical_name}'"
+
+        rows = execute(query.compact.join(' '), retries: 1)
+        !rows.empty?
+      end
+
+      # (see Adapter#stats)
+      def stats(table, date_column: nil, **qualifiers)
+        db_table = Table.new(table, **qualifiers)
+        sql = <<-SQL
+                    SELECT count(*) ROW_COUNT
+                        #{date_column.nil? ? nil : ", min(#{date_column}) DATE_START"}
+                        #{date_column.nil? ? nil : ", max(#{date_column}) DATE_END"}
+                    FROM #{db_table.fully_qualified_table_name}
+        SQL
+
+        rows = execute(sql, retries: 1)
+        row = rows[0]
+
+        TableStats.new(
+          date_start: row[1],
+          date_end: row[2],
+          row_count: row[0]
+        )
+      end
+
+      # (see Adapter#metadata)
+      def metadata(table, **qualifiers)
+        db_table = Table.new table, **qualifiers
+        sql = "SHOW COLUMNS FROM #{db_table.fully_qualified_table_name}"
+
+        _, cols = execute(sql, format: :native, retries: 1)
+
+        cols.each do |col|
+          dt = col[1].start_with?('row(') ? 'struct' : col[1]
+          db_table << Column.new(
+            name: col[0],
+            data_type: dt
+          )
+        end
+
+        db_table
+      end
+
+      def schema?
+        config.key?(:schema)
+      end
+
+      # (see Adapter#execute)
+      def execute(sql, format: :array, retries: 2)
+        result = with_debug(sql) do
+          with_retry(retries) do
+            if format == :object
+              connection.run_with_names(sql)
+            else
+              connection.run(sql)
+            end
+          end
+        end
+
+        case format
+        when :native
+          result
+        when :csv
+          result_to_csv(result)
+        when :array
+          result[1]
+        when :object
+          result
+        else
+          raise UnsupportedCapability, "Unknown format type: #{format}. Should be :native, :array, :object, or :csv"
+        end
+      rescue ::Trino::Client::TrinoQueryError => e
+        raise ExecutionError, e.message
+      end
+
+      # (see Adapter#execute_stream)
+      def execute_stream(sql, io, stats: nil, retries: 1)
+        with_debug(sql) do
+          with_retry(retries) do
+            connection.query(sql) do |result|
+              io.write(CSV.generate_line(result.columns.map(&:name)))
+              result.each_row do |row|
+                stats << row if stats
+                io << CSV.generate_line(row)
+              end
+            end
+          end
+        end
+
+        io.rewind
+        io
+      end
+
+      # (see Adapter#stream)
+      def stream(sql, &block)
+        with_debug(sql) do
+          connection.query(sql) do |result|
+            result.each_row(&block)
+          end
+        end
+      end
+
+      def valid_config?
+        super
+        require 'trino-client'
+      rescue LoadError
+        raise ConfigError, "Required 'trino-client' gem missing. Please add it to your Gemfile."
+      end
+
+      private
+
+      def result_to_csv(result)
+        columns, rows = result
+        CSV.generate do |csv|
+          csv << columns.map(&:name)
+          rows.each do |row|
+            csv << row
+          end
+        end
+      end
+    end
+  end
+end