sequel-impala 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 03ea5f2607bc4908064302d49640df3a4e34eaa3
-  data.tar.gz: 18c93756bb5918f32cb6f32612856963f634e966
+  metadata.gz: 8c663975ad49b8b733ca3dc6df8bb14960f52844
+  data.tar.gz: 6d044a81b67c39eb7176dc5fd1422427713d3ac0
 SHA512:
-  metadata.gz: 72ca2b1c7177ecc8c2db06e8b266f4a1ff67085cf7ef77d464e6b7667ce896870d905a691acf4fc2b34e67c90777dc3767cc9b3a1e3a2a9252967beaf507b566
-  data.tar.gz: fb15a8bf19c03e54179666df88da7ffc0f0be90d48a65ada20722da03f46c4fa26d910a2f3c554a93149ea29f92f565fa3bbeb43b875c8ceacfcf413b64aa161
+  metadata.gz: 9fd026394ec7f9fce434ff4490b2aa78d8a00237cb338af52bf0b58a11d4da1cd28aefdfd7d1c8a62c607bdc9c2a80b945377540683f4456af91ecb67998971f
+  data.tar.gz: 5e14cdd9f9fcee4ef7ecf89176e96dc3feece4f530b862be6e0b44bac801a9d71ad5cc65327b0145591fc5e9533f8d46d797f676422fc1f2970f3424c6a62168

data/CHANGELOG.md

@@ -0,0 +1,45 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+## [1.1.0] - 2016-11-08
+
+### Added
+- Database#compute_stats
+- Database#values
+- Database#refresh
+- Database#invalidate_search_path
+- Database#set
+- Database#{profile/profile_for} (jeremyevans)
+- Database#except_strategy (jeremyevans)
+- Support for Kerberizing impala-ruby (colinmarc)
+- Support for QueryID from impala-ruby
+- Support for Sequel::Mock
+- Experimental support for progress
+
+### Changed
+- Change types of integer/biginteger to int/bigint
+- Enable keepalive on connection to database
+- Format of this [CHANGELOG](http://keepachangelog.com/en/0.3.0/)
+- Updated thrift (colinmarc)
+- Intersect uses INNER JOIN
+
+## [1.0.1] - 2016-09-20
+
+### Added
+- rbhive adapter (jeremyevans)
+- :empty_null=>:ruby option to csv_to_parquet extension, which can support quoted CSV cells (jeremyevans)
+
+### Fixed
+- Disconnect detection in impala and rbhive adapters (jeremyevans)
+- :search_path option handling when using Sequel::SQL::AliasedExpressions (jeremyevans)
+- Make implicit qualify return an SQL::Identifier if given an unqualified string (jeremyevans)
+- Speed up multi_insert and import (jeremyevans)
+- Optimize csv_to_parquet extension by not spawning shells or unnecessary processes (jeremyevans)
+
+### Changed
+- Transfer ownership of gem over to Outcomes Insights (aguynamedryan)
+
+## [1.0.0] - 2015-12-04
+
+### Added
+- Initial public release

data/lib/impala.rb

@@ -10,16 +10,24 @@ require 'impala/version'
 require 'thrift'
 require 'time'
 require 'impala/protocol'
+begin
+require 'impala/sasl_transport'
+rescue LoadError
+  # gssapi not supported by operating system, continue as impala adapter
+  # can be used in buffered (non-SASL) mode.
+end
+require 'impala/progress_reporter'
 require 'impala/cursor'
 require 'impala/connection'
+require 'impala/thrift_patch'
 
 module Impala
   DEFAULT_HOST = 'localhost'
   DEFAULT_PORT = 21000
-  class Error < StandardError; end
-  class InvalidQueryError < Error; end
-  class ConnectionError < Error; end
-  class CursorError < Error; end
+  class InvalidQueryError < StandardError; end
+  class ConnectionError < StandardError; end
+  class CursorError < StandardError; end
+  class ParsingError < StandardError; end
 
   # Connect to an Impala server. If a block is given, it will close the
   # connection after yielding the connection to the block.
@@ -29,8 +37,8 @@ module Impala
   #    finishes
   # @return [Connection] the open connection, or, if a block is
   #    passed, the return value of the block
-  def self.connect(host=DEFAULT_HOST, port=DEFAULT_PORT)
-    connection = Connection.new(host, port)
+  def self.connect(host=DEFAULT_HOST, port=DEFAULT_PORT, options={})
+    connection = Connection.new(host, port, options)
 
     if block_given?
       begin

data/lib/impala/connection.rb

@@ -2,13 +2,15 @@ module Impala
   # This object represents a connection to an Impala server. It can be used to
   # perform queries on the database.
   class Connection
-    LOG_CONTEXT_ID = "impala-ruby"
+    attr_accessor :host, :port
 
     # Don't instantiate Connections directly; instead, use {Impala.connect}.
-    def initialize(host, port)
+    def initialize(host, port, options={})
       @host = host
       @port = port
       @connected = false
+      @options = options.dup
+      @options[:transport] ||= :buffered
       open
     end
 
@@ -20,9 +22,7 @@ module Impala
     def open
       return if @connected
 
-      socket = Thrift::Socket.new(@host, @port)
-
-      @transport = Thrift::BufferedTransport.new(socket)
+      @transport = thrift_transport(host, port)
       @transport.open
 
       proto = Thrift::BinaryProtocol.new(@transport)
@@ -30,6 +30,39 @@ module Impala
       @connected = true
     end
 
+    def thrift_transport(server, port)
+      socket = thrift_socket(server, port, @options[:timeout])
+
+      case @options[:transport]
+      when :buffered
+        return Thrift::BufferedTransport.new(socket)
+      when :sasl
+        opts = parse_sasl_params(@options[:sasl_params])
+        mechanism = opts.delete(:mechanism)
+        return SASLTransport.new(socket, mechanism, opts)
+      else
+        raise "Unrecognised transport type '#{@options[:transport]}'"
+      end
+    end
+
+    def thrift_socket(server, port, timeout)
+      socket = Thrift::Socket.new(server, port)
+      socket.timeout = timeout
+      socket
+    end
+
+    # Processes SASL connection params and returns a hash with symbol keys or a nil
+    def parse_sasl_params(sasl_params)
+      # Symbilize keys in a hash
+      if sasl_params.kind_of?(Hash)
+        return sasl_params.inject({}) do |memo,(k,v)|
+          memo[k.to_sym] = v;
+          memo
+        end
+      end
+      return nil
+    end
+
     # Close this connection. It can still be reopened with {#open}.
     def close
       return unless @connected
@@ -73,8 +106,13 @@ module Impala
       query = sanitize_query(raw_query)
       handle = send_query(query, query_options)
 
-      check_result(handle)
-      Cursor.new(handle, @service)
+      cursor = Cursor.new(handle, @service, @options)
+      cursor.wait!
+      cursor
+    end
+
+    def close_handle(handle)
+      @service.close(handle)
     end
 
     private
@@ -96,22 +134,7 @@ module Impala
         "#{key.upcase}=#{value}"
       end
 
-      @service.executeAndWait(query, LOG_CONTEXT_ID)
-    end
-
-    def check_result(handle)
-      state = @service.get_state(handle)
-      if state == Protocol::Beeswax::QueryState::EXCEPTION
-        close_handle(handle)
-        raise ConnectionError.new("The query was aborted")
-      end
-    rescue
-      close_handle(handle)
-      raise
-    end
-
-    def close_handle(handle)
-      @service.close(handle)
+      @service.query(query)
     end
   end
 end

data/lib/impala/cursor.rb

@@ -39,11 +39,11 @@ module Impala
 
     NULL = 'NULL'.freeze
 
-    attr_reader :columns
-
     attr_reader :typecast_map
 
-    def initialize(handle, service)
+    attr_reader :handle
+
+    def initialize(handle, service, options = {})
       @handle = handle
       @service = service
 
@@ -52,7 +52,13 @@ module Impala
       @done = false
       @open = true
       @typecast_map = TYPECAST_MAP.dup
-      @columns = metadata.schema.fieldSchemas.map(&:name)
+      @options = options.dup
+      @progress_reporter = ProgressReporter.new(self, @options)
+      @poll_every = options.fetch(:poll_every, 0.1)
+    end
+
+    def columns
+      @columns ||= metadata.schema.fieldSchemas.map(&:name)
     end
 
     def inspect
@@ -100,6 +106,22 @@ module Impala
       @open
     end
 
+    # Returns true if the query is done running, and results can be fetched.
+    def query_done?
+      [
+        Protocol::Beeswax::QueryState::EXCEPTION,
+        Protocol::Beeswax::QueryState::FINISHED
+      ].include?(@service.get_state(@handle))
+    end
+
+    # Blocks until the query done running.
+    def wait!
+      until query_done?
+        periodic_callback
+        sleep @poll_every
+      end
+    end
+
     # Returns true if there are any more rows to fetch.
     def has_more?
       !@done || !@row_buffer.empty?
@@ -109,8 +131,25 @@ module Impala
       @service.GetRuntimeProfile(@handle)
     end
 
+    def exec_summary
+      @service.GetExecSummary(@handle)
+    end
+
+    # Returns the progress for the query.
+    def progress
+      summary = exec_summary
+      summary.progress.num_completed_scan_ranges.to_f / summary.progress.total_scan_ranges.to_f
+    end
+
     private
 
+    attr :progress_reporter
+
+    def periodic_callback
+      return unless progress_reporter.show?
+      progress_reporter.report
+    end
+
     def metadata
       @metadata ||= @service.get_results_metadata(@handle)
     end
@@ -119,8 +158,13 @@ module Impala
       fetch_batch until @done || @row_buffer.count >= BUFFER_SIZE
     end
 
+    def exceptional?
+      @service.get_state(@handle) == Protocol::Beeswax::QueryState::EXCEPTION
+    end
+
     def fetch_batch
       raise CursorError.new("Cursor has expired or been closed") unless @open
+      raise ConnectionError.new("The query was aborted") if exceptional?
 
       begin
         res = @service.fetch(@handle, false, BUFFER_SIZE)

data/lib/impala/progress_reporter.rb

@@ -0,0 +1,40 @@
+module Impala
+  class ProgressReporter
+    def initialize(cursor, options = {})
+      @cursor = cursor
+      @progress_every = options.fetch(:progress_every, 60).to_f
+      @show_progress = options.fetch(:show_progress, false)
+      @start_time = @last_progress = Time.now
+    end
+
+    def show?
+      @show_progress
+    end
+
+    def report
+      return unless enough_time_elapsed?
+      @last_progress = Time.now
+      message = sprintf("Progress %.02f%% after %.02f seconds",
+                        cursor.progress * 100,
+                        total_time_elapsed)
+      progress_stream.puts message
+    end
+
+  private
+
+    attr :cursor, :show_progress, :last_progress, :progress_every, :start_time
+
+    def progress_stream
+      return show_progress if show_progress.is_a?(IO)
+      STDERR
+    end
+
+    def enough_time_elapsed?
+      (Time.now - last_progress) > progress_every
+    end
+
+    def total_time_elapsed
+      (Time.now - start_time)
+    end
+  end
+end

data/lib/impala/protocol/beeswax_constants.rb

@@ -1,5 +1,5 @@
 #
-# Autogenerated by Thrift Compiler (0.9.1)
+# Autogenerated by Thrift Compiler (0.9.3)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #

data/lib/impala/protocol/beeswax_service.rb

@@ -1,5 +1,5 @@
 #
-# Autogenerated by Thrift Compiler (0.9.1)
+# Autogenerated by Thrift Compiler (0.9.3)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #
@@ -30,25 +30,6 @@ module Impala
             raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'query failed: unknown result')
           end
 
-          def recv_result()
-            result = receive_message(Query_result)
-            raise result.error unless result.error.nil?
-            return result if result.ready
-          end
-
-          def executeAndWait2(query, clientCtx)
-            send_query(query)
-            sleep_time = 0.015625
-            max_sleep_time = 1
-            until result = recv_result
-              sleep(sleep_time)
-              sleep_time *= 2 unless sleep_time >= max_sleep_time
-            end
-            result.success
-            send_executeAndWait(query, clientCtx)
-            return recv_executeAndWait()
-          end
-
           def executeAndWait(query, clientCtx)
             send_executeAndWait(query, clientCtx)
             return recv_executeAndWait()

data/lib/impala/protocol/beeswax_types.rb

@@ -1,5 +1,5 @@
 #
-# Autogenerated by Thrift Compiler (0.9.1)
+# Autogenerated by Thrift Compiler (0.9.3)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #

data/lib/impala/protocol/exec_stats_constants.rb

@@ -0,0 +1,13 @@
+#
+# Autogenerated by Thrift Compiler (0.9.3)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+require 'exec_stats_types'
+
+module Impala
+  module Protocol
+  end
+end

data/lib/impala/protocol/exec_stats_types.rb

@@ -0,0 +1,133 @@
+#
+# Autogenerated by Thrift Compiler (0.9.3)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+require 'status_types'
+require 'types_types'
+
+
+module Impala
+  module Protocol
+    module TExecState
+      REGISTERED = 0
+      PLANNING = 1
+      QUEUED = 2
+      RUNNING = 3
+      FINISHED = 4
+      CANCELLED = 5
+      FAILED = 6
+      VALUE_MAP = {0 => "REGISTERED", 1 => "PLANNING", 2 => "QUEUED", 3 => "RUNNING", 4 => "FINISHED", 5 => "CANCELLED", 6 => "FAILED"}
+      VALID_VALUES = Set.new([REGISTERED, PLANNING, QUEUED, RUNNING, FINISHED, CANCELLED, FAILED]).freeze
+    end
+
+    class TExecStats
+      include ::Thrift::Struct, ::Thrift::Struct_Union
+      LATENCY_NS = 1
+      CPU_TIME_NS = 2
+      CARDINALITY = 3
+      MEMORY_USED = 4
+
+      FIELDS = {
+        LATENCY_NS => {:type => ::Thrift::Types::I64, :name => 'latency_ns', :optional => true},
+        CPU_TIME_NS => {:type => ::Thrift::Types::I64, :name => 'cpu_time_ns', :optional => true},
+        CARDINALITY => {:type => ::Thrift::Types::I64, :name => 'cardinality', :optional => true},
+        MEMORY_USED => {:type => ::Thrift::Types::I64, :name => 'memory_used', :optional => true}
+      }
+
+      def struct_fields; FIELDS; end
+
+      def validate
+      end
+
+      ::Thrift::Struct.generate_accessors self
+    end
+
+    class TPlanNodeExecSummary
+      include ::Thrift::Struct, ::Thrift::Struct_Union
+      NODE_ID = 1
+      FRAGMENT_ID = 2
+      LABEL = 3
+      LABEL_DETAIL = 4
+      NUM_CHILDREN = 5
+      ESTIMATED_STATS = 6
+      EXEC_STATS = 7
+      IS_ACTIVE = 8
+      IS_BROADCAST = 9
+
+      FIELDS = {
+        NODE_ID => {:type => ::Thrift::Types::I32, :name => 'node_id'},
+        FRAGMENT_ID => {:type => ::Thrift::Types::I32, :name => 'fragment_id'},
+        LABEL => {:type => ::Thrift::Types::STRING, :name => 'label'},
+        LABEL_DETAIL => {:type => ::Thrift::Types::STRING, :name => 'label_detail', :optional => true},
+        NUM_CHILDREN => {:type => ::Thrift::Types::I32, :name => 'num_children'},
+        ESTIMATED_STATS => {:type => ::Thrift::Types::STRUCT, :name => 'estimated_stats', :class => ::Impala::Protocol::TExecStats, :optional => true},
+        EXEC_STATS => {:type => ::Thrift::Types::LIST, :name => 'exec_stats', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Impala::Protocol::TExecStats}, :optional => true},
+        IS_ACTIVE => {:type => ::Thrift::Types::LIST, :name => 'is_active', :element => {:type => ::Thrift::Types::BOOL}, :optional => true},
+        IS_BROADCAST => {:type => ::Thrift::Types::BOOL, :name => 'is_broadcast', :optional => true}
+      }
+
+      def struct_fields; FIELDS; end
+
+      def validate
+        raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field node_id is unset!') unless @node_id
+        raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field fragment_id is unset!') unless @fragment_id
+        raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field label is unset!') unless @label
+        raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field num_children is unset!') unless @num_children
+      end
+
+      ::Thrift::Struct.generate_accessors self
+    end
+
+    class TExecProgress
+      include ::Thrift::Struct, ::Thrift::Struct_Union
+      TOTAL_SCAN_RANGES = 1
+      NUM_COMPLETED_SCAN_RANGES = 2
+
+      FIELDS = {
+        TOTAL_SCAN_RANGES => {:type => ::Thrift::Types::I64, :name => 'total_scan_ranges', :optional => true},
+        NUM_COMPLETED_SCAN_RANGES => {:type => ::Thrift::Types::I64, :name => 'num_completed_scan_ranges', :optional => true}
+      }
+
+      def struct_fields; FIELDS; end
+
+      def validate
+      end
+
+      ::Thrift::Struct.generate_accessors self
+    end
+
+    class TExecSummary
+      include ::Thrift::Struct, ::Thrift::Struct_Union
+      STATE = 1
+      STATUS = 2
+      NODES = 3
+      EXCH_TO_SENDER_MAP = 4
+      ERROR_LOGS = 5
+      PROGRESS = 6
+
+      FIELDS = {
+        STATE => {:type => ::Thrift::Types::I32, :name => 'state', :enum_class => ::Impala::Protocol::TExecState},
+        STATUS => {:type => ::Thrift::Types::STRUCT, :name => 'status', :class => ::Impala::Protocol::TStatus, :optional => true},
+        NODES => {:type => ::Thrift::Types::LIST, :name => 'nodes', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Impala::Protocol::TPlanNodeExecSummary}, :optional => true},
+        EXCH_TO_SENDER_MAP => {:type => ::Thrift::Types::MAP, :name => 'exch_to_sender_map', :key => {:type => ::Thrift::Types::I32}, :value => {:type => ::Thrift::Types::I32}, :optional => true},
+        ERROR_LOGS => {:type => ::Thrift::Types::LIST, :name => 'error_logs', :element => {:type => ::Thrift::Types::STRING}, :optional => true},
+        PROGRESS => {:type => ::Thrift::Types::STRUCT, :name => 'progress', :class => ::Impala::Protocol::TExecProgress, :optional => true}
+      }
+
+      def struct_fields; FIELDS; end
+
+      def validate
+        raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field state is unset!') unless @state
+        unless @state.nil? || ::Impala::Protocol::TExecState::VALID_VALUES.include?(@state)
+          raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field state!')
+        end
+      end
+
+      ::Thrift::Struct.generate_accessors self
+    end
+
+  end
+end