rbhive 1.0.0.pre → 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 224509bb79d39f4f06ec081b96e96a4cbb3be341
+  data.tar.gz: f4df37b710b762c9f49238ba34e281773e7e7976
+SHA512:
+  metadata.gz: 26a94f954cf6c76bbb6ed0ad345d56798beef6a933a756bba7778c1be5d66969606237052c0aed44978ecb073ad46836af0f830529220baec18f8d1f20e64fc8
+  data.tar.gz: ba9b396a9fcf388712cf5deefe2f9a16120fffc117b3e53631702f4a1d7df872d8bd209b8e0b1cedd485b6af364854b0a25577cd06ee8718fbee7363110eae6a

data/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 Versioning prior to 0.5.3 was not tracked, so this changelog only lists changes introduced after 0.5.3.
 
+## 1.0.0
+
+* Asynchronous execution with Hiveserver2
+* Misc bugfixes
+
+
 ## 0.6.0
 
 0.6.0 introduces one backwards-incompatible change:

data/README.md

@@ -1,5 +1,7 @@
 # RBHive - A Ruby Thrift client for Apache Hive
 
+[![Code Climate](https://codeclimate.com/github/forward3d/rbhive/badges/gpa.svg)](https://codeclimate.com/github/forward3d/rbhive)
+
 RBHive is a simple Ruby gem to communicate with the [Apache Hive](http://hive.apache.org)
 Thrift servers.
 
@@ -97,17 +99,23 @@ Connecting with the defaults:
       connection.fetch('SHOW TABLES')
     end
 
+Connecting with a Logger:
+
+    RBHive.tcli_connect('hive.server.address', 10_000, { logger: Logger.new(STDOUT) }) do |connection|
+      connection.fetch('SHOW TABLES')
+    end
+
 Connecting with a specific Hive version (0.12 in this case):
 
-    RBHive.tcli_connect('hive.server.address', 10_000, {:hive_version => 12}) do |connection|
+    RBHive.tcli_connect('hive.server.address', 10_000, { hive_version: 12 }) do |connection|
       connection.fetch('SHOW TABLES')
     end
 
 Connecting with a specific Hive version (0.12) and using the `:http` transport:
 
-    RBHive.tcli_connect('hive.server.address', 10_000, {:hive_version => 12, :transport => :http}) do |connection|
+    RBHive.tcli_connect('hive.server.address', 10_000, { hive_version: 12, transport: :http }) do |connection|
       connection.fetch('SHOW TABLES')
-    end 
+    end
 
 We have not tested the SASL connection, as we don't run SASL; pull requests and testing are welcomed.
 
@@ -144,6 +152,7 @@ In addition, you can explicitly set the Thrift protocol version according to thi
 In versions of Hive later than 0.12, the Thrift server supports asynchronous execution.
 
 The high-level view of using this feature is as follows:
+
 1. Submit your query using `async_execute(query)`. This function returns a hash
    with the following keys: `:guid`, `:secret`, and `:session`. You don't need to
    care about the internals of this hash - all methods that interact with an async

data/lib/rbhive/result_set.rb

@@ -4,34 +4,34 @@ module RBHive
       @schema = schema
       super(rows.map {|r| @schema.coerce_row(r) })
     end
-    
+
     def column_names
       @schema.column_names
     end
-    
+
     def column_type_map
       @schema.column_type_map
     end
-    
+
     def to_csv(out_file=nil)
-      to_seperated_output(",", out_file)
+      to_separated_output(",", out_file)
     end
-    
+
     def to_tsv(out_file=nil)
-      to_seperated_output("\t", out_file)
+      to_separated_output("\t", out_file)
     end
-    
+
     def as_arrays
       @as_arrays ||= self.map{ |r| @schema.coerce_row_to_array(r) }
     end
-    
+
     private
-    
-    def to_seperated_output(sep, out_file)
+
+    def to_separated_output(sep, out_file)
       rows = self.map { |r| @schema.coerce_row_to_array(r).join(sep) }
       sv = rows.join("\n")
       return sv if out_file.nil?
       File.open(out_file, 'w+') { |f| f << sv }
     end
   end
-end
+end

data/lib/rbhive/schema_definition.rb

@@ -4,6 +4,8 @@ module RBHive
   class SchemaDefinition
     attr_reader :schema
   
+    NAN = Float::NAN rescue 0.0/0.0
+    INFINITY = Float::INFINITY rescue 1.0/0.0
     TYPES = { 
       :boolean  => :to_s,
       :string   => :to_s,
@@ -63,8 +65,8 @@ module RBHive
   
     def coerce_column(column_name, value)
       type = column_type_map[column_name]
-      return 1.0/0.0 if(type != :string && value == "Infinity")
-      return 0.0/0.0 if(type != :string && value == "NaN")
+      return INFINITY if (type != :string && value == "Infinity")
+      return NAN if (type != :string && value == "NaN")
       return coerce_complex_value(value) if type.to_s =~ /^array/
       conversion_method = TYPES[type]
       conversion_method ? value.send(conversion_method) : value

data/lib/rbhive/t_c_l_i_connection.rb

@@ -46,9 +46,10 @@ module RBHive
     :PROTOCOL_V6 => 5,
     :PROTOCOL_V7 => 6
   }
-  
-  def tcli_connect(server, port=10_000, options)
-    connection = RBHive::TCLIConnection.new(server, port, options)
+
+  def tcli_connect(server, port = 10_000, options)
+    logger = options.key?(:logger) ? options.delete(:logger) : StdOutLogger.new
+    connection = RBHive::TCLIConnection.new(server, port, options, logger)
     ret = nil
     begin
       connection.open
@@ -65,7 +66,7 @@ module RBHive
       end
     end
 
-    return ret
+    ret
   end
   module_function :tcli_connect
 
@@ -80,7 +81,7 @@ module RBHive
   class TCLIConnection
     attr_reader :client
 
-    def initialize(server, port=10_000, options={}, logger=StdOutLogger.new)
+    def initialize(server, port = 10_000, options = {}, logger = StdOutLogger.new)
       options ||= {} # backwards compatibility
       raise "'options' parameter must be a hash" unless options.is_a?(Hash)
       
@@ -191,14 +192,16 @@ module RBHive
     # Async execute
     def async_execute(query)
       @logger.info("Executing query asynchronously: #{query}")
-      op_handle = @client.ExecuteStatement(
+      exec_result = @client.ExecuteStatement(
         Hive2::Thrift::TExecuteStatementReq.new(
           sessionHandle: @session.sessionHandle,
           statement: query,
           runAsync: true
         )
-      ).operationHandle
-      
+      )
+      raise_error_if_failed!(exec_result)
+      op_handle = exec_result.operationHandle
+
       # Return handles to get hold of this query / session again
       {
         session: @session.sessionHandle, 
@@ -235,7 +238,7 @@ module RBHive
       response = @client.GetOperationStatus(
         Hive2::Thrift::TGetOperationStatusReq.new(operationHandle: prepare_operation_handle(handles))
       )
-      puts response.operationState
+
       case response.operationState
       when Hive2::Thrift::TOperationState::FINISHED_STATE
         return :finished
@@ -253,6 +256,8 @@ module RBHive
         return :unknown
       when Hive2::Thrift::TOperationState::PENDING_STATE
         return :pending
+      when nil
+        raise "No operation state found for handles - has the session been closed?"
       else
         return :state_not_in_protocol
       end
@@ -262,7 +267,7 @@ module RBHive
     def async_fetch(handles, max_rows = 100)
       # Can't get data from an unfinished query
       unless async_is_complete?(handles)
-        raise "Can't perform fetch on a query in state: #{async_state(handles[:guid], handles[:secret])}"
+        raise "Can't perform fetch on a query in state: #{async_state(handles)}"
       end
       
       # Fetch and
@@ -275,7 +280,7 @@ module RBHive
       raise "No block given for the batch fetch request!" unless block_given?
       # Can't get data from an unfinished query
       unless async_is_complete?(handles)
-        raise "Can't perform fetch on a query in state: #{async_state(handles[:guid], handles[:secret])}"
+        raise "Can't perform fetch on a query in state: #{async_state(handles)}"
       end
 
       # Now let's iterate over the results
@@ -428,7 +433,9 @@ module RBHive
     def raise_error_if_failed!(result)
       return if result.status.statusCode == 0
       error_message = result.status.errorMessage || 'Execution failed!'
-      raise error_message
+      raise RBHive::TCLIConnectionError.new(error_message)
     end
   end
+
+  class TCLIConnectionError < StandardError; end
 end

data/lib/rbhive/t_c_l_i_schema_definition.rb

@@ -4,10 +4,11 @@ module RBHive
   class TCLISchemaDefinition
     attr_reader :schema
 
+    NAN = Float::NAN rescue 0.0/0.0
+    INFINITY = Float::INFINITY rescue 1.0/0.0
     TYPES = {
       :boolean  => :to_s,
       :string   => :to_s,
-      :bigint   => :to_i,
       :float    => :to_f,
       :double   => :to_f,
       :int      => :to_i,
@@ -65,8 +66,8 @@ module RBHive
 
     def coerce_column(column_name, value)
       type = column_type_map[column_name]
-      return 1.0/0.0 if(type != :string && value == "Infinity")
-      return 0.0/0.0 if(type != :string && value == "NaN")
+      return INFINITY if (type != :string && value == "Infinity")
+      return NAN if (type != :string && value == "NaN")
       return coerce_complex_value(value) if type.to_s =~ /^array/
       conversion_method = TYPES[type]
       conversion_method ? value.send(conversion_method) : value

data/lib/rbhive/table_schema.rb

@@ -8,6 +8,7 @@ module RBHive
       @field_sep = options[:field_sep] || "\t"
       @line_sep = options[:line_sep] || "\n"
       @collection_sep = options[:collection_sep] || "|"
+      @stored_as = options[:stored_as] || :textfile
       @columns = []
       @partitions = []
       @serde_name = nil
@@ -31,10 +32,14 @@ module RBHive
     def create_table_statement()
       %[CREATE #{external}TABLE #{table_statement}
   ROW FORMAT #{row_format_statement}
-  STORED AS TEXTFILE
+  STORED AS #{stored_as}
   #{location}]
     end
 
+    def stored_as
+      @stored_as.to_s.upcase
+    end
+
     def row_format_statement
       if @serde_name
         serde_statement

data/lib/rbhive/version.rb

@@ -1,3 +1,3 @@
 module RBHive
-  VERSION = '1.0.0.pre'
+  VERSION = '1.0.0'
 end

data/rbhive.gemspec

@@ -4,24 +4,24 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'rbhive/version'
 
 Gem::Specification.new do |spec|
-  spec.name = "rbhive"
+  spec.name = 'rbhive'
   spec.version = RBHive::VERSION
-  spec.authors = ["Forward3D","KolobocK"]
-  spec.description = "Simple gem for executing Hive queries and collecting the results"
-  spec.summary = "Simple gem for executing Hive queries"
-  spec.email = ["andy@forward.co.uk","kolobock@gmail.com", "developers@forward3d.com"]
-  spec.homepage = %q{http://github.com/forward3d/rbhive}
-  spec.license = "MIT"
-  
+  spec.authors = %w(Forward3D KolobocK)
+  spec.description = 'Simple gem for executing Hive queries and collecting the results'
+  spec.summary = 'Simple gem for executing Hive queries'
+  spec.email = ['developers@forward3d.com', 'kolobock@gmail.com']
+  spec.homepage = 'http://github.com/forward3d/rbhive'
+  spec.license = 'MIT'
+
   spec.files = `git ls-files`.split($/)
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
-  
-  spec.add_dependency('thrift', '= 0.9.0')
+  spec.require_paths = ['lib']
+
+  spec.add_dependency('thrift', '~> 0.9')
   spec.add_dependency('json')
 
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "bundler", ">= 1.3"
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'bundler', '>= 1.3'
 
 end

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rbhive
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre
-  prerelease: 6
+  version: 1.0.0
 platform: ruby
 authors:
 - Forward3D
@@ -10,82 +9,73 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-31 00:00:00.000000000 Z
+date: 2018-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thrift
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: '0.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.3'
 description: Simple gem for executing Hive queries and collecting the results
 email:
-- andy@forward.co.uk
-- kolobock@gmail.com
 - developers@forward3d.com
+- kolobock@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - CHANGELOG.md
 - Gemfile
 - LICENSE
@@ -122,29 +112,25 @@ files:
 homepage: http://github.com/forward3d/rbhive
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 2597338757284379755
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>'
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.6.11
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Simple gem for executing Hive queries
 test_files: []