impala 0.2.0 → 0.3.0

data/README.md

@@ -1,6 +1,6 @@
 # impala-ruby
 
-This is an ruby client for [Cloudera's Impala][1]. You use it like this:
+This is a ruby client for [Cloudera Impala][1]. You use it like this:
 
 ```ruby
 require 'impala'

data/impala.gemspec

@@ -13,6 +13,8 @@ Gem::Specification.new do |gem|
   gem.homepage      = "https://github.com/colinmarc/impala-ruby"
 
   gem.add_dependency('thrift', '~> 0.9.1')
+  gem.add_dependency('rack')
+  gem.add_dependency('thin')
 
   gem.add_development_dependency('rake')
   gem.add_development_dependency('pry')

data/lib/impala.rb

@@ -14,7 +14,7 @@ require 'impala/cursor'
 require 'impala/connection'
 
 module Impala
-  KNOWN_COMMANDS = ['select', 'insert', 'show', 'describe', 'use', 'explain', 'create', 'drop']
+  KNOWN_COMMANDS = ['select', 'insert', 'show', 'describe', 'use', 'explain', 'create', 'drop', 'invalidate', 'with']
   DEFAULT_HOST = 'localhost'
   DEFAULT_PORT = 21000
   class InvalidQueryError < StandardError; end

data/lib/impala/connection.rb

@@ -2,7 +2,7 @@ module Impala
   # This object represents a connection to an Impala server. It can be used to
   # perform queries on the database.
   class Connection
-    SLEEP_INTERVAL = 0.1
+    LOG_CONTEXT_ID = "impala-ruby"
 
     # Don't instantiate Connections directly; instead, use {Impala.connect}.
     def initialize(host, port)
@@ -43,7 +43,7 @@ module Impala
       @connected
     end
 
-    # Refresh the metadata store
+    # Refresh the metadata store.
     def refresh
       raise ConnectionError.new("Connection closed") unless open?
       @service.ResetCatalog
@@ -53,21 +53,27 @@ module Impala
     # load the entire result set into memory, so if you're dealing with lots
     # of rows, {#execute} may work better.
     # @param [String] query the query you want to run
+    # @param [Hash] query_options the options to set user and configuration
+    #   except for :user, see TImpalaQueryOptions in ImpalaService.thrift
+    # @option query_options [String] :user the user runs the query
     # @return [Array<Hash>] an array of hashes, one for each row.
-    def query(raw_query)
-      execute(raw_query).fetch_all
+    def query(raw_query, query_options = {})
+      execute(raw_query, query_options).fetch_all
     end
 
     # Perform a query and return a cursor for iterating over the results.
     # @param [String] query the query you want to run
+    # @param [Hash] query_options the options to set user and configuration
+    #   except for :user, see TImpalaQueryOptions in ImpalaService.thrift
+    # @option query_options [String] :user the user runs the query
     # @return [Cursor] a cursor for the result rows
-    def execute(raw_query)
+    def execute(raw_query, query_options = {})
       raise ConnectionError.new("Connection closed") unless open?
 
       query = sanitize_query(raw_query)
-      handle = send_query(query)
+      handle = send_query(query, query_options)
 
-      wait_for_result(handle)
+      check_result(handle)
       Cursor.new(handle, @service)
     end
 
@@ -85,25 +91,23 @@ module Impala
       ([command] + words[1..-1]).join(' ')
     end
 
-    def send_query(sanitized_query)
+    def send_query(sanitized_query, query_options)
       query = Protocol::Beeswax::Query.new
       query.query = sanitized_query
 
-      @service.query(query)
+      query.hadoop_user = query_options.delete(:user) if query_options[:user]
+      query.configuration = query_options.map do |key, value|
+        "#{key.upcase}=#{value}"
+      end
+
+      @service.executeAndWait(query, LOG_CONTEXT_ID)
     end
 
-    def wait_for_result(handle)
-      #TODO select here, or something
-      while true
-        state = @service.get_state(handle)
-        if state == Protocol::Beeswax::QueryState::FINISHED
-          break
-        elsif state == Protocol::Beeswax::QueryState::EXCEPTION
-          close_handle(handle)
-          raise ConnectionError.new("The query was aborted")
-        end
-
-        sleep(SLEEP_INTERVAL)
+    def check_result(handle)
+      state = @service.get_state(handle)
+      if state == Protocol::Beeswax::QueryState::EXCEPTION
+        close_handle(handle)
+        raise ConnectionError.new("The query was aborted")
       end
     rescue
       close_handle(handle)

data/lib/impala/version.rb

@@ -1,3 +1,3 @@
 module Impala
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/test/test_impala.rb

@@ -46,7 +46,7 @@ describe Impala::Connection do
     end
   end
 
-  describe '#wait_for_result' do
+  describe '#check_result' do
     before do
       Impala::Connection.any_instance.stubs(:open)
       @connection = Impala::Connection.new('test', 1234)
@@ -58,7 +58,46 @@ describe Impala::Connection do
       handle = stub()
       @service.expects(:close).with(handle).once
       @service.expects(:get_state).raises(StandardError)
-      assert_raises(StandardError) { @connection.send(:wait_for_result, handle) }
+      assert_raises(StandardError) { @connection.send(:check_result, handle) }
     end
   end
-end
+
+  describe '#execute' do
+    before do
+      Impala::Connection.any_instance.stubs(:open)
+      Impala::Cursor.stubs(:new)
+      @connection = Impala::Connection.new('test', 1234)
+      @connection.stubs(:open? => true, :sanitize_query => 'sanitized_query', :check_result => nil)
+    end
+
+    it 'should call Protocol::ImpalaService::Client#executeAndWait with the sanitized query' do
+      query = Impala::Protocol::Beeswax::Query.new
+      query.query = 'sanitized_query'
+      query.configuration = []
+
+      @service = stub()
+      @service.expects(:executeAndWait).with(query, Impala::Connection::LOG_CONTEXT_ID).once
+      @connection.instance_variable_set('@service', @service)
+
+      @connection.execute('query')
+    end
+
+    it 'should call Protocol::ImpalaService::Client#executeAndWait with the hadoop_user and configuration if passed as parameter' do
+      query = Impala::Protocol::Beeswax::Query.new
+      query.query = 'sanitized_query'
+      query.hadoop_user = 'impala'
+      query.configuration = %w|NUM_SCANNER_THREADS=8 MEM_LIMIT=3221225472|
+
+      @service = stub()
+      @service.expects(:executeAndWait).with(query, Impala::Connection::LOG_CONTEXT_ID).once
+      @connection.instance_variable_set('@service', @service)
+
+      opt = {
+        :user => 'impala',
+        :num_scanner_threads => 8,
+        :mem_limit => 3221225472
+      }
+      @connection.execute('query', opt)
+    end
+  end
+end

data/test/test_impala_connected.rb

@@ -16,140 +16,171 @@ def connect
   Impala.connect(host, port)
 end
 
-describe 'basic connected tests' do
+describe 'connected tests' do
   before do
     skip unless IMPALA_SERVER
     @connection = connect
   end
 
-  it 'can connect' do
-    assert_instance_of(Impala::Connection, @connection)
-    assert(@connection.open?, "the connection should be open")
-  end
+  describe 'basic tests' do
+    it 'can connect' do
+      assert_instance_of(Impala::Connection, @connection)
+      assert(@connection.open?, "the connection should be open")
+    end
 
-  it 'can refresh the catalog' do
-    @connection.refresh
-  end
+    it 'can refresh the catalog' do
+      @connection.refresh
+    end
 
-  it 'can run a basic query' do
-    ret = @connection.query('SELECT "foo" AS foo')
-    assert_equal([{:foo=>'foo'}], ret, "the result should be a list of hashes")
-  end
+    it 'can refresh metadata' do
+      @connection.query('invalidate metadata')
+    end
 
-  it 'can handle boolean values' do
-    ret = @connection.query('SELECT TRUE AS foo')
-    assert_equal([{:foo=>true}], ret, "the result should be a bool")
-  end
+    it 'can run a basic query' do
+      ret = @connection.query('SELECT "foo" AS foo')
+      assert_equal([{:foo=>'foo'}], ret, "the result should be a list of hashes")
+    end
 
-  it 'can handle double values' do
-    ret = @connection.query("SELECT 1.23 AS foo")
-    assert_equal([{:foo=>1.23}], ret, "the result should be a float")
-  end
+    it 'can run a basic query with some query options as specified user' do
+      ret = @connection.query('SELECT "foo" AS foo',
+                              :user => 'someoneelse',
+                              :mem_limit => 1234567890,
+                              :max_scan_range_length => 1024 * 1024 * 1024)
+      assert_equal([{:foo=>'foo'}], ret, "the result should be a list of hashes")
+    end
 
-  it 'can handle float values' do
-    ret = @connection.query("SELECT CAST(1.23 AS float) as foo")
-    assert_instance_of(Float, ret.first[:foo], "the result should be a float")
-  end
+    it 'can handle boolean values' do
+      ret = @connection.query('SELECT TRUE AS foo')
+      assert_equal([{:foo=>true}], ret, "the result should be a bool")
+    end
 
-  it 'can handle timestamp values' do
-    ret = @connection.query("SELECT NOW() AS foo")
-    assert_instance_of(Time, ret.first[:foo])
-  end
+    it 'can handle double values' do
+      ret = @connection.query('SELECT 1.23 AS foo')
+      assert_equal([{:foo=>1.23}], ret, "the result should be a float")
+    end
 
-  it 'can successfully refresh the metadata store' do
-    ret = @connection.refresh
-  end
-end
+    it 'can handle float values' do
+      ret = @connection.query('SELECT CAST(1.23 AS float) as foo')
+      assert_instance_of(Float, ret.first[:foo], "the result should be a float")
+    end
 
-describe 'with a test database' do
-  before do
-    @database = '_impala_ruby_test'
-    @connection.query("CREATE DATABASE IF NOT EXISTS #{@database}")
-  end
+    it 'can handle timestamp values' do
+      ret = @connection.query('SELECT NOW() AS foo')
+      assert_instance_of(Time, ret.first[:foo], "the result should be a timestamp")
+    end
 
-  after do
-    @connection.query('DROP DATABASE IF EXISTS _impala_ruby_test')
-  end
+    it 'can handle null values' do
+      ret = @connection.query('SELECT NULL AS nothing')
+      assert_equal(nil, ret.first[:nothing], "the result should be nil")
+    end
 
-  it 'can use the database' do
-    @connection.query("USE #{@database}")
-    @connection.query("USE default")
+    it 'can successfully refresh the metadata store' do
+      ret = @connection.refresh
+    end
   end
 
-  describe 'and a test table' do
+  describe 'with a test database' do
     before do
-      @table = "#{@database}.foobar"
-      @connection.query("CREATE TABLE #{@table} (i INT)")
+      @database = '_impala_ruby_test'
+      @connection.query("CREATE DATABASE IF NOT EXISTS #{@database}")
     end
 
     after do
-      @connection.query("DROP TABLE #{@table}")
+      @connection.query("DROP DATABASE IF EXISTS #{@database}") if @connection
     end
 
-    it 'deals with empty tables correctly when using #query' do
-      res = @connection.query("SELECT * FROM #{@table}")
-      assert_equal([], res, "the result set should be empty")
+    it 'can use the database' do
+      @connection.query("USE #{@database}")
+      @connection.query("USE default")
     end
 
-    it 'deals with empty tables correctly when using a cursor' do
-      cursor = @connection.execute("SELECT * FROM #{@table}")
-      assert_equal(false, cursor.has_more?, "has_more? should be false")
-      assert_nil(cursor.fetch_row, "calls to fetch_row should be nil")
-    end
-
-    describe 'with data' do
+    describe 'and a test table' do
       before do
-        @connection.query("INSERT INTO #{@table} (i) SELECT 1")
-        @connection.query("INSERT INTO #{@table} (i) SELECT 1")
-        @connection.query("INSERT INTO #{@table} (i) SELECT 1")
+        @table = "#{@database}.foobar"
+        @connection.query("CREATE TABLE #{@table} (i INT)")
       end
 
-      it 'can insert into the table' do
-        @connection.query("INSERT INTO #{@table} (i) SELECT 2")
+      after do
+        @connection.query("DROP TABLE #{@table}") if @connection
       end
 
-      it 'can select from the table using #query' do
+      it 'deals with empty tables correctly when using #query' do
         res = @connection.query("SELECT * FROM #{@table}")
-        assert_equal([{:i => 1}, {:i => 1}, {:i => 1}], res)
+        assert_equal([], res, "the result set should be empty")
       end
 
-      it 'can create a cursor and fetch one row at a time' do
+      it 'deals with empty tables correctly when using a cursor' do
         cursor = @connection.execute("SELECT * FROM #{@table}")
-        assert_instance_of(Impala::Cursor, cursor, "the result should be a cursor")
+        assert_equal(false, cursor.has_more?, "has_more? should be false")
+        assert_nil(cursor.fetch_row, "calls to fetch_row should be nil")
+      end
 
-        3.times do
-          row = cursor.fetch_row
-          assert_equal({:i=>1}, row, "the row should be a hash with the correct result")
+      describe 'with data' do
+        before do
+          @connection.query("INSERT INTO #{@table} (i) SELECT 1")
+          @connection.query("INSERT INTO #{@table} (i) SELECT 1")
+          @connection.query("INSERT INTO #{@table} (i) SELECT 1")
         end
 
-        assert_equal(false, cursor.has_more?, "has_more? should be false")
-        assert_nil(cursor.fetch_row, "subsequent calls to fetch_row should be nil")
-      end
+        it 'can handle the keywoard "with"' do
+          res = @connection.query("with bar as (select * from #{@table}) select * from bar")
+          assert_equal([{:i => 1}, {:i => 1}, {:i => 1}], res)
+        end
 
-      it 'can use a cursor to deal with lots of data' do
-        10.times { @connection.query("INSERT INTO #{@table} SELECT * FROM #{@table}") }
-        @connection.query("INSERT INTO #{@table} (i) SELECT 1")
-        count = @connection.query("SELECT COUNT(*) as n from #{@table}")[0][:n]
-        assert(count > Impala::Cursor::BUFFER_SIZE) # otherwise the test is pointless
+        it 'can insert into the table' do
+          @connection.query("INSERT INTO #{@table} (i) SELECT 2")
+        end
 
-        cursor = @connection.execute("SELECT * FROM #{@table}")
-        assert_instance_of(Impala::Cursor, cursor, "the result should be a cursor")
+        it 'can select from the table using #query' do
+          res = @connection.query("SELECT * FROM #{@table}")
+          assert_equal([{:i => 1}, {:i => 1}, {:i => 1}], res)
+        end
+
+        it 'can create a cursor and fetch one row at a time' do
+          cursor = @connection.execute("SELECT * FROM #{@table}")
+          assert_instance_of(Impala::Cursor, cursor, "the result should be a cursor")
+
+          3.times do
+            row = cursor.fetch_row
+            assert_equal({:i=>1}, row, "the row should be a hash with the correct result")
+          end
 
-        # fetch one to fill the buffer
-        row = cursor.fetch_row
-        assert_equal({:i=>1}, row, "the row should be a hash with the correct result")
+          assert_equal(false, cursor.has_more?, "has_more? should be false")
+          assert_nil(cursor.fetch_row, "subsequent calls to fetch_row should be nil")
+        end
+
+        it 'can use a cursor to deal with lots of data' do
+          10.times { @connection.query("INSERT INTO #{@table} SELECT * FROM #{@table}") }
+          @connection.query("INSERT INTO #{@table} (i) SELECT 1")
+          count = @connection.query("SELECT COUNT(*) as n from #{@table}")[0][:n]
+          assert(count > Impala::Cursor::BUFFER_SIZE) # otherwise the test is pointless
 
-        buffer_size = cursor.instance_variable_get('@row_buffer').size
-        assert_equal(Impala::Cursor::BUFFER_SIZE - 1, buffer_size, "it should only buffer #{Impala::Cursor::BUFFER_SIZE} rows into memory")
+          cursor = @connection.execute("SELECT * FROM #{@table}")
+          assert_instance_of(Impala::Cursor, cursor, "the result should be a cursor")
 
-        (count - 1).times do
+          # fetch one to fill the buffer
           row = cursor.fetch_row
           assert_equal({:i=>1}, row, "the row should be a hash with the correct result")
+
+          buffer_size = cursor.instance_variable_get('@row_buffer').size
+          assert_equal(Impala::Cursor::BUFFER_SIZE - 1, buffer_size, "it should only buffer #{Impala::Cursor::BUFFER_SIZE} rows into memory")
+
+          (count - 1).times do
+            row = cursor.fetch_row
+            assert_equal({:i=>1}, row, "the row should be a hash with the correct result")
+          end
+
+          assert_equal(false, cursor.has_more?, "has_more? should be false")
+          assert_nil(cursor.fetch_row, "subsequent calls to fetch_row should be nil")
         end
 
-        assert_equal(false, cursor.has_more?, "has_more? should be false")
-        assert_nil(cursor.fetch_row, "subsequent calls to fetch_row should be nil")
+        it 'can handle interspersed NULL values' do
+          @connection.query("INSERT INTO #{@table} (i) SELECT NULL")
+          res = @connection.query("SELECT * FROM #{@table} ORDER BY i DESC LIMIT 4")
+          assert_equal([{:i => 1}, {:i => 1}, {:i => 1}, {:i => nil}], res)
+        end
       end
+
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: impala
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-17 00:00:00.000000000 Z
+date: 2013-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thrift
@@ -27,6 +27,38 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.9.1
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thin
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement