hbase-ruby 1.1.0 → 1.1.1

data/History.txt

@@ -1,3 +1,6 @@
+v1.1.1 Nov 7, 2009
+* Improved the scanner functionality. Decided it was worth a version bump and release.
+
 v1.1.0 Nov 7, 2009
 * Bumped to version 1.1.0 to avoid confusion with an existing hbase-ruby on gemcutter that was at version 1.0
 * Reimplemented entirely with HBase Stargate (old REST API was obsoleted by this)

data/README.textile

@@ -57,7 +57,7 @@ row = client.show_row('users', 'sishen')                 # show the data of row
 row2 = client.create_row('users', 'sishen', Time.now.to_i, {:name => 'habbit:football', :value => 'i like football'}) # create the row 'sishen' with the data in the table 'users'
 client.delete_row('users', 'sishen', nil, 'habbit:football')  # delete the row 'sishen' of table 'users' with the optional column 'habbit:football'
 
-# Scanner Operation
+# Scanner Operation (see spec/hbase/operation/scanner_operation_spec.rb for more examples)
 scanner = client.open_scanner('users', {:start_row => "row2", :batch => 5, :columns => ["habbit:"]}) # See more options from HBase::Model::Scanner.AVAILABLE_OPTS
 rows = client.get_rows(scanner)
 client.close_scanner(scanner)

data/Rakefile

@@ -5,7 +5,7 @@ begin
 
   Jeweler::Tasks.new do |gemspec|
     gemspec.name = "hbase-ruby"
-    gemspec.authors = ['Ye Dingding', 'Greg Lu']
+    gemspec.authors = ['Ye Dingding', 'Openplaces']
     gemspec.email = 'greg.lu@gmail.com'
     gemspec.homepage = "http://github.com/greglu/hbase-ruby"
     gemspec.summary = "A pure ruby client for HBase using the Stargate interface."

data/VERSION

@@ -1 +1 @@
-1.1.0
+1.1.1

data/lib/hbase/client.rb

@@ -28,6 +28,11 @@ module HBase
       safe_request { @connection.get(@url.path + path, {"Accept" => "application/json"}) }
     end
 
+    # Needed for scanner functionality
+    def get_response(path)
+      safe_response { @connection.get(@url.path + path, {"Accept" => "application/json"}) }
+    end
+
     def post(path, data = nil)
       safe_request { @connection.post(@url.path + path, data, {'Content-Type' => 'text/xml'}) }
     end

data/lib/hbase/exception.rb

@@ -17,3 +17,5 @@ class HBase::TableFailDisableError < HBase::Exception; end
 class HBase::TableFailEnableError < HBase::Exception; end
 
 class HBase::RowNotFoundError < HBase::Exception; end
+
+class HBase::ScannerError < HBase::Exception; end

data/lib/hbase/model/scanner.rb

@@ -3,11 +3,14 @@ module HBase
     class Scanner < Record
       AVAILABLE_OPTS = {  :start_row => "startRow", :end_row => "endRow",
                           :start_time => "startTime", :end_time => "endTime",
-                          :batch => "batch", :limit => "batch" }
+                          :batch => "batch" }
 
       attr_accessor :table_name
-      attr_accessor :scanner_id
       attr_accessor :scanner_url
+      attr_accessor :batch_size
+
+      # Deprecation: scanner_url is used instead of just the ID
+      attr_accessor :scanner_id
     end
   end
 end

data/lib/hbase/operation/scanner_operation.rb

@@ -3,7 +3,7 @@ module HBase
     module ScannerOperation
       # Trying to maintain some API stability for now
       def open_scanner(table_name, columns, start_row, stop_row = nil, timestamp = nil)
-        warn "[DEPRECATION] This method is deprecated. Use #open_scanner(table_name, options) instead."
+        warn "[DEPRECATION] This method is deprecated. Use #open_scanner(table_name, options = {}) instead."
 
         open_scanner(table_name, {:columns => columns, :start_row => start_row, :stop_row => stop_row, :timestamp => timestamp})
       end
@@ -11,16 +11,15 @@ module HBase
       def open_scanner(table_name, options = {})
         raise ArgumentError, "options should be given as a Hash" unless options.instance_of? Hash
         columns = options.delete(:columns)
+        batch = options.delete(:batch) || "10"
 
         begin
           request = Request::ScannerRequest.new(table_name)
 
-          xml_data = "<?xml version='1.0' encoding='UTF-8' standalone='yes'?><Scanner "
+          xml_data = "<?xml version='1.0' encoding='UTF-8' standalone='yes'?><Scanner batch='#{batch}' "
           options.each do |key,value|
             if Model::Scanner::AVAILABLE_OPTS.include? key
-              xml_data << "#{Model::Scanner::AVAILABLE_OPTS[key]}='"
-              xml_data << ( (key == :batch) ? value.to_s : [value.to_s].flatten.pack('m') )
-              xml_data << "' "
+              xml_data << "#{Model::Scanner::AVAILABLE_OPTS[key]}='#{[value.to_s].flatten.pack('m')}' "
             else
               warn "[open_scanner] Received invalid option key :#{key}"
             end
@@ -37,25 +36,34 @@ module HBase
 
           scanner = Response::ScannerResponse.new(post_response(request.open, xml_data), :open_scanner).parse
           scanner.table_name = table_name
+          scanner.batch_size = batch
           scanner
         rescue Net::ProtocolError => e
-          if e.to_s.include?("TableNotFoundException")
-            raise TableNotFoundError, "Table #{table_name} Not Found!"
-          else
-            raise StandardError, e.to_s
-          end
+          raise StandardError, e.to_s
         end
       end
 
       def get_rows(scanner, limit = nil)
-        warn "[DEPRECATION] Use of 'limit' here is deprecated. Instead, define the batch size when creating the scanner." if limit
         begin
           request = Request::ScannerRequest.new(scanner.table_name)
-          rows = Response::ScannerResponse.new(get(request.get_rows(scanner)), :get_rows).parse
-          rows.each do |row|
-            row.table_name = scanner.table_name
+          request_url = request.get_rows(scanner) # The url to the scanner is the same for each batch
+
+          rows = []
+          begin
+            # Loop until we've reached the limit, or the scanner was exhausted (HTTP 204 returned)
+            until (limit && rows.size >= limit) || (response = get_response(request_url)).code == "204"
+              rows.concat Response::ScannerResponse.new(response.body, :get_rows).parse
+
+              rows.each do |row|
+                row.table_name = scanner.table_name
+              end
+            end
+          rescue Exception => e
+            raise HBase::ScannerError, "Scanner failed while getting rows. #{e.message}"
           end
-          rows
+
+          # Prune the last few rows if the limit was passed.
+          (limit) ? rows.slice(0, limit) : rows
         rescue StandardError => e
           if e.to_s.include?("TableNotFoundException")
             raise TableNotFoundError, "Table #{table_name} Not Found!"

data/lib/hbase/response/scanner_response.rb

@@ -15,7 +15,11 @@ module HBase
           when Net::HTTPCreated
             HBase::Model::Scanner.new(:scanner_url => raw_data["Location"])
           else
-            raise StandardError, "Unable to open scanner. Received the following message: #{raw_data.message}"
+            if raw_data.message.include?("TableNotFoundException")
+              raise TableNotFoundError, "Table #{table_name} Not Found!"
+            else
+              raise StandardError, "Unable to open scanner. Received the following message: #{raw_data.message}"
+            end
           end
         when :get_rows
           # Dispatch it to RowResponse, since that method is made

data/spec/hbase/operation/scanner_operation_spec.rb

@@ -12,6 +12,12 @@ describe HBase::Operation::ScannerOperation do
     @client.create_row('test-hbase-ruby', 'row3', nil, {:name => 'col1:', :value => "row3-col1"})
   end
 
+  it "should throw TableNotFoundError if a scanner is requested for an non-existant table" do
+    lambda {
+      scanner = @client.open_scanner("test-dsg-ruby")
+    }.should raise_error
+  end
+
   it "should open a scanner and close it successfully" do
     scanner = @client.open_scanner("test-hbase-ruby")
     scanner.should.is_a? HBase::Model::Scanner
@@ -25,17 +31,33 @@ describe HBase::Operation::ScannerOperation do
     }.should_not raise_error
   end
 
-  it "should scan the whole table when given no options" do
+  it "should scan the whole table when given no options and no limit" do
     scanner = @client.open_scanner("test-hbase-ruby")
 
     rows = @client.get_rows(scanner)
-    rows.size.should == 1
-    rows.first.name.should == "row1"
+    rows.size.should == 3
+    rows.each do |row|
+      row.should be_an_instance_of HBase::Model::Row
+      ["row1", "row2", "row3"].should include(row.name)
+    end
+
+    @client.close_scanner(scanner).should be_true
+  end
+
+  it "should scan the whole table but limit the results when given a limit" do
+    scanner = @client.open_scanner("test-hbase-ruby")
+
+    rows = @client.get_rows(scanner, 2)
+    rows.size.should == 2
+    rows.each do |row|
+      row.should be_an_instance_of HBase::Model::Row
+      ["row1", "row2"].should include(row.name)
+    end
 
     @client.close_scanner(scanner).should be_true
   end
 
-  it "should scan the whole table when given a batch size larger than the number of rows" do
+  it "should return all rows when given a batch size larger than the number of rows" do
     scanner = @client.open_scanner("test-hbase-ruby", {:batch => 5})
 
     rows = @client.get_rows(scanner)

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification 
 name: hbase-ruby
 version: !ruby/object:Gem::Version 
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors: 
 - Ye Dingding
-- Greg Lu
+- Openplaces
 autorequire: 
 bindir: bin
 cert_chain: []