sdbtools 0.1.0 → 0.2.0

data/Rakefile

@@ -19,6 +19,7 @@ END
     gem.add_dependency 'highline',    '~> 1.5'
     gem.add_dependency 'progressbar', '~> 0.0.3'
     gem.add_dependency 'libxml-ruby', '~> 1.1'
+    gem.add_dependency 'arrayfields', '~> 4.7'
     gem.add_development_dependency "rspec", ">= 1.2.9"
   end
   Jeweler::GemcutterTasks.new

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/lib/sdbtools.rb

@@ -1,5 +1,6 @@
 require 'fattr'
 require 'right_aws'
+require File.expand_path('selection', File.dirname(__FILE__))
 
 module SDBTools
 
@@ -8,16 +9,21 @@ module SDBTools
   class Operation
     include Enumerable
 
+    attr_reader :method
+    attr_reader :args
+    attr_reader :starting_token
+
     def initialize(sdb, method, *args)
       @options = args.last.is_a?(Hash) ? args.pop : {}
       @sdb     = sdb
       @method  = method
       @args    = args
+      @starting_token = @options[:starting_token]
     end
 
     # Yields once for each result set, until there is no next token.
     def each
-      next_token = @options[:starting_token]
+      next_token = starting_token
       begin
         args = @args.dup
         args << next_token

data/lib/selection.rb

@@ -0,0 +1,189 @@
+require 'arrayfields'
+require 'logger'
+
+module SDBTools
+  class Selection
+    include Enumerable
+
+    MAX_RESULT_LIMIT     = 250
+    DEFAULT_RESULT_LIMIT = 100
+
+    attr_accessor :domain
+    attr_accessor :attributes
+    attr_accessor :conditions
+    attr_reader   :limit
+    attr_accessor :offset
+    attr_accessor :order_by
+    attr_accessor :order
+    attr_accessor :sdb
+    attr_accessor :logger
+
+    # For testing
+    attr_writer :starting_token
+
+    def initialize(sdb, domain, options={})
+      @sdb        = sdb
+      @domain     = domain.to_s
+      @attributes = options.fetch(:attributes) { :all }
+      @conditions = Array(options[:conditions])
+      @order      = options.fetch(:order) { :ascending }
+      @order_by   = options.fetch(:order_by) { :none }
+      @order_by   = @order_by.to_s unless @order_by == :none
+      self.limit  = options.fetch(:limit) { DEFAULT_RESULT_LIMIT }
+      @offset     = options.fetch(:offset) { 0 }.to_i
+      @logger     = options.fetch(:logger){::Logger.new($stderr)}
+    end
+
+    def to_s
+      "SELECT #{output_list} FROM #{quote_name(domain)}#{match_expression}#{sort_instructions}#{limit_clause}"
+    end
+
+    def count_expression
+      "SELECT count(*) FROM #{quote_name(domain)}#{match_expression}#{sort_instructions}#{limit_clause_for_count}"
+    end
+
+    def offset_count_expression
+      "SELECT count(*) FROM #{quote_name(domain)}#{match_expression}#{sort_instructions} LIMIT #{offset}"
+    end
+
+    def count
+      @count ||= count_operation.inject(0){|count, results| 
+        count += results[:items].first["Domain"]["Count"].first.to_i
+      }
+    end
+
+    alias_method :size, :count
+    alias_method :length, :count
+
+    def each
+      return if limit == 0
+      num_items = 0
+      select_operation.each do |results|
+        results[:items].each do |item|
+          yield(item.keys.first, item.values.first)
+          num_items += 1
+          return if limit != :none && num_items >= limit
+        end
+      end
+    end
+
+    def results
+      @results ||= inject(Arrayfields.new){|results, (name, value)|
+        results[name] = value
+        results
+      }
+    end
+
+    def count_operation
+      Operation.new(sdb, :select, count_expression, :starting_token => starting_token)
+    end
+
+    def offset_count_operation
+      Operation.new(sdb, :select, offset_count_expression)
+    end
+
+    def select_operation
+      Operation.new(sdb, :select, to_s, :starting_token => starting_token)
+    end
+
+    def starting_token
+      @starting_token ||=
+        case offset
+        when 0 then nil
+        else
+          op    = offset_count_operation
+          count = 0
+          op.each do |results|
+          count += results[:items].first["Domain"]["Count"].first.to_i
+          if count == offset || results[:next_token].nil?
+            return results[:next_token]
+          end
+        end
+          raise "Failed to find offset #{offset}"
+        end
+    end
+
+    def limit=(new_limit)
+      # We can't yet support large limits. In order to do so, it will be necessary
+      # to implement limit chunking, where the limit is split across multiple
+      # requests of 250 items and a final request of limit % 250 items.
+      case new_limit
+      when :none, (0..MAX_RESULT_LIMIT) then 
+        @limit = new_limit
+      else
+        raise RangeError, "Limit must be 0..250 or :none"
+      end
+    end
+
+    private
+
+    def quote_name(name)
+      if name.to_s =~ /^[A-Z$_][A-Z0-9$_]*$/i
+        name.to_s
+      else
+        "`" + name.to_s.gsub("`", "``") + "`"
+      end
+    end
+
+    def quote_value(value)
+      '"' + value.to_s.gsub(/"/, '""') + '"'
+    end
+
+    def output_list
+      case attributes
+      when Array then attributes.map{|a| quote_name(a)}.join(", ")
+      when :all then "*"
+      else raise ScriptError, "Bad attributes: #{attributes.inspect}"
+      end
+    end
+
+    def match_expression
+      case conditions.size
+      when 0 then ""
+      else " WHERE #{prepared_conditions.join(' ')}"
+      end
+    end
+
+    def prepared_conditions
+      conditions.map { |condition|
+        case condition
+        when String then condition
+        when Array then
+          values   = condition.dup
+          template = values.shift.to_s
+          template.gsub(/\?/) {|match|
+            quote_value(values.shift.to_s)
+          }
+        else
+          raise ScriptError, "Bad condition: #{condition.inspect}"
+        end
+      }
+    end
+
+    def limit_clause
+      case limit
+      when :none then " LIMIT #{MAX_RESULT_LIMIT}"
+      when DEFAULT_RESULT_LIMIT then ""
+      else 
+        batch_limit = limit < MAX_RESULT_LIMIT ? limit : MAX_RESULT_LIMIT
+        " LIMIT #{batch_limit}"
+      end
+    end
+
+    def limit_clause_for_count
+      case limit
+      when :none then ""
+      else limit_clause
+      end
+    end
+
+    def sort_instructions
+      case order_by
+      when :none then ""
+      else 
+        direction = (order == :ascending ? "ASC" : "DESC")
+        " ORDER BY #{quote_name(order_by)} #{direction}"
+      end
+    end
+  end
+end

data/spec/selection_spec.rb

@@ -0,0 +1,295 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+module SDBTools
+  describe Selection do
+    def count_results(count, token=nil)
+      {
+        :items => [
+          "Domain" => {
+            "Count" => [count.to_s]
+          }
+        ],
+        :next_token => token
+      }      
+    end
+
+    def select_results(names, token=nil)
+      {
+        :items => names.map{|n| {n => {"#{n}_attr" => ["#{n}_val"]}}},
+        :next_token => token
+      }      
+    end
+
+    before :each do
+      @sdb = stub("SDB")
+    end
+
+    context "with domain THE_DOMAIN" do
+      before :each do
+        @it = Selection.new(@sdb, "THE_DOMAIN")
+      end
+
+      specify { @it.to_s.should == "SELECT * FROM THE_DOMAIN" }
+
+      it "should be able to generate a count expression" do
+        @it.count_expression.should be == "SELECT count(*) FROM THE_DOMAIN"
+      end
+
+      it "should be able to generate a count operation" do
+        op = @it.count_operation
+        op.method.should be == :select
+        op.args.should be == ["SELECT count(*) FROM THE_DOMAIN"]
+        op.starting_token.should be_nil
+      end
+
+      it "should generate a nil start token" do
+        @it.starting_token.should be_nil
+      end
+    end
+
+    context "with domain 1DO`MAIN" do
+      before :each do
+        @it = Selection.new(@sdb, "1DO`MAIN")
+      end
+
+      specify { @it.to_s.should == "SELECT * FROM `1DO``MAIN`" }
+    end
+
+    context "with explicit attributes" do
+      before :each do
+        @it = Selection.new(@sdb, "DOMAIN", :attributes => [:foo, :bar])
+      end
+
+      it "should select the attributes" do
+        @it.to_s.should == "SELECT foo, bar FROM DOMAIN"
+      end
+    end
+
+    context "with funky attribute names" do
+      before :each do
+        @it = Selection.new(@sdb, "DOMAIN", 
+          :attributes => ["attr foo", "1`bar", "baz"])
+      end
+
+      it "should quote the funky attribute names" do
+        @it.to_s.should == %Q{SELECT `attr foo`, `1``bar`, baz FROM DOMAIN}
+      end
+    end
+
+    context "with conditions" do
+      before :each do
+        @it = Selection.new(@sdb, "DOMAIN", 
+          :attributes => ["attr foo", "bar"],
+          :conditions => [
+            'bar == "buz"', 
+            "AND",
+            ['`attr foo` between ? and ?', 4, '1"2']])
+      end
+
+      it "should format quote and interpolate the conditions" do
+        @it.to_s.should be == 
+          %Q{SELECT `attr foo`, bar FROM DOMAIN WHERE bar == "buz" AND `attr foo` between "4" and "1""2"}
+      end
+
+    end                         # "
+
+    context "with a limit of :none" do
+      before :each do
+        @it = Selection.new(@sdb, "DOMAIN",
+          :attributes => [:foo, :bar], 
+          :conditions => ["foo == 'bar'"],
+          :limit      => :none)
+      end
+
+      it "should append a limit clause" do
+        @it.to_s.should be == "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' LIMIT 250"
+      end
+
+      it "should be able to generate a count expression" do
+        @it.count_expression.should be == 
+          "SELECT count(*) FROM DOMAIN WHERE foo == 'bar'"
+      end
+    end
+
+    context "with a limit" do
+      before :each do
+        @it = Selection.new(@sdb, "DOMAIN",
+          :attributes => [:foo, :bar], 
+          :conditions => ["foo == 'bar'"],
+          :limit      => 10)
+      end
+
+      it "should append a limit clause" do
+        @it.to_s.should be == "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' LIMIT 10"
+      end
+
+      it "should be able to generate a count expression" do
+        @it.count_expression.should be == 
+          "SELECT count(*) FROM DOMAIN WHERE foo == 'bar' LIMIT 10"
+      end
+
+      it "should limit results" do
+        results = select_results(Array.new(100, "foo"))
+        @sdb.stub!(:select).and_return(results)
+        @it.to_a.size.should == 10
+      end
+    end
+
+    context "ordered by an attribute" do
+      before :each do
+        @it = Selection.new(@sdb, "DOMAIN",
+          :attributes      => [:foo, :bar], 
+          :conditions      => ["foo == 'bar'"],
+          :order_by        => "foo")
+      end
+
+      it "should append an order clause" do
+        @it.to_s.should be == 
+          "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' ORDER BY foo ASC"
+      end
+
+      it "should be able to generate a count expression" do
+        @it.count_expression.should be == 
+          "SELECT count(*) FROM DOMAIN WHERE foo == 'bar' ORDER BY foo ASC"
+      end
+    end
+
+    context "in descending order" do
+      before :each do
+        @it = Selection.new(@sdb, "DOMAIN",
+          :attributes      => [:foo, :bar], 
+          :conditions      => ["foo == 'bar'"],
+          :order_by        => "foo",
+          :order           => :descending)
+      end
+
+      it "should append an order clause" do
+        @it.to_s.should be == 
+          "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' ORDER BY foo DESC"
+      end
+
+      it "should be able to generate a count expression" do
+        @it.count_expression.should be == 
+          "SELECT count(*) FROM DOMAIN WHERE foo == 'bar' ORDER BY foo DESC"
+      end
+
+      it "should be able to generate an offset count expression" do
+        @it.offset_count_expression.should be == 
+          "SELECT count(*) FROM DOMAIN WHERE foo == 'bar' ORDER BY foo DESC LIMIT 0"
+      end
+
+      context "with a limit" do
+        before :each do
+          @it.limit = 250
+        end
+        
+        specify { @it.to_s.should be == 
+          "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' ORDER BY foo DESC LIMIT 250"
+        }
+      end
+    end
+
+    # We can't yet support large limits. In order to do so, it will be necessary
+    # to implement limit chunking, where the limit is split across multiple
+    # requests of 250 items and a final request of limit % 250 items.
+    it "should reject limits > 250" do
+      lambda do
+        Selection.new(@sdb, "DOMAIN", :limit => 251) 
+      end.should raise_error
+      lambda do
+        Selection.new(@sdb, "DOMAIN").limit = 251
+      end.should raise_error
+    end
+
+    context "with an offset" do
+      before :each do
+        @it = Selection.new(@sdb, "DOMAIN",
+          :attributes => [:foo, :bar], 
+          :conditions => ["foo == 'bar'"],
+          :limit      => 10,
+          :offset     => 200)
+
+        # Note: counts are relative to the last count, not absolute
+        @first_results = count_results(100, "TOKEN1")
+        @second_results = count_results(100, "TOKEN2")
+        @third_results = count_results(100, nil)
+        @sdb.stub!(:select).
+          and_return(@first_results, @second_results, @third_results)
+      end
+
+      it "should be able to generate an offset count expression" do
+        @it.offset_count_expression.should be == 
+          "SELECT count(*) FROM DOMAIN WHERE foo == 'bar' LIMIT 200"
+      end
+
+      it "should be able to generate an offset count operation" do
+        op = @it.offset_count_operation
+        op.method.should be == :select
+        op.args.should be == [
+          "SELECT count(*) FROM DOMAIN WHERE foo == 'bar' LIMIT 200"
+        ]
+        op.starting_token.should be_nil
+      end
+
+      it "should use an SDB count to determine starting token" do
+        @sdb = stub("SDB")
+        @sdb.should_receive(:select).
+          with(@it.offset_count_expression, nil).
+          ordered.
+          and_return(@first_results)
+        @sdb.should_receive(:select).
+          with(@it.offset_count_expression, "TOKEN1").
+          ordered.
+          and_return(@second_results)
+        @sdb.should_not_receive(:select).
+          with(@it.offset_count_expression, "TOKEN2")
+        @it.sdb = @sdb
+        @it.starting_token.should be == "TOKEN2"
+      end
+
+      it "should be able to generate a select operation" do
+        op = @it.select_operation
+        op.args.should be == [
+          "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' LIMIT 10"
+        ]
+        op.starting_token.should be == "TOKEN2"
+      end
+
+      it "should be able to count items in the selection" do
+        @sdb = stub("SDB")
+        @sdb.should_receive(:select).
+          with(@it.count_expression, "TOKEN2").
+          and_return(count_results(4, "TOKEN5"))
+        @sdb.should_receive(:select).
+          with(@it.count_expression, "TOKEN5").
+          and_return(count_results(5, nil))
+        @it.sdb = @sdb
+        @it.starting_token = "TOKEN2"
+        @it.size.should be == 9
+        @it.length.should be == 9
+        @it.count.should be == 9
+      end
+
+      it "should be able to get selection results" do
+        @sdb = stub("SDB")
+        @sdb.should_receive(:select).
+          with(@it.to_s, "TOKEN2").
+          and_return(select_results(["foo", "bar"], "TOKEN5"))
+        @sdb.should_receive(:select).
+          with(@it.to_s, "TOKEN5").
+          and_return(select_results(["baz", "buz"], nil))
+        @it.sdb = @sdb
+        @it.starting_token = "TOKEN2"
+        @it.results.values.should be == [
+          { "foo_attr" => ["foo_val"]},
+          { "bar_attr" => ["bar_val"]},
+          { "baz_attr" => ["baz_val"]},
+          { "buz_attr" => ["buz_val"]}
+        ]
+        @it.results.keys.should be == [
+          "foo", "bar", "baz", "buz"
+        ]
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: sdbtools
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Avdi Grimm
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-18 00:00:00 -05:00
+date: 2010-01-19 00:00:00 -05:00
 default_executable: sdbtool
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -72,6 +72,16 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: "1.1"
     version: 
+- !ruby/object:Gem::Dependency 
+  name: arrayfields
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: "4.7"
+    version: 
 - !ruby/object:Gem::Dependency 
   name: rspec
   type: :development
@@ -98,7 +108,9 @@ files:
 - VERSION
 - bin/sdbtool
 - lib/sdbtools.rb
+- lib/selection.rb
 - spec/operation_spec.rb
+- spec/selection_spec.rb
 - spec/spec_helper.rb
 has_rdoc: true
 homepage: http://github.com/devver/sdbtools
@@ -129,5 +141,6 @@ signing_key:
 specification_version: 3
 summary: A high-level OO interface to Amazon SimpleDB
 test_files: 
+- spec/selection_spec.rb
 - spec/operation_spec.rb
 - spec/spec_helper.rb