sdbtools 0.3.0 → 0.4.0

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.4.0

data/lib/sdbtools.rb

@@ -1,149 +1,15 @@
 require 'fattr'
 require 'right_aws'
-require File.expand_path('selection', File.dirname(__FILE__))
+$:.unshift(File.expand_path(File.dirname(__FILE__)))
+require 'sdbtools/operation'
+require 'sdbtools/database'
+require 'sdbtools/domain'
+require 'sdbtools/selection'
+require 'sdbtools/transaction'
+require 'sdbtools/measured_sdb_interface'
 
 module SDBTools
 
-  # An Operation represents a SimpleDB operation and handles the details of
-  # re-requesting "next tokens" until the operation is complete.
-  class Operation
-    include Enumerable
-
-    attr_reader :method
-    attr_reader :args
-    attr_reader :starting_token
-
-    def initialize(sdb, method, *args)
-      @options = args.last.is_a?(Hash) ? args.pop : {}
-      @sdb     = sdb
-      @method  = method
-      @args    = args
-      @starting_token = @options[:starting_token]
-    end
-
-    # Yields once for each result set, until there is no next token.
-    def each
-      next_token = starting_token
-      begin
-        args = @args.dup
-        args << next_token
-        results = @sdb.send(@method, *args)
-        yield(results)
-        next_token = results[:next_token]
-      end while next_token
-    end
-
-  end
-
-  class Database
-    def initialize(access_key=nil, secret_key=nil, options={})
-      @sdb = RightAws::SdbInterface.new(access_key, secret_key, options)
-      @logger = options.fetch(:logger){::Logger.new($stderr)}
-    end
-
-    def domains
-      domains_op = Operation.new(@sdb, :list_domains, nil)
-      domains_op.inject([]) {|domains, results|
-        domains.concat(results[:domains])
-        domains
-      }
-    end
-
-    def domain(domain_name)
-      Domain.new(@sdb, domain_name)
-    end
-
-    def domain_exists?(domain_name)
-      domains.include?(domain_name)
-    end
-
-    def create_domain(domain_name)
-      @sdb.create_domain(domain_name)
-      domain(domain_name)
-    end
-
-    def delete_domain(domain_name)
-      @sdb.delete_domain(domain_name)
-    end
-
-    def make_dump(domain, filename)
-      Dump.new(domain, filename, @logger)
-    end
-
-    def make_load(domain, filename)
-      Load.new(domain, filename, @logger)
-    end
-
-    private
-  end
-
-  class Domain
-    attr_reader :name
-
-    def initialize(sdb, name)
-      @sdb        = sdb
-      @name       = name
-      @item_names = nil
-      @count      = nil
-    end
-
-    def [](item_name)
-      @sdb.get_attributes(name, item_name)[:attributes]
-    end
-
-    def item_names
-      return @item_names if @item_names
-      query = Operation.new(@sdb, :query, @name, nil, nil)
-      @item_names = query.inject([]) {|names, results| 
-        names.concat(results[:items])
-        names
-      }
-    end
-
-    def count
-      return @count if @count
-      op = Operation.new(@sdb, :select, "select count(*) from #{name}")
-      @count = op.inject(0) {|count, results|
-        count += results[:items].first["Domain"]["Count"].first.to_i
-        count
-      }
-    end
-
-    def items(item_names)
-      names  = item_names.map{|n| "'#{n}'"}.join(', ')
-      query  = "select * from #{name} where itemName() in (#{names})"
-      select = Operation.new(@sdb, :select, query)
-      select.inject({}) {|items, results|
-        results[:items].each do |item|
-          item_name  = item.keys.first
-          item_value = item.values.first
-          items[item_name] = item_value
-        end
-        items
-      }
-    end
-
-    def put(item_name, attributes)
-      @sdb.put_attributes(@name, item_name, attributes)
-    end
-
-    def select(query)
-      op = Operation.new(@sdb, :select, "select * from #{name} where #{query}")
-      op.inject([]){|items,results|
-        batch_items = results[:items].map{|pair|
-          item = pair.values.first
-          item.merge!({'itemName()' => pair.keys.first})
-          item
-        }
-        items.concat(batch_items)
-      }
-    end
-
-    def delete(item_name)
-      @sdb.delete_attributes(@name, item_name)
-    end
-  end
-
   class Task
     fattr :callback    => lambda{}
     fattr :chunk_size  => 100

data/lib/sdbtools/database.rb

@@ -0,0 +1,49 @@
+module SDBTools
+  class Database
+    attr_reader   :sdb
+    attr_accessor :logger
+
+    def initialize(access_key=nil, secret_key=nil, options={})
+      @logger = (options[:logger] ||= ::Logger.new($stderr))
+      @sdb    =         MeasuredSdbInterface.new(
+        options.delete(:sdb_interface) {
+          RightAws::SdbInterface.new(access_key, secret_key, options)
+        })
+    end
+
+    def domains
+      domains_op = Operation.new(@sdb, :list_domains, nil)
+      domains_op.inject([]) {|domains, (results, operation)|
+        domains.concat(results[:domains])
+        domains
+      }
+    end
+
+    def domain(domain_name)
+      Domain.new(@sdb, domain_name)
+    end
+
+    def domain_exists?(domain_name)
+      domains.include?(domain_name)
+    end
+
+    def create_domain(domain_name)
+      @sdb.create_domain(domain_name)
+      domain(domain_name)
+    end
+
+    def delete_domain(domain_name)
+      @sdb.delete_domain(domain_name)
+    end
+
+    def make_dump(domain, filename)
+      Dump.new(domain, filename, @logger)
+    end
+
+    def make_load(domain, filename)
+      Load.new(domain, filename, @logger)
+    end
+
+    private
+  end
+end

data/lib/sdbtools/domain.rb

@@ -0,0 +1,74 @@
+module SDBTools
+  class Domain
+    attr_reader :name
+
+    def initialize(sdb, name)
+      @sdb        = sdb
+      @name       = name
+      @item_names = nil
+      @count      = nil
+    end
+
+    def [](item_name)
+      @sdb.get_attributes(name, item_name)[:attributes]
+    end
+
+    def item_names
+      return @item_names if @item_names
+      query = Operation.new(@sdb, :query, @name, nil, nil)
+      @item_names = query.inject([]) {|names, results| 
+        names.concat(results[:items])
+        names
+      }
+    end
+
+    def count
+      return @count if @count
+      @count = selection.count
+    end
+
+    def items(item_names)
+      names  = item_names.map{|n| "'#{n}'"}.join(', ')
+      query  = "select * from #{name} where itemName() in (#{names})"
+      select = Operation.new(@sdb, :select, query)
+      select.inject({}) {|items, results|
+        results[:items].each do |item|
+          item_name  = item.keys.first
+          item_value = item.values.first
+          items[item_name] = item_value
+        end
+        items
+      }
+    end
+
+    def get(item_name, attribute_name=nil)
+      @sdb.get_attributes(@name, item_name, attribute_name)
+    end
+
+    def put(item_name, attributes, options={})
+      replace = options[:replace] ? :replace : false
+      @sdb.put_attributes(@name, item_name, attributes, replace)
+    end
+
+    def selection(options={})
+      Selection.new(@sdb, name, options)
+    end
+
+    # Somewhat deprecated. Use #selection() instead
+    def select(query)
+      op = Operation.new(@sdb, :select, "select * from #{name} where #{query}")
+      op.inject([]){|items,(results, operation)|
+        batch_items = results[:items].map{|pair|
+          item = pair.values.first
+          item.merge!({'itemName()' => pair.keys.first})
+          item
+        }
+        items.concat(batch_items)
+      }
+    end
+
+    def delete(item_name, attributes={})
+      @sdb.delete_attributes(@name, item_name, attributes)
+    end
+  end
+end

data/lib/sdbtools/measured_sdb_interface.rb

@@ -0,0 +1,48 @@
+require 'sdbtools/transaction'
+require 'delegate'
+
+module SDBTools
+  class MeasuredSdbInterface < DelegateClass(RightAws::SdbInterface)
+    def create_domain(*args, &block)
+      Transaction.open("create_domain") do |t|
+        t.measure_aws_call{super(*args, &block)}
+      end
+    end
+    
+    def delete_domain(*args, &block)
+      Transaction.open("delete_domain") do |t|
+        t.measure_aws_call{super(*args, &block)}
+      end
+    end
+
+    def list_domains(*args, &block)
+      Transaction.open("list_domains") do |t|
+        t.measure_aws_call{super(*args, &block)}
+      end
+    end
+
+    def put_attributes(*args, &block)
+      Transaction.open("put_attributes") do |t|
+        t.measure_aws_call{super(*args, &block)}
+      end
+    end
+
+    def get_attributes(*args, &block)
+      Transaction.open("get_attributes") do |t|
+        t.measure_aws_call{super(*args, &block)}
+      end
+    end
+
+    def select(*args, &block)
+      Transaction.open("select #{args.first}") do |t|
+        t.measure_aws_call{super(*args, &block)}
+      end
+    end
+
+    def query(*args, &block)
+      Transaction.open("query") do |t|
+        t.measure_aws_call{super(*args, &block)}
+      end
+    end
+  end
+end

data/lib/sdbtools/operation.rb

@@ -0,0 +1,34 @@
+module SDBTools
+  # An Operation represents a SimpleDB operation and handles the details of
+  # re-requesting "next tokens" until the operation is complete.
+  class Operation
+    include Enumerable
+
+    attr_reader :method
+    attr_reader :args
+    attr_reader :starting_token
+
+    def initialize(sdb, method, *args)
+      @options = args.last.is_a?(Hash) ? args.pop : {}
+      @sdb     = sdb
+      @method  = method
+      @args    = args
+      @starting_token = @options[:starting_token]
+    end
+
+    # Yields once for each result set, until there is no next token.
+    def each
+      Transaction.open(":#{method} operation") do |t|
+        next_token = starting_token
+        begin
+          args = @args.dup
+          args << next_token
+          results = @sdb.send(@method, *args)
+          yield(results, self)
+          next_token = results[:next_token]
+        end while next_token
+      end
+    end
+
+  end
+end

data/lib/sdbtools/selection.rb

@@ -0,0 +1,212 @@
+require 'arrayfields'
+require 'logger'
+
+module SDBTools
+  class Selection
+    include Enumerable
+
+    MAX_BATCH_LIMIT      = 250
+    DEFAULT_RESULT_LIMIT = 100
+
+    attr_accessor :domain
+    attr_accessor :attributes
+    attr_accessor :conditions
+    attr_reader   :limit
+    attr_reader   :batch_limit
+    attr_accessor :offset
+    attr_accessor :order_by
+    attr_accessor :order
+    attr_accessor :sdb
+    attr_accessor :logger
+
+    # For testing
+    attr_writer :starting_token
+
+    def self.quote_name(name)
+      if name.to_s =~ /^[A-Z$_][A-Z0-9$_]*$/i
+        name.to_s
+      else
+        "`" + name.to_s.gsub("`", "``") + "`"
+      end
+    end
+
+    def self.quote_value(value)
+      '"' + value.to_s.gsub(/"/, '""') + '"'
+    end
+
+    def initialize(sdb, domain, options={})
+      @sdb             = sdb
+      @domain          = domain.to_s
+      @attributes      = options.fetch(:attributes) { :all }
+      @conditions      = options[:conditions].to_s
+      @order           = options.fetch(:order) { :ascending }
+      @order_by        = options.fetch(:order_by) { :none }
+      @order_by        = @order_by.to_s unless @order_by == :none
+      self.limit       = options.fetch(:limit) { :none }
+      self.batch_limit = options.fetch(:batch_limit) { DEFAULT_RESULT_LIMIT }.to_i
+      @offset          = options.fetch(:offset) { 0 }.to_i
+      @logger          = options.fetch(:logger){::Logger.new($stderr)}
+    end
+
+    def to_s(query_limit=limit, offset=0)
+      "SELECT #{output_list}"       \
+      " FROM #{quote_name(domain)}" \
+      "#{match_expression}"         \
+      "#{sort_instructions}"        \
+      "#{limit_clause(query_limit,offset)}"
+    end
+
+    def count_expression
+      "SELECT count(*) FROM #{quote_name(domain)}#{match_expression}#{sort_instructions}#{limit_clause_for_count}"
+    end
+
+    def offset_count_expression
+      "SELECT count(*) FROM #{quote_name(domain)}#{match_expression}#{sort_instructions} LIMIT #{offset}"
+    end
+
+    def count
+      Transaction.open(count_expression) do |t|
+        @count ||= count_operation.inject(0){|count, (results, operation)| 
+          count += results[:items].first["Domain"]["Count"].first.to_i
+        }
+      end
+    end
+
+    alias_method :size, :count
+    alias_method :length, :count
+
+    def each
+      return if limit == 0
+      num_items = 0
+      Transaction.open(to_s) do
+        select_operation(limit, num_items).each do |results, operation|
+          results[:items].each do |item|
+            yield(item.keys.first, item.values.first)
+            num_items += 1
+            return if limit != :none && num_items >= limit
+          end
+          operation.args[0] = to_s(limit, num_items)
+        end
+      end
+    rescue RightAws::AwsError => error
+      if error.message =~ /InvalidQueryExpression/
+        raise error, error.message.to_s + " (#{to_s(limit, num_items)})", error.backtrace
+      else
+        raise
+      end
+    end
+
+    def results
+      @results ||= inject(Arrayfields.new){|results, (name, value)|
+        results[name] = value
+        results
+      }
+    end
+
+    def count_operation
+      Operation.new(sdb, :select, count_expression, :starting_token => starting_token)
+    end
+
+    def offset_count_operation
+      Operation.new(sdb, :select, offset_count_expression)
+    end
+
+    def select_operation(query_limit=limit, offset=0)
+      Operation.new(sdb, :select, to_s(query_limit, offset), :starting_token => starting_token)
+    end
+
+    def starting_token
+      @starting_token ||=
+        case offset
+        when 0 then nil
+        else
+          op    = offset_count_operation
+          count = 0
+          op.each do |results, operation|
+          count += results[:items].first["Domain"]["Count"].first.to_i
+          if count == offset || results[:next_token].nil?
+            return results[:next_token]
+          end
+        end
+          raise "Failed to find offset #{offset}"
+        end
+    end
+
+    def limit=(new_limit)
+      case new_limit
+      when :none, Integer then 
+        @limit = new_limit
+      else
+        raise ArgumentError, "Limit must be integer or :none"
+      end
+    end
+
+    def batch_limit=(new_limit)
+      case new_limit
+      when (1..MAX_BATCH_LIMIT) then 
+        @batch_limit = new_limit
+      else
+        raise RangeError, "Limit must be 1..#{MAX_BATCH_LIMIT}"
+      end
+    end
+
+    private
+
+    def quote_name(name)
+      self.class.quote_name(name)
+    end
+
+    def quote_value(value)
+      self.class.quote_value(value)
+    end
+
+    def output_list
+      case attributes
+      when Array then attributes.map{|a| quote_name(a)}.join(", ")
+      when :all then "*"
+      else raise ScriptError, "Bad attributes: #{attributes.inspect}"
+      end
+    end
+
+    def match_expression
+      if conditions.empty? then "" else " WHERE #{conditions}" end
+    end
+
+    def limit_clause(query_limit=:none, offset=0)
+      case query_limit
+      when :none then format_limit_clause(batch_limit)
+      else
+        remaining_query_limit = query_limit - offset
+        this_batch_limit = 
+          remaining_query_limit < batch_limit ? remaining_query_limit : batch_limit
+        format_limit_clause(this_batch_limit)
+      end
+    end
+
+    def format_limit_clause(this_batch_limit)
+      case this_batch_limit
+      when DEFAULT_RESULT_LIMIT then ""
+      else " LIMIT #{this_batch_limit}"
+      end
+    end
+
+    # There are special rules for the LIMIT clause when executing a count(*)
+    # select. Normally it specifies batch size, capped at 250. But for count(*)
+    # expressions it specifies the max count, which may be arbitrarily large
+    def limit_clause_for_count
+      case limit
+      when :none then ""
+      else " LIMIT #{limit}"
+      end
+    end
+
+    def sort_instructions
+      case order_by
+      when :none then ""
+      else 
+        direction = (order == :ascending ? "ASC" : "DESC")
+        " ORDER BY #{quote_name(order_by)} #{direction}"
+      end
+    end
+  end
+end

data/lib/sdbtools/transaction.rb

@@ -0,0 +1,119 @@
+require 'benchmark'
+
+module SDBTools
+
+  # SimpleDB is not remotely transactional.  A Transaction in this context is
+  # just a way to group together a series of SimpleDB requests for the purpose
+  # of benchmarking.
+  class Transaction
+
+    def self.open(description, on_close=self.on_close)
+      transaction = self.new(description, on_close)
+      transaction_stack.push(transaction)
+      yield(transaction)
+    ensure
+      transaction = transaction_stack.pop
+      transaction.close
+    end
+
+    # Set the default on_close action. An on_close action receives a Transaction
+    # object which is being closed
+    def self.on_close=(action)
+      @on_close_action = action
+    end
+
+    # Get the default on_close action
+    def self.on_close
+      if current
+        current.on_close
+      else
+        @on_close_action ||= lambda{|t|}
+      end
+    end
+
+    # Usage:
+    #   Transaction.on_close = Transaction.log_transaction_close(logger)
+    def self.log_transaction_close(logger, cutoff_level=:none)
+      pattern = "%s \"%s\" (User %0.6u; System %0.6y; CPU %0.6t; Clock %0.6r; Box %0.6f; Reqs %d; Items %d)"
+      lambda do |t|
+        if cutoff_level == :none || cutoff_level <= t.nesting_level
+          prefix = "*" * (t.nesting_level + 1)
+          logger.info(
+            t.times.format(
+              pattern,
+              prefix, 
+              t.description,
+              t.box_usage, 
+              t.request_count, 
+              t.item_count))
+        end
+      end
+    end
+
+    def self.add_stats(box_usage, request_count, item_count, times)
+      current and current.add_stats(box_usage, request_count, item_count, times)
+    end
+
+    def self.current
+      transaction_stack.last
+    end
+
+    def self.transaction_stack
+      Thread.current[:sdbtools_transaction_stack] ||= []
+    end
+
+    attr_reader :description
+    attr_reader :box_usage
+    attr_reader :request_count
+    attr_reader :item_count
+    attr_reader :times
+    attr_reader :on_close
+    attr_reader :nesting_level
+
+    def initialize(description, on_close_action=self.class.on_close)
+      @description   = description
+      @box_usage     = 0.0
+      @request_count = 0
+      @item_count    = 0
+      @on_close      = on_close_action
+      @times         = Benchmark::Tms.new
+      @nesting_level = self.class.transaction_stack.size
+    end
+
+    def close
+      self.class.add_stats(@box_usage, @request_count, @item_count, @times)
+      @on_close.call(self)
+      self
+    end
+
+    def add_stats(box_usage, request_count, item_count, times=Benchmark::Tms.new)
+      @request_count += request_count.to_i
+      @box_usage     += box_usage.to_f
+      @item_count    += item_count
+      @times         += times
+    end
+
+    def add_stats_from_aws_response(response)
+      response   = response.nil? ? {} : response
+      box_usage  = response[:box_usage].to_f
+      item_count = Array(response[:items]).size + Array(response[:domains]).size
+      item_count += 1 if response.key?(:attributes)
+      add_stats(box_usage, 1, item_count)
+      response
+    end
+
+    def measure_aws_call
+      add_stats_from_aws_response(time{yield})
+    end
+
+    # Benchmark the block and add its times to the transaction
+    def time
+      result = nil
+      self.times.add! do 
+        result = yield 
+      end
+      result
+    end
+
+  end
+end

data/script/console

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+require 'logger'
+require 'irb'
+
+require File.expand_path('../lib/sdbtools', File.dirname(__FILE__))
+include SDBTools
+Transaction.on_close = Transaction.log_transaction_close(::Logger.new($stderr))
+IRB.start

data/spec/domain_spec.rb

@@ -0,0 +1,39 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+module SDBTools
+  describe Domain do
+    before :each do
+      @sdb = stub("SDB")
+    end
+
+    context "given a name and an SDB interface" do
+      before :each do
+        @it = Domain.new(@sdb, "foo")
+      end
+
+      it "should be able to get selected item attributes" do
+        @sdb.should_receive(:get_attributes).
+          with("foo", "bar", "baz")
+          @it.get("bar", "baz")
+      end
+
+      it "should be able to delete selected item attributes" do
+        @sdb.should_receive(:delete_attributes).
+          with("foo", "bar", {"baz" => ["buz"]})
+          @it.delete("bar", {"baz" => ["buz"]})
+      end
+
+      it "should be able to write selected item attributes" do
+        @sdb.should_receive(:put_attributes).
+          with("foo", "bar", {"baz" => ["buz"]}, false)
+          @it.put("bar", {"baz" => ["buz"]})
+      end
+
+      it "should be able to overwrite selected item attributes" do
+        @sdb.should_receive(:put_attributes).
+          with("foo", "bar", {"baz" => ["buz"]}, :replace)
+          @it.put("bar", {"baz" => ["buz"]}, :replace => true)
+      end
+    end
+  end
+end

data/spec/operation_spec.rb

@@ -16,5 +16,15 @@ module SDBTools
         @it.each do break; end
       end
     end
+
+    it "should pass itself into the given block" do
+      op = :unset
+      @sdb.stub!(:select).and_return({})
+      @it = Operation.new(@sdb, :select, "ARG")
+      op = @it.each do |results, operation|
+        break operation
+      end
+      op.should equal(@it)
+    end
   end
 end

data/spec/selection_spec.rb

@@ -80,10 +80,7 @@ module SDBTools
       before :each do
         @it = Selection.new(@sdb, "DOMAIN", 
           :attributes => ["attr foo", "bar"],
-          :conditions => [
-            'bar == "buz"', 
-            "AND",
-            ['`attr foo` between ? and ?', 4, '1"2']])
+          :conditions => 'bar == "buz" AND `attr foo` between "4" and "1""2"')
       end
 
       it "should format quote and interpolate the conditions" do
@@ -102,7 +99,7 @@ module SDBTools
       end
 
       it "should append a limit clause" do
-        @it.to_s.should be == "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' LIMIT 250"
+        @it.to_s.should be == "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar'"
       end
 
       it "should be able to generate a count expression" do
@@ -135,6 +132,38 @@ module SDBTools
       end
     end
 
+    context "with a high limit" do
+      before :each do
+        @it = Selection.new(@sdb, "DOMAIN",
+          :attributes => [:foo, :bar], 
+          :conditions => ["foo == 'bar'"],
+          :limit      => 251,
+          :batch_limit => 250)
+      end
+
+      it "should append a limit clause" do
+        @it.to_s.should be == "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' LIMIT 250"
+      end
+
+      it "should be able to generate a count expression" do
+        @it.count_expression.should be == 
+          "SELECT count(*) FROM DOMAIN WHERE foo == 'bar' LIMIT 251"
+      end
+
+      it "should limit results" do
+        @sdb = stub("SDB")
+        @sdb.should_receive(:select).
+          with("SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' LIMIT 250", nil).
+          and_return(select_results(("1".."250"), "TOKEN1"))
+        @sdb.should_receive(:select).
+          with("SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' LIMIT 1", "TOKEN1").
+          and_return(select_results(["251"], nil))
+        @it.sdb = @sdb
+        @it.results.size.should == 251
+        @it.results.keys.should == ("1".."251").to_a
+      end
+    end
+
     context "ordered by an attribute" do
       before :each do
         @it = Selection.new(@sdb, "DOMAIN",
@@ -180,11 +209,11 @@ module SDBTools
 
       context "with a limit" do
         before :each do
-          @it.limit = 250
+          @it.limit = 12
         end
         
         specify { @it.to_s.should be == 
-          "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' ORDER BY foo DESC LIMIT 250"
+          "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' ORDER BY foo DESC LIMIT 12"
         }
       end
     end
@@ -192,12 +221,12 @@ module SDBTools
     # We can't yet support large limits. In order to do so, it will be necessary
     # to implement limit chunking, where the limit is split across multiple
     # requests of 250 items and a final request of limit % 250 items.
-    it "should reject limits > 250" do
+    it "should reject batch limits > 250" do
       lambda do
-        Selection.new(@sdb, "DOMAIN", :limit => 251) 
+        Selection.new(@sdb, "DOMAIN", :batch_limit => 251) 
       end.should raise_error
       lambda do
-        Selection.new(@sdb, "DOMAIN").limit = 251
+        Selection.new(@sdb, "DOMAIN").batch_limit = 251
       end.should raise_error
     end
 
@@ -273,10 +302,10 @@ module SDBTools
       it "should be able to get selection results" do
         @sdb = stub("SDB")
         @sdb.should_receive(:select).
-          with(@it.to_s, "TOKEN2").
+          with(@it.to_s(10,0), "TOKEN2").
           and_return(select_results(["foo", "bar"], "TOKEN5"))
         @sdb.should_receive(:select).
-          with(@it.to_s, "TOKEN5").
+          with(@it.to_s(10,2), "TOKEN5").
           and_return(select_results(["baz", "buz"], nil))
         @it.sdb = @sdb
         @it.starting_token = "TOKEN2"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: sdbtools
 version: !ruby/object:Gem::Version 
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors: 
 - Avdi Grimm
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-23 00:00:00 -05:00
+date: 2010-01-24 00:00:00 -05:00
 default_executable: sdbtool
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -78,7 +78,14 @@ files:
 - VERSION
 - bin/sdbtool
 - lib/sdbtools.rb
-- lib/selection.rb
+- lib/sdbtools/database.rb
+- lib/sdbtools/domain.rb
+- lib/sdbtools/measured_sdb_interface.rb
+- lib/sdbtools/operation.rb
+- lib/sdbtools/selection.rb
+- lib/sdbtools/transaction.rb
+- script/console
+- spec/domain_spec.rb
 - spec/operation_spec.rb
 - spec/selection_spec.rb
 - spec/spec_helper.rb
@@ -111,6 +118,7 @@ signing_key:
 specification_version: 3
 summary: A high-level OO interface to Amazon SimpleDB
 test_files: 
+- spec/domain_spec.rb
 - spec/selection_spec.rb
 - spec/operation_spec.rb
 - spec/spec_helper.rb

data/lib/selection.rb

@@ -1,197 +0,0 @@
-require 'arrayfields'
-require 'logger'
-
-module SDBTools
-  class Selection
-    include Enumerable
-
-    MAX_RESULT_LIMIT     = 250
-    DEFAULT_RESULT_LIMIT = 100
-
-    attr_accessor :domain
-    attr_accessor :attributes
-    attr_accessor :conditions
-    attr_reader   :limit
-    attr_accessor :offset
-    attr_accessor :order_by
-    attr_accessor :order
-    attr_accessor :sdb
-    attr_accessor :logger
-
-    # For testing
-    attr_writer :starting_token
-
-    def self.quote_name(name)
-      if name.to_s =~ /^[A-Z$_][A-Z0-9$_]*$/i
-        name.to_s
-      else
-        "`" + name.to_s.gsub("`", "``") + "`"
-      end
-    end
-
-    def self.quote_value(value)
-      '"' + value.to_s.gsub(/"/, '""') + '"'
-    end
-
-    def initialize(sdb, domain, options={})
-      @sdb        = sdb
-      @domain     = domain.to_s
-      @attributes = options.fetch(:attributes) { :all }
-      @conditions = Array(options[:conditions])
-      @order      = options.fetch(:order) { :ascending }
-      @order_by   = options.fetch(:order_by) { :none }
-      @order_by   = @order_by.to_s unless @order_by == :none
-      self.limit  = options.fetch(:limit) { DEFAULT_RESULT_LIMIT }
-      @offset     = options.fetch(:offset) { 0 }.to_i
-      @logger     = options.fetch(:logger){::Logger.new($stderr)}
-    end
-
-    def to_s
-      "SELECT #{output_list} FROM #{quote_name(domain)}#{match_expression}#{sort_instructions}#{limit_clause}"
-    end
-
-    def count_expression
-      "SELECT count(*) FROM #{quote_name(domain)}#{match_expression}#{sort_instructions}#{limit_clause_for_count}"
-    end
-
-    def offset_count_expression
-      "SELECT count(*) FROM #{quote_name(domain)}#{match_expression}#{sort_instructions} LIMIT #{offset}"
-    end
-
-    def count
-      @count ||= count_operation.inject(0){|count, results| 
-        count += results[:items].first["Domain"]["Count"].first.to_i
-      }
-    end
-
-    alias_method :size, :count
-    alias_method :length, :count
-
-    def each
-      return if limit == 0
-      num_items = 0
-      select_operation.each do |results|
-        results[:items].each do |item|
-          yield(item.keys.first, item.values.first)
-          num_items += 1
-          return if limit != :none && num_items >= limit
-        end
-      end
-    end
-
-    def results
-      @results ||= inject(Arrayfields.new){|results, (name, value)|
-        results[name] = value
-        results
-      }
-    end
-
-    def count_operation
-      Operation.new(sdb, :select, count_expression, :starting_token => starting_token)
-    end
-
-    def offset_count_operation
-      Operation.new(sdb, :select, offset_count_expression)
-    end
-
-    def select_operation
-      Operation.new(sdb, :select, to_s, :starting_token => starting_token)
-    end
-
-    def starting_token
-      @starting_token ||=
-        case offset
-        when 0 then nil
-        else
-          op    = offset_count_operation
-          count = 0
-          op.each do |results|
-          count += results[:items].first["Domain"]["Count"].first.to_i
-          if count == offset || results[:next_token].nil?
-            return results[:next_token]
-          end
-        end
-          raise "Failed to find offset #{offset}"
-        end
-    end
-
-    def limit=(new_limit)
-      # We can't yet support large limits. In order to do so, it will be necessary
-      # to implement limit chunking, where the limit is split across multiple
-      # requests of 250 items and a final request of limit % 250 items.
-      case new_limit
-      when :none, (0..MAX_RESULT_LIMIT) then 
-        @limit = new_limit
-      else
-        raise RangeError, "Limit must be 0..250 or :none"
-      end
-    end
-
-    private
-
-    def quote_name(name)
-      self.class.quote_name(name)
-    end
-
-    def quote_value(value)
-      self.class.quote_value(value)
-    end
-
-    def output_list
-      case attributes
-      when Array then attributes.map{|a| quote_name(a)}.join(", ")
-      when :all then "*"
-      else raise ScriptError, "Bad attributes: #{attributes.inspect}"
-      end
-    end
-
-    def match_expression
-      case conditions.size
-      when 0 then ""
-      else " WHERE #{prepared_conditions.join(' ')}"
-      end
-    end
-
-    def prepared_conditions
-      conditions.map { |condition|
-        case condition
-        when String then condition
-        when Array then
-          values   = condition.dup
-          template = values.shift.to_s
-          template.gsub(/\?/) {|match|
-            quote_value(values.shift.to_s)
-          }
-        else
-          raise ScriptError, "Bad condition: #{condition.inspect}"
-        end
-      }
-    end
-
-    def limit_clause
-      case limit
-      when :none then " LIMIT #{MAX_RESULT_LIMIT}"
-      when DEFAULT_RESULT_LIMIT then ""
-      else 
-        batch_limit = limit < MAX_RESULT_LIMIT ? limit : MAX_RESULT_LIMIT
-        " LIMIT #{batch_limit}"
-      end
-    end
-
-    def limit_clause_for_count
-      case limit
-      when :none then ""
-      else limit_clause
-      end
-    end
-
-    def sort_instructions
-      case order_by
-      when :none then ""
-      else 
-        direction = (order == :ascending ? "ASC" : "DESC")
-        " ORDER BY #{quote_name(order_by)} #{direction}"
-      end
-    end
-  end
-end