dm-adapter-simpledb 1.2.0 → 1.3.0

data/History.txt

@@ -1,3 +1,20 @@
+== 1.3.0 2010-01-19
+
+* 1 major enhancement:
+  * Now supports :offset in queries. Combined with improved :limit support, this
+    makes it possible to efficiently paginate result sets.
+
+* 1 minor enhancement:
+  * Better quoting of select statements using the 'sdbtools' library.
+
+== 1.2.0 2010-01-13
+
+* 2 minor enhancements:
+  * Better support for String attributes with multiple values which were not
+    created by the String chunking system.
+  * Now tries to take model and repository into account when determining storage
+    name for a table.
+
 == 1.1.0 2009-11-24
 
 * 2 major enhancements:

data/README

@@ -16,6 +16,7 @@ Features:
  * Array properties
  * Basic aggregation support (Model.count("..."))
  * String "chunking" permits attributes to exceed the 1024-byte limit
+ * Support for efficient :limit and :offset, for result set paging
 
 Note: as of version 1.0.0, this gem supports supports the DataMapper 0.10.*
 series and breaks backwards compatibility with DataMapper 0.9.*.
@@ -39,6 +40,7 @@ http://github.com/devver/dm-adapter-simpledb/
 == TODO
 
  * More complete handling of NOT conditions in queries
+ * Support for ORs, parens, and nested queries in general
  * Robust quoting in SELECT calls
  * Handle exclusive ranges natively
    Implement as inclusive range + filter step
@@ -48,15 +50,6 @@ http://github.com/devver/dm-adapter-simpledb/
    - Store floats using exponential notation
  * Option to store Date/Time/DateTime as ISO8601
  * Full aggregate support (min/max/etc)
- * Offset support
-   Note, from the SimpleDB documentation:
-   
-   "The next token returned by count(*) and select are interchangeable as long
-   as the where and order by clauses match. For example, if you want to return
-   the 200 items after the first 10,000 (similar to an offset), you can perform
-   a count with a limit clause of 10,000 and use the next token to return the
-   next 200 items with select."
-
  * Option to use libxml if available
  * Parallelized queries for increased throughput
  * Support of normalized 1:1 table:domain schemes that works with associations
@@ -64,6 +57,7 @@ http://github.com/devver/dm-adapter-simpledb/
  * Support BatchPutAttributes
  * Silence SSL warnings
    See http://pivotallabs.com/users/carl/blog/articles/1079-standup-blog-11-24-2009-model-validations-without-backing-store-associations-to-array-and-ssl-with-aws
+ * Token cache for reduced requests when given an offset
 
 == Usage
   

data/Rakefile

@@ -81,7 +81,7 @@ END
     gem.add_dependency('dm-migrations',         '~> 0.10.0')
     gem.add_dependency('dm-types',              '~> 0.10.0')
     gem.add_dependency('uuidtools',             '~> 2.0')
-    gem.add_dependency('right_aws',             '~> 1.10')
+    gem.add_dependency('sdbtools',              '~> 0.2')
   end
   Jeweler::GemcutterTasks.new
 rescue LoadError

data/VERSION

@@ -1 +1 @@
-1.2.0
+1.3.0

data/lib/dm-adapter-simpledb.rb

@@ -8,6 +8,7 @@ require 'dm-aggregates'
 require 'digest/sha1'
 require 'right_aws'
 require 'uuidtools'
+require 'sdbtools'
 
 require 'dm-adapter-simpledb/sdb_array'
 require 'dm-adapter-simpledb/utils'

data/lib/dm-adapter-simpledb/adapters/simpledb_adapter.rb

@@ -92,7 +92,16 @@ module DataMapper
         query.conditions.operands.reject!{ |op|
           !unsupported_conditions.include?(op)
         }
-        records = query.filter_records(proto_resources)
+
+        # This used to be a simple call to Query#filter_records(), but that
+        # caused the result limit to be re-imposed on an already limited result
+        # set, with the upshot that too few records were returned. So here we do
+        # everything filter_records() does EXCEPT limiting.
+        records = proto_resources
+        records = records.uniq if query.unique?
+        records = query.match_records(records)
+        records = query.sort_records(records)
+
 
         records
       end
@@ -123,7 +132,10 @@ module DataMapper
       end
       
       def query(query_call, query_limit = 999999999)
-        select(query_call, query_limit).collect{|x| x.values[0]}
+        SDBTools::Operation.new(sdb, :select, query_call).inject([]){
+          |a, results| 
+          a.concat(results[:items].map{|i| i.values.first})
+        }[0...query_limit]
       end
       
       def aggregate(query)
@@ -147,7 +159,7 @@ module DataMapper
         token = :none
         begin
           results = sdb.get_attributes(domain, '__dm_consistency_token', '__dm_consistency_token')
-          tokens  = results[:attributes]['__dm_consistency_token']
+          tokens  = Array(results[:attributes]['__dm_consistency_token'])
         end until tokens.include?(@current_consistency_token)
       end
 
@@ -227,43 +239,59 @@ module DataMapper
             else
               raise ArgumentError, "Unsupported inclusion op: #{op.value.inspect}"
             end
+          when :or
+              # TODO There's no reason not to support OR
+              unsupported_conditions << op
           else raise "Invalid query op: #{op.inspect}"
           end
         end
         [conditions,order,unsupported_conditions]
       end
       
-      def select(query_call, query_limit)
-        items = []
-        time = Benchmark.realtime do
-          sdb_continuation_key = nil
-          while (results = sdb.select(query_call, sdb_continuation_key)) do
-            sdb_continuation_key = results[:next_token]
-            items += results[:items]
-            break if items.length > query_limit
-            break if sdb_continuation_key.nil?
-          end
-        end; DataMapper.logger.debug(format_log_entry(query_call, time))
-        items[0...query_limit]
-      end
-      
       #gets all results or proper number of results depending on the :limit
       def get_results(query, conditions, order)
         fields_to_request = query.fields.map{|f| f.field}
         fields_to_request << DmAdapterSimpledb::Record::METADATA_KEY
-        output_list = fields_to_request.join(', ')
-        query_call = "SELECT #{output_list} FROM #{domain} "
-        query_call << "WHERE #{conditions.compact.join(' AND ')}" if conditions.length > 0
-        query_call << " #{order}"
-        if query.limit!=nil
-          query_limit = query.limit
-          query_call << " LIMIT #{query.limit}" 
+        
+        selection = SDBTools::Selection.new(
+          sdb,
+          domain,
+          :attributes => fields_to_request)
+
+        if query.order && query.order.length > 0
+          query_object = query.order[0]
+          #anything sorted on must be a condition for SDB
+          conditions << "#{query_object.target.name} IS NOT NULL"
+          selection.order_by = query_object.target.name
+          selection.order    = case query_object.operator
+                               when :asc then :ascending
+                               when :desc then :descending
+                               else raise "Unrecognized sort direction"
+                               end
+        end
+        selection.conditions = conditions.compact.inject([]){|conds, cond|
+          conds << "AND" unless conds.empty?
+          conds << cond
+        }
+        if query.limit.nil?
+          selection.limit = :none
         else
-          #on large items force the max limit
-          query_limit = 999999999 #TODO hack for query.limit being nil
-          #query_call << " limit 2500" #this doesn't work with continuation keys as it halts at the limit passed not just a limit per query.
+          selection.limit = query.limit
+        end
+        unless query.offset.nil?
+          selection.offset = query.offset
         end
-        records = select(query_call, query_limit)
+
+        items = []
+        time = Benchmark.realtime do
+          # TODO update Record to be created from name/attributes pair
+          selection.each do |name, value| 
+            items << {name => value}
+          end
+        end
+        DataMapper.logger.debug(format_log_entry(selection.to_s, time))
+
+        items
       end
       
       # Creates an item name for a query

data/spec/integration/limit_and_order_spec.rb

@@ -105,6 +105,12 @@ describe 'with multiple records saved' do
       results = Hero.all(:limit => 110)
       results.should have(110).entries
     end
+
+    it "should be able to page through results" do
+      results1 = Hero.all(:limit => 10, :order => [:id.asc])
+      results2 = Hero.all(:offset => 9, :limit => 10, :order => [:id.asc])
+      results1.to_a.last.name.should be == results2.to_a.first.name
+    end
   end
 
 end

data/spec/unit/simpledb_adapter_spec.rb

@@ -74,7 +74,20 @@ describe DataMapper::Adapters::SimpleDBAdapter do
         ])
       @record = Product.first
     end
-    
   end
 
+  # it "should be able to request items with an offset" do
+  #   @sdb.should_receive(:select).
+  #     with(/count(\*).*LIMIT 10000/, anything).
+  #     exactly(1).times.
+  #     ordered.
+  #     and_return(:next_token => "TOKEN")
+  #   @sdb.should_receive(:select).
+  #     with(anything, "TOKEN").
+  #     exactly(1).times.
+  #     ordered.
+  #     and_return(:items => [])
+  #   @record = Product.all(:offset => 10000, :limit => 10)
+  # end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: dm-adapter-simpledb
 version: !ruby/object:Gem::Version 
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors: 
 - Jeremy Boles
@@ -13,7 +13,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-13 00:00:00 -05:00
+date: 2010-01-19 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -67,14 +67,14 @@ dependencies:
         version: "2.0"
     version: 
 - !ruby/object:Gem::Dependency 
-  name: right_aws
+  name: sdbtools
   type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        version: "1.10"
+        version: "0.2"
     version: 
 description: |
   A DataMapper adapter for Amazon's SimpleDB service.