mongoid-scroll 0.2.0 → 0.2.1

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+0.2.1 (3/21/2013)
+=================
+
+* Fix: scroll over a collection that has duplicate values while data is being modified in a way that causes a change in the natural sort order - [@dblock](https://github.com/dblock).
+
 0.2.0 (3/14/2013)
 =================
 

data/README.md

@@ -101,8 +101,8 @@ Indexes and Performance
 A query without a cursor is identical to a query without a scroll.
 
 ``` ruby
-# db.feed_items.find().sort({ position: 1 }).limit(5)
-Feed::Item.desc(:position).limit(5).scroll
+# db.feed_items.find().sort({ position: 1 }).limit(7)
+Feed::Item.desc(:position).limit(7).scroll
 ```
 
 Subsequent queries use an `$or` to avoid skipping items with the same value as the one at the current cursor position.
@@ -111,8 +111,8 @@ Subsequent queries use an `$or` to avoid skipping items with the same value as t
 # db.feed_items.find({ "$or" : [
 #   { "position" : { "$gt" : 13 }},
 #   { "position" : 13, "_id": { "$gt" : ObjectId("511d7c7c3b5552c92400000e") }}
-# ]}).sort({ position: 1 })
-Feed:Item.desc(:position).limit(5).scroll(cursor)
+# ]}).sort({ position: 1 }).limit(7)
+Feed:Item.desc(:position).limit(7).scroll(cursor)
 ```
 
 This means you need to hit an index on `position` and `_id`.

data/lib/mongoid/criterion/scrollable.rb

@@ -13,14 +13,14 @@ module Mongoid
         end
         # scroll field and direction
         scroll_field = criteria.options[:sort].keys.first
-        scroll_direction = criteria.options[:sort].values.first.to_i == 1 ? '$gt' : '$lt'
+        scroll_direction = criteria.options[:sort].values.first.to_i
         # scroll cursor from the parameter, with value and tiebreak_id
         field = criteria.klass.fields[scroll_field.to_s]
         cursor_options = { field_type: field.type, field_name: scroll_field, direction: scroll_direction }
         cursor = cursor.is_a?(Mongoid::Scroll::Cursor) ? cursor : Mongoid::Scroll::Cursor.new(cursor, cursor_options)
         # scroll
         if block_given?
-          criteria.where(cursor.criteria).each do |record|
+          criteria.where(cursor.criteria).order_by(_id: scroll_direction).each do |record|
             yield record, Mongoid::Scroll::Cursor.from_record(record, cursor_options)
           end
         else

data/lib/mongoid/scroll/cursor.rb

@@ -6,14 +6,15 @@ module Mongoid
 
       def initialize(value = nil, options = {})
         @field_type, @field_name = Mongoid::Scroll::Cursor.extract_field_options(options)
-        @direction = options[:direction] || '$gt'
+        @direction = options[:direction] || 1
         parse(value)
       end
 
       def criteria
         mongo_value = value.class.mongoize(value) if value
-        cursor_criteria = { field_name => { direction => mongo_value } } if mongo_value
-        tiebreak_criteria = { field_name => mongo_value, :_id => { '$gt' => tiebreak_id } } if mongo_value && tiebreak_id
+        compare_direction = direction == 1 ? "$gt" : "$lt"
+        cursor_criteria = { field_name => { compare_direction => mongo_value } } if mongo_value
+        tiebreak_criteria = { field_name => mongo_value, :_id => { compare_direction => tiebreak_id } } if mongo_value && tiebreak_id
         (cursor_criteria || tiebreak_criteria) ? { '$or' => [ cursor_criteria, tiebreak_criteria].compact } : {}
       end
 

data/lib/mongoid/scroll/version.rb

@@ -1,5 +1,5 @@
 module Mongoid
   module Scroll
-    VERSION = '0.2.0'
+    VERSION = '0.2.1'
   end
 end

data/lib/moped/scrollable.rb

@@ -2,27 +2,31 @@ module Moped
   module Scrollable
 
     def scroll(cursor = nil, options = { field_type: Moped::BSON::ObjectId }, &block)
+      query = Query.new(collection, operation.selector.dup)
+      query.operation.skip = operation.skip
+      query.operation.limit = operation.limit
       # we don't support scrolling over a criteria with multiple fields
-      if operation.selector["$orderby"] && operation.selector["$orderby"].keys.size != 1
-        raise Mongoid::Scroll::Errors::MultipleSortFieldsError.new(sort: operation.selector["$orderby"])
-      elsif ! operation.selector.has_key?("$orderby") || operation.selector["$orderby"].empty?
+      if query.operation.selector["$orderby"] && query.operation.selector["$orderby"].keys.size != 1
+        raise Mongoid::Scroll::Errors::MultipleSortFieldsError.new(sort: query.operation.selector["$orderby"])
+      elsif ! query.operation.selector.has_key?("$orderby") || query.operation.selector["$orderby"].empty?
         # introduce a default sort order if there's none
-        sort("_id" => 1)
+        query.sort(_id: 1)
       end
       # scroll field and direction
-      scroll_field = operation.selector["$orderby"].keys.first
-      scroll_direction = operation.selector["$orderby"].values.first.to_i == 1 ? '$gt' : '$lt'
+      scroll_field = query.operation.selector["$orderby"].keys.first
+      scroll_direction = query.operation.selector["$orderby"].values.first.to_i
       # scroll cursor from the parameter, with value and tiebreak_id
       cursor_options = { field_name: scroll_field, field_type: options[:field_type], direction: scroll_direction }
       cursor = cursor.is_a?(Mongoid::Scroll::Cursor) ? cursor : Mongoid::Scroll::Cursor.new(cursor, cursor_options)
-      operation.selector["$query"].merge!(cursor.criteria)
+      query.operation.selector["$query"] = query.operation.selector["$query"].merge(cursor.criteria)
+      query.operation.selector["$orderby"] = query.operation.selector["$orderby"].merge(_id: scroll_direction)
       # scroll
       if block_given?
-        each do |record|
+        query.each do |record|
           yield record, Mongoid::Scroll::Cursor.from_record(record, cursor_options)
         end
       else
-        self
+        query
       end
     end
 

data/spec/mongoid/criteria_spec.rb

@@ -96,20 +96,29 @@ describe Mongoid::Criteria do
   context "with overlapping data" do
     before :each do
       3.times { Feed::Item.create! a_integer: 5 }
+      Feed::Item.first.update_attributes!(name: Array(1000).join('a'))
     end
-    it "scrolls" do
-      records = []
-      cursor = nil
-      Feed::Item.desc(:a_integer).limit(2).scroll do |record, next_cursor|
-        records << record
-        cursor = next_cursor
-      end
-      records.size.should == 2
-      Feed::Item.desc(:a_integer).scroll(cursor) do |record, next_cursor|
-        records << record
+    it "natural order is different from order by id" do
+      # natural order isn't necessarily going to be the same as _id order
+      # if a document is updated and grows in size, it may need to be relocated and
+      # thus cause the natural order to change
+      Feed::Item.order_by("$natural" => 1).to_a.should_not eq Feed::Item.order_by(_id: 1).to_a
+    end
+    [ { a_integer: 1 }, { a_integer: -1 }].each do |sort_order|
+      it "scrolls by #{sort_order}" do
+        records = []
+        cursor = nil
+        Feed::Item.order_by(sort_order).limit(2).scroll do |record, next_cursor|
+          records << record
+          cursor = next_cursor
+        end
+        records.size.should == 2
+        Feed::Item.order_by(sort_order).scroll(cursor) do |record, next_cursor|
+          records << record
+        end
+        records.size.should == 3
+        records.should eq Feed::Item.all.sort(_id: sort_order[:a_integer]).to_a
       end
-      records.size.should == 3
-      records.should eq Feed::Item.all.to_a
     end
   end
 end

data/spec/moped/query_spec.rb

@@ -23,7 +23,7 @@ describe Moped::Query do
       Mongoid.default_session['feed_items'].find
     end
     it "adds a default sort by _id" do
-      subject.scroll.operation.selector["$orderby"].should == { "_id" => 1 }
+      subject.scroll.operation.selector["$orderby"].should == { _id: 1 }
     end
   end
   context "with data" do
@@ -98,20 +98,29 @@ describe Moped::Query do
   context "with overlapping data" do
     before :each do
       3.times { Feed::Item.create! a_integer: 5 }
+      Feed::Item.first.update_attributes!(name: Array(1000).join('a'))
     end
-    it "scrolls" do
-      records = []
-      cursor = nil
-      Mongoid.default_session['feed_items'].find.sort(a_integer: -1).limit(2).scroll do |record, next_cursor|
-        records << record
-        cursor = next_cursor
-      end
-      records.size.should == 2
-      Mongoid.default_session['feed_items'].find.sort(a_integer: -1).scroll(cursor) do |record, next_cursor|
-        records << record
+    it "natural order is different from order by id" do
+      # natural order isn't necessarily going to be the same as _id order
+      # if a document is updated and grows in size, it may need to be relocated and
+      # thus cause the natural order to change
+      Feed::Item.order_by("$natural" => 1).to_a.should_not eq Feed::Item.order_by(_id: 1).to_a
+    end
+    [ { a_integer: 1 }, { a_integer: -1 }].each do |sort_order|
+      it "scrolls by #{sort_order}" do
+        records = []
+        cursor = nil
+        Mongoid.default_session['feed_items'].find.sort(sort_order).limit(2).scroll do |record, next_cursor|
+          records << record
+          cursor = next_cursor
+        end
+        records.size.should == 2
+        Mongoid.default_session['feed_items'].find.sort(sort_order).scroll(cursor) do |record, next_cursor|
+          records << record
+        end
+        records.size.should == 3
+        records.should eq Mongoid.default_session['feed_items'].find.sort(_id: sort_order[:a_integer]).to_a
       end
-      records.size.should == 3
-      records.should eq Mongoid.default_session['feed_items'].find.to_a
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongoid-scroll
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-15 00:00:00.000000000 Z
+date: 2013-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongoid
@@ -95,7 +95,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4314128693864897181
+      hash: -2452181324971075888
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: