mongoid-scroll 0.3.2 → 0.3.7

data/examples/mongoid_scroll_feed.rb

@@ -4,6 +4,14 @@ Bundler.setup(:default, :development)
 require 'mongoid-scroll'
 require 'faker'
 
+if defined?(Moped)
+  Moped.logger = Logger.new($stdout)
+  Moped.logger.level = Logger::DEBUG
+else
+  Mongoid.logger.level = Logger::INFO
+  Mongo::Logger.logger.level = Logger::INFO if Mongoid::Compatibility::Version.mongoid5?
+end
+
 Mongoid.connect_to 'mongoid_scroll_demo'
 Mongoid.purge!
 
@@ -27,9 +35,6 @@ total_items.times do |_i|
   Feed::Item.create! title: Faker::Lorem.sentence, position: rands.pop
 end
 
-Moped.logger = Logger.new($stdout)
-Moped.logger.level = Logger::DEBUG
-
 Feed::Item.create_indexes
 
 total_shown = 0

data/examples/moped_scroll_feed.rb

@@ -7,6 +7,8 @@ require 'faker'
 Mongoid.connect_to 'mongoid_scroll_demo'
 Mongoid.purge!
 
+raise 'No Moped' unless Object.const_defined?(:Moped)
+
 # total items to insert
 total_items = 20
 # a MongoDB query will be executed every scroll_by items

data/lib/config/locales/en.yml

@@ -18,5 +18,5 @@ en:
           unsupported_field_type:
             message: "Unsupported field type."
             summary: "The type of the field '%{field}' is not supported: %{type}."
-            resolution: "Please open a feature request in https://github.com/dblock/mongoid-scroll."
+            resolution: "Please open a feature request in https://github.com/mongoid/mongoid-scroll."
 

data/lib/mongo/scrollable.rb

@@ -0,0 +1,34 @@
+module Mongo
+  module Scrollable
+    def scroll(cursor = nil, options = nil, &_block)
+      view = self
+      # we don't support scrolling over a view with multiple fields
+      raise Mongoid::Scroll::Errors::MultipleSortFieldsError.new(sort: view.sort) if view.sort && view.sort.keys.size != 1
+      # scroll field and direction
+      scroll_field = view.sort ? view.sort.keys.first : :_id
+      scroll_direction = view.sort ? view.sort.values.first.to_i : 1
+      # scroll cursor from the parameter, with value and tiebreak_id
+      options = { field_type: BSON::ObjectId } unless options
+      cursor_options = { field_name: scroll_field, direction: scroll_direction }.merge(options)
+      cursor = cursor.is_a?(Mongoid::Scroll::Cursor) ? cursor : Mongoid::Scroll::Cursor.new(cursor, cursor_options)
+      # make a view
+      view = Mongo::Collection::View.new(
+        view.collection,
+        view.selector.merge(cursor.criteria),
+        sort: (view.sort || {}).merge(_id: scroll_direction),
+        skip: skip,
+        limit: limit
+      )
+      # scroll
+      if block_given?
+        view.each do |record|
+          yield record, Mongoid::Scroll::Cursor.from_record(record, cursor_options)
+        end
+      else
+        view
+      end
+    end
+  end
+end
+
+Mongo::Collection::View.send(:include, Mongo::Scrollable)

data/lib/mongoid-scroll.rb

@@ -3,13 +3,10 @@ require 'i18n'
 I18n.load_path << File.join(File.dirname(__FILE__), 'config', 'locales', 'en.yml')
 
 require 'mongoid'
+require 'mongoid-compatibility'
 require 'mongoid/scroll/version'
-require 'mongoid/scroll/mongoid'
 require 'mongoid/scroll/errors'
 require 'mongoid/scroll/cursor'
-
 require 'moped/scrollable' if Object.const_defined?(:Moped)
-require 'mongoid/criterion/scrollable'
-
-Moped::Query.send(:include, Moped::Scrollable) if Object.const_defined?(:Moped)
-Mongoid::Criteria.send(:include, Mongoid::Criterion::Scrollable)
+require 'mongo/scrollable' if Object.const_defined?(:Mongo)
+require 'mongoid/criteria/scrollable'

data/lib/mongoid/criteria/scrollable.rb

@@ -0,0 +1,81 @@
+module Mongoid
+  class Criteria
+    module Scrollable
+      def scroll(cursor = nil, &_block)
+        raise_multiple_sort_fields_error if multiple_sort_fields?
+        criteria = dup
+        criteria.merge!(default_sort) if no_sort_option?
+        cursor_options = build_cursor_options(criteria)
+        cursor = cursor.is_a?(Mongoid::Scroll::Cursor) ? cursor : new_cursor(cursor, cursor_options)
+        cursor_criteria = build_cursor_criteria(criteria, cursor)
+        if block_given?
+          cursor_criteria.order_by(_id: scroll_direction(criteria)).each do |record|
+            yield record, cursor_from_record(record, cursor_options)
+          end
+        else
+          cursor_criteria
+        end
+      end
+
+      private
+
+      def raise_multiple_sort_fields_error
+        raise Mongoid::Scroll::Errors::MultipleSortFieldsError.new(sort: criteria.options.sort)
+      end
+
+      def multiple_sort_fields?
+        options.sort && options.sort.keys.size != 1
+      end
+
+      def no_sort_option?
+        options.sort.blank? || options.sort.empty?
+      end
+
+      def default_sort
+        asc(:_id)
+      end
+
+      def scroll_field(criteria)
+        criteria.options.sort.keys.first
+      end
+
+      def scroll_direction(criteria)
+        criteria.options.sort.values.first.to_i
+      end
+
+      def build_cursor_options(criteria)
+        {
+          field_type: scroll_field_type(criteria),
+          field_name: scroll_field(criteria),
+          direction: scroll_direction(criteria)
+        }
+      end
+
+      def new_cursor(cursor, cursor_options)
+        Mongoid::Scroll::Cursor.new(cursor, cursor_options)
+      end
+
+      def build_cursor_criteria(criteria, cursor)
+        cursor_criteria = criteria.dup
+        cursor_criteria.selector = { '$and' => [criteria.selector, cursor.criteria] }
+        cursor_criteria
+      end
+
+      def cursor_from_record(record, cursor_options)
+        Mongoid::Scroll::Cursor.from_record(record, cursor_options)
+      end
+
+      def scroll_field_type(criteria)
+        scroll_field = scroll_field(criteria)
+        field = criteria.klass.fields[scroll_field.to_s]
+        field.foreign_key? && field.object_id_field? ? bson_type : field.type
+      end
+
+      def bson_type
+        Mongoid::Compatibility::Version.mongoid3? ? Moped::BSON::ObjectId : BSON::ObjectId
+      end
+    end
+  end
+end
+
+Mongoid::Criteria.send(:include, Mongoid::Criteria::Scrollable)

data/lib/mongoid/scroll/cursor.rb

@@ -14,8 +14,12 @@ module Mongoid
         compare_direction = direction == 1 ? '$gt' : '$lt'
         cursor_criteria = { field_name => { compare_direction => mongo_value } } if mongo_value
         tiebreak_criteria = { field_name => mongo_value, :_id => { compare_direction => tiebreak_id } } if mongo_value && tiebreak_id
-        cursor_selector = Origin::Selector.new
-        cursor_selector.merge!('$or' => [cursor_criteria, tiebreak_criteria].compact) if cursor_criteria || tiebreak_criteria
+        cursor_selector = if Mongoid::Compatibility::Version.mongoid6? || Mongoid::Compatibility::Version.mongoid7?
+                            Mongoid::Criteria::Queryable::Selector.new
+                          else
+                            Origin::Selector.new
+                          end
+        cursor_selector['$or'] = [cursor_criteria, tiebreak_criteria].compact if cursor_criteria || tiebreak_criteria
         cursor_selector.__evolve_object_id__
       end
 
@@ -39,16 +43,16 @@ module Mongoid
         return unless value
         parts = value.split(':')
         unless parts.length >= 2
-          fail Mongoid::Scroll::Errors::InvalidCursorError.new(cursor: value)
+          raise Mongoid::Scroll::Errors::InvalidCursorError.new(cursor: value)
         end
         id = parts[-1]
         value = parts[0...-1].join(':')
         @value = Mongoid::Scroll::Cursor.parse_field_value(field_type, field_name, value)
-        if Mongoid::Scroll.mongoid3?
-          @tiebreak_id = Moped::BSON::ObjectId(id)
-        else
-          @tiebreak_id = BSON::ObjectId.from_string(id)
-        end
+        @tiebreak_id = if Mongoid::Compatibility::Version.mongoid3?
+                         Moped::BSON::ObjectId(id)
+                       else
+                         BSON::ObjectId.from_string(id)
+                       end
       end
 
       class << self
@@ -58,7 +62,7 @@ module Mongoid
           elsif options && (field = options[:field])
             [field.type.to_s, field.name.to_s]
           else
-            fail ArgumentError.new 'Missing options[:field_name] and/or options[:field_type].'
+            raise ArgumentError.new 'Missing options[:field_name] and/or options[:field_type].'
           end
         end
 
@@ -72,7 +76,7 @@ module Mongoid
           when 'Float' then value.to_f
           when 'Integer' then value.to_i
           else
-            fail Mongoid::Scroll::Errors::UnsupportedFieldTypeError.new(field: field_name, type: field_type)
+            raise Mongoid::Scroll::Errors::UnsupportedFieldTypeError.new(field: field_name, type: field_type)
           end
         end
 
@@ -85,7 +89,7 @@ module Mongoid
           when 'Float' then value.to_f
           when 'Integer' then value.to_i
           else
-            fail Mongoid::Scroll::Errors::UnsupportedFieldTypeError.new(field: field_name, type: field_type)
+            raise Mongoid::Scroll::Errors::UnsupportedFieldTypeError.new(field: field_name, type: field_type)
           end
         end
       end

data/lib/mongoid/scroll/errors/base.rb

@@ -27,7 +27,7 @@ module Mongoid
 
         private
 
-        BASE_KEY = 'mongoid.scroll.errors.messages' #:nodoc:
+        BASE_KEY = 'mongoid.scroll.errors.messages'.freeze #:nodoc:
 
         # Given the key of the specific error and the options hash, translate the
         # message.

data/lib/mongoid/scroll/version.rb

@@ -1,5 +1,5 @@
 module Mongoid
   module Scroll
-    VERSION = '0.3.2'
+    VERSION = '0.3.7'.freeze
   end
 end

data/lib/moped/scrollable.rb

@@ -1,17 +1,16 @@
 module Moped
   module Scrollable
     def scroll(cursor = nil, options = nil, &_block)
-      options = if Mongoid::Scroll.mongoid3?
-                  { field_type: Moped::BSON::ObjectId }
-                else
-                  { field_type: BSON::ObjectId }
-                end unless options
+      unless options
+        bson_type = Mongoid::Compatibility::Version.mongoid3? ? Moped::BSON::ObjectId : BSON::ObjectId
+        options = { field_type: bson_type }
+      end
       query = Query.new(collection, operation.selector.dup)
       query.operation.skip = operation.skip
       query.operation.limit = operation.limit
       # we don't support scrolling over a criteria with multiple fields
       if query.operation.selector['$orderby'] && query.operation.selector['$orderby'].keys.size != 1
-        fail Mongoid::Scroll::Errors::MultipleSortFieldsError.new(sort: query.operation.selector['$orderby'])
+        raise Mongoid::Scroll::Errors::MultipleSortFieldsError.new(sort: query.operation.selector['$orderby'])
       elsif !query.operation.selector.key?('$orderby') || query.operation.selector['$orderby'].empty?
         # introduce a default sort order if there's none
         query.sort(_id: 1)
@@ -35,3 +34,5 @@ module Moped
     end
   end
 end
+
+Moped::Query.send(:include, Moped::Scrollable)

data/mongoid-scroll.gemspec

@@ -10,9 +10,10 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = '>= 1.3.6'
   s.files = `git ls-files`.split("\n")
   s.require_paths = ['lib']
-  s.homepage = 'http://github.com/dblock/mongoid-scroll'
+  s.homepage = 'http://github.com/mongoid/mongoid-scroll'
   s.licenses = ['MIT']
   s.summary = 'Mongoid extensions to enable infinite scroll.'
   s.add_dependency 'mongoid', '>= 3.0'
+  s.add_dependency 'mongoid-compatibility'
   s.add_dependency 'i18n'
 end

data/spec/mongo/collection_view_spec.rb

@@ -0,0 +1,126 @@
+require 'spec_helper'
+
+if Object.const_defined?(:Mongo)
+  describe Mongo::Collection::View do
+    context 'scrollable' do
+      subject do
+        Mongoid.default_client['feed_items'].find
+      end
+      it ':scroll' do
+        expect(subject).to respond_to(:scroll)
+      end
+    end
+    context 'with multiple sort fields' do
+      subject do
+        Mongoid.default_client['feed_items'].find.sort(name: 1, value: -1)
+      end
+      it 'raises Mongoid::Scroll::Errors::MultipleSortFieldsError' do
+        expect { subject.scroll }.to raise_error Mongoid::Scroll::Errors::MultipleSortFieldsError,
+                                                 /You're attempting to scroll over data with a sort order that includes multiple fields: name, value./
+      end
+    end
+    context 'with no sort' do
+      subject do
+        Mongoid.default_client['feed_items'].find
+      end
+      it 'adds a default sort by _id' do
+        expect(subject.scroll.sort).to eq('_id' => 1)
+      end
+    end
+    context 'with data' do
+      before :each do
+        10.times do |i|
+          Mongoid.default_client['feed_items'].insert_one(
+            a_string: i.to_s,
+            a_integer: i,
+            a_datetime: DateTime.mongoize(DateTime.new(2013, i + 1, 21, 1, 42, 3, 'UTC')),
+            a_date: Date.mongoize(Date.new(2013, i + 1, 21)),
+            a_time: Time.mongoize(Time.at(Time.now.to_i + i))
+          )
+        end
+      end
+      context 'default' do
+        it 'scrolls all' do
+          records = []
+          Mongoid.default_client['feed_items'].find.scroll do |record, _next_cursor|
+            records << record
+          end
+          expect(records.size).to eq 10
+          expect(records).to eq Mongoid.default_client['feed_items'].find.to_a
+        end
+      end
+      { a_string: String, a_integer: Integer, a_date: Date, a_datetime: DateTime }.each_pair do |field_name, field_type|
+        context field_type do
+          it 'scrolls all with a block' do
+            records = []
+            Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(nil, field_type: field_type) do |record, _next_cursor|
+              records << record
+            end
+            expect(records.size).to eq 10
+            expect(records).to eq Mongoid.default_client['feed_items'].find.to_a
+          end
+          it 'scrolls all with a break' do
+            records = []
+            cursor = nil
+            Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(5).scroll(nil, field_type: field_type) do |record, next_cursor|
+              records << record
+              cursor = next_cursor
+            end
+            expect(records.size).to eq 5
+            Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(cursor, field_type: field_type) do |record, next_cursor|
+              records << record
+              cursor = next_cursor
+            end
+            expect(records.size).to eq 10
+            expect(records).to eq Mongoid.default_client['feed_items'].find.to_a
+          end
+          it 'scrolls in descending order' do
+            records = []
+            Mongoid.default_client['feed_items'].find.sort(field_name => -1).limit(3).scroll(nil, field_type: field_type, field_name: field_name) do |record, _next_cursor|
+              records << record
+            end
+            expect(records.size).to eq 3
+            expect(records).to eq Mongoid.default_client['feed_items'].find.sort(field_name => -1).limit(3).to_a
+          end
+          it 'map' do
+            record = Mongoid.default_client['feed_items'].find.limit(3).scroll(nil, field_type: field_type, field_name: field_name).map { |r| r }.last
+            cursor = Mongoid::Scroll::Cursor.from_record(record, field_type: field_type, field_name: field_name)
+            expect(cursor).to_not be nil
+            expect(cursor.to_s.split(':')).to eq [
+              Mongoid::Scroll::Cursor.transform_field_value(field_type, field_name, record[field_name.to_s]).to_s,
+              record['_id'].to_s
+            ]
+          end
+        end
+      end
+    end
+    context 'with overlapping data', if: MongoDB.mmapv1? do
+      before :each do
+        3.times { Feed::Item.create! a_integer: 5 }
+        Feed::Item.first.update_attributes!(name: Array(1000).join('a'))
+      end
+      it 'natural order is different from order by id' do
+        # natural order isn't necessarily going to be the same as _id order
+        # if a document is updated and grows in size, it may need to be relocated and
+        # thus cause the natural order to change
+        expect(Feed::Item.order_by('$natural' => 1).to_a).to_not eq Feed::Item.order_by(_id: 1).to_a
+      end
+      [{ a_integer: 1 }, { a_integer: -1 }].each do |sort_order|
+        it "scrolls by #{sort_order}" do
+          records = []
+          cursor = nil
+          Mongoid.default_client['feed_items'].find.sort(sort_order).limit(2).scroll do |record, next_cursor|
+            records << record
+            cursor = next_cursor
+          end
+          expect(records.size).to eq 2
+          Mongoid.default_client['feed_items'].find.sort(sort_order).scroll(cursor) do |record, _next_cursor|
+            records << record
+          end
+          expect(records.size).to eq 3
+          expect(records).to eq Mongoid.default_client['feed_items'].find.sort(sort_order.merge(_id: sort_order[:a_integer])).to_a
+        end
+      end
+    end
+  end
+end

data/spec/mongoid/criteria_spec.rb

@@ -13,6 +13,7 @@ describe Mongoid::Criteria do
                                                /You're attempting to scroll over data with a sort order that includes multiple fields: name, value./
     end
   end
+
   context 'with no sort' do
     subject do
       Feed::Item.all
@@ -42,7 +43,30 @@ describe Mongoid::Criteria do
         expect(records.size).to eq 10
         expect(records).to eq Feed::Item.all.to_a
       end
+      it 'does not change original criteria' do
+        criteria = Feed::Item.where(:a_time.gt => Time.new(2013, 7, 22, 1, 2, 3))
+        original_criteria = criteria.dup
+        criteria.limit(2).scroll
+        expect(criteria).to eq original_criteria
+        cursor = nil
+        criteria.limit(2).scroll(cursor) do |_record, next_cursor|
+          cursor = next_cursor
+        end
+        criteria.scroll(cursor) do |_record, next_cursor|
+          cursor = next_cursor
+        end
+        expect(criteria).to eq original_criteria
+      end
     end
+
+    context 'with a foreign key' do
+      it 'sorts by object id' do
+        records = []
+        Feed::Item.asc('publisher_id').scroll { |r, _| records << r }
+        expect(records).not_to be_empty
+      end
+    end
+
     { a_string: String, a_integer: Integer, a_date: Date, a_datetime: DateTime }.each_pair do |field_name, field_type|
       context field_type do
         it 'scrolls all with a block' do
@@ -68,6 +92,17 @@ describe Mongoid::Criteria do
           expect(records.size).to eq 10
           expect(records).to eq Feed::Item.all.to_a
         end
+        it 'scrolls from a cursor' do
+          last_record = nil
+          cursor = nil
+          Feed::Item.asc(field_name).limit(5).scroll do |record, next_cursor|
+            last_record = record
+            cursor = next_cursor
+          end
+          sixth_item = Feed::Item.asc(field_name).to_a[5]
+          from_item = Feed::Item.asc(field_name).scroll(cursor).to_a.first
+          expect(from_item).to eq sixth_item
+        end
         it 'scrolls in descending order' do
           records = []
           Feed::Item.desc(field_name).limit(3).scroll do |record, _next_cursor|
@@ -132,9 +167,9 @@ describe Mongoid::Criteria do
     end
     it 'merges cursor criteria when no sort field is given' do
       criteria = Feed::Item.where(:a_time.gt => Time.new(2013, 7, 22, 1, 2, 3))
-      feed_item_1 = Feed::Item.where(name: 'Feed Item 1').first
-      cursor_input = "#{feed_item_1.id}:#{feed_item_1.id}"
-      field_type = Mongoid::Scroll.mongoid3? ? Moped::BSON::ObjectId : BSON::ObjectId
+      feed_item = Feed::Item.where(name: 'Feed Item 1').first
+      cursor_input = "#{feed_item.id}:#{feed_item.id}"
+      field_type = Mongoid::Compatibility::Version.mongoid3? ? Moped::BSON::ObjectId : BSON::ObjectId
       cursor_options = { field_type: field_type, field_name: '_id', direction: 1 }
       cursor = Mongoid::Scroll::Cursor.new(cursor_input, cursor_options)
       records = []
@@ -160,7 +195,7 @@ describe Mongoid::Criteria do
       expect(records.map(&:name)).to eq ['embedded']
     end
   end
-  context 'with overlapping data' do
+  context 'with overlapping data', if: MongoDB.mmapv1? do
     before :each do
       3.times { Feed::Item.create! a_integer: 5 }
       Feed::Item.first.update_attributes!(name: Array(1000).join('a'))