mongoid-scroll 0.3.6 → 1.0.0

data/lib/mongoid/scroll/base_cursor.rb

@@ -0,0 +1,123 @@
+module Mongoid
+  module Scroll
+    class BaseCursor
+      attr_accessor :value, :tiebreak_id, :field_type, :field_name, :direction, :include_current
+
+      def initialize(value, options = {})
+        @value = value
+        @tiebreak_id = options[:tiebreak_id]
+        @field_type = options[:field_type]
+        @field_name = options[:field_name]
+        @direction = options[:direction] || 1
+        @include_current = options[:include_current] || false
+      end
+
+      def criteria
+        mongo_value = value.class.mongoize(value) if value
+        cursor_criteria = { field_name => { compare_direction => mongo_value } } if mongo_value
+        tiebreak_criteria = { field_name => mongo_value, :_id => { tiebreak_compare_direction => tiebreak_id } } if mongo_value && tiebreak_id
+        cursor_selector = if Mongoid::Compatibility::Version.mongoid6_or_newer?
+                            Mongoid::Criteria::Queryable::Selector.new
+                          else
+                            Origin::Selector.new
+                          end
+        cursor_selector['$or'] = [cursor_criteria, tiebreak_criteria].compact if cursor_criteria || tiebreak_criteria
+        cursor_selector.__evolve_object_id__
+      end
+
+      def sort_options
+        {
+          field_type: field_type,
+          field_name: field_name,
+          direction: direction
+        }
+      end
+
+      def to_s
+        raise NotImplementedError.new(:to_s)
+      end
+
+      class << self
+        def from_record(record, options)
+          cursor = new(nil, options)
+          record_value = record.respond_to?(cursor.field_name) ? record.send(cursor.field_name) : record[cursor.field_name]
+          cursor.value = cursor.send(:parse_field_value, cursor.field_type, cursor.field_name, record_value)
+          cursor.tiebreak_id = record['_id']
+          cursor
+        end
+      end
+
+      private
+
+      def parse_field_value(field_type, field_name, value)
+        return nil unless value
+
+        case field_type.to_s
+        when 'BSON::ObjectId' then BSON::ObjectId.from_string(value)
+        when 'String' then value.to_s == '' ? nil : value.to_s
+        when 'DateTime' then value.is_a?(DateTime) ? value : Time.at(value.to_i).to_datetime
+        when 'Time' then value.is_a?(Time) ? value : Time.at(value.to_i)
+        when 'Date' then value.is_a?(Date) ? value : Time.at(value.to_i).utc.to_date
+        when 'Float' then value.to_f
+        when 'Integer' then value.to_i
+        else
+          raise Mongoid::Scroll::Errors::UnsupportedFieldTypeError.new(field: field_name, type: field_type)
+        end
+      end
+
+      def transform_field_value(field_type, field_name, value)
+        return nil unless value
+
+        case field_type.to_s
+        when 'BSON::ObjectId' then value.to_s
+        when 'String' then value.to_s
+        when 'Date' then Time.utc(value.year, value.month, value.day).to_i
+        when 'DateTime', 'Time' then value.utc.to_i
+        when 'Float' then value.to_f
+        when 'Integer' then value.to_i
+        else
+          raise Mongoid::Scroll::Errors::UnsupportedFieldTypeError.new(field: field_name, type: field_type)
+        end
+      end
+
+      def extract_field_options(options)
+        if options && (field_name = options[:field_name]) && (field_type = options[:field_type])
+          {
+            field_type: field_type.to_s,
+            field_name: field_name.to_s,
+            direction: options[:direction] || 1,
+            include_current: options[:include_current] || false
+          }
+        elsif options && (field = options[:field])
+          {
+            field_type: field.type.to_s,
+            field_name: field.name.to_s,
+            direction: options[:direction] || 1,
+            include_current: options[:include_current] || false
+          }
+        end
+      end
+
+      def compare_direction
+        direction == 1 ? '$gt' : '$lt'
+      end
+
+      def tiebreak_compare_direction
+        if include_current
+          case compare_direction
+          when '$gt'
+            '$gte'
+          when '$lt'
+            '$lte'
+          end
+        else
+          compare_direction
+        end
+      end
+
+      def parse(_value)
+        raise NotImplementedError.new(:parse)
+      end
+    end
+  end
+end

data/lib/mongoid/scroll/cursor.rb

@@ -1,97 +1,34 @@
 module Mongoid
   module Scroll
-    class Cursor
-      attr_accessor :value, :tiebreak_id, :field_type, :field_name, :direction
-
+    class Cursor < BaseCursor
       def initialize(value = nil, options = {})
-        @field_type, @field_name = Mongoid::Scroll::Cursor.extract_field_options(options)
-        @direction = options[:direction] || 1
-        parse(value)
-      end
-
-      def criteria
-        mongo_value = value.class.mongoize(value) if value
-        compare_direction = direction == 1 ? '$gt' : '$lt'
-        cursor_criteria = { field_name => { compare_direction => mongo_value } } if mongo_value
-        tiebreak_criteria = { field_name => mongo_value, :_id => { compare_direction => tiebreak_id } } if mongo_value && tiebreak_id
-        cursor_selector = if Mongoid::Compatibility::Version.mongoid6? || Mongoid::Compatibility::Version.mongoid7?
-                            Mongoid::Criteria::Queryable::Selector.new
-                          else
-                            Origin::Selector.new
-                          end
-        cursor_selector['$or'] = [cursor_criteria, tiebreak_criteria].compact if cursor_criteria || tiebreak_criteria
-        cursor_selector.__evolve_object_id__
-      end
-
-      class << self
-        def from_record(record, options)
-          cursor = Mongoid::Scroll::Cursor.new(nil, options)
-          value = record.respond_to?(cursor.field_name) ? record.send(cursor.field_name) : record[cursor.field_name]
-          cursor.value = Mongoid::Scroll::Cursor.parse_field_value(cursor.field_type, cursor.field_name, value)
-          cursor.tiebreak_id = record['_id']
-          cursor
+        options = extract_field_options(options)
+        raise ArgumentError.new 'Missing options[:field_name] and/or options[:field_type].' unless options
+        if value
+          parts = value.split(':') if value
+          unless parts && parts.length >= 2
+            raise Mongoid::Scroll::Errors::InvalidCursorError.new(cursor: value)
+          end
+          value = parse_field_value(
+            options[:field_type],
+            options[:field_name],
+            parts[0...-1].join(':')
+          )
+          options[:tiebreak_id] = BSON::ObjectId.from_string(parts[-1])
+          super value, options
+        else
+          super nil, options
         end
       end
 
       def to_s
-        tiebreak_id ? [Mongoid::Scroll::Cursor.transform_field_value(field_type, field_name, value), tiebreak_id].join(':') : nil
-      end
-
-      private
-
-      def parse(value)
-        return unless value
-        parts = value.split(':')
-        unless parts.length >= 2
-          raise Mongoid::Scroll::Errors::InvalidCursorError.new(cursor: value)
-        end
-        id = parts[-1]
-        value = parts[0...-1].join(':')
-        @value = Mongoid::Scroll::Cursor.parse_field_value(field_type, field_name, value)
-        @tiebreak_id = if Mongoid::Compatibility::Version.mongoid3?
-                         Moped::BSON::ObjectId(id)
-                       else
-                         BSON::ObjectId.from_string(id)
-                       end
-      end
-
-      class << self
-        def extract_field_options(options)
-          if options && (field_name = options[:field_name]) && (field_type = options[:field_type])
-            [field_type.to_s, field_name.to_s]
-          elsif options && (field = options[:field])
-            [field.type.to_s, field.name.to_s]
-          else
-            raise ArgumentError.new 'Missing options[:field_name] and/or options[:field_type].'
-          end
-        end
-
-        def parse_field_value(field_type, field_name, value)
-          case field_type.to_s
-          when 'BSON::ObjectId', 'Moped::BSON::ObjectId' then value
-          when 'String' then value.to_s
-          when 'DateTime' then value.is_a?(DateTime) ? value : Time.at(value.to_i).to_datetime
-          when 'Time' then value.is_a?(Time) ? value : Time.at(value.to_i)
-          when 'Date' then value.is_a?(Date) ? value : Time.at(value.to_i).utc.to_date
-          when 'Float' then value.to_f
-          when 'Integer' then value.to_i
-          else
-            raise Mongoid::Scroll::Errors::UnsupportedFieldTypeError.new(field: field_name, type: field_type)
-          end
-        end
-
-        def transform_field_value(field_type, field_name, value)
-          case field_type.to_s
-          when 'BSON::ObjectId', 'Moped::BSON::ObjectId' then value
-          when 'String' then value.to_s
-          when 'Date' then Time.utc(value.year, value.month, value.day).to_i
-          when 'DateTime', 'Time' then value.utc.to_i
-          when 'Float' then value.to_f
-          when 'Integer' then value.to_i
-          else
-            raise Mongoid::Scroll::Errors::UnsupportedFieldTypeError.new(field: field_name, type: field_type)
-          end
-        end
+        tiebreak_id ? [
+          transform_field_value(
+            field_type,
+            field_name,
+            value
+          ), tiebreak_id
+        ].join(':') : nil
       end
     end
   end

data/lib/mongoid/scroll/errors/base.rb

@@ -38,7 +38,7 @@ module Mongoid
         #
         # Returns a localized error message string.
         def translate(key, options)
-          ::I18n.translate("#{BASE_KEY}.#{key}", { locale: :en }.merge(options)).strip
+          ::I18n.translate("#{BASE_KEY}.#{key}", **{ locale: :en }.merge(options)).strip
         end
 
         # Create the problem.

data/lib/mongoid/scroll/errors/invalid_base64_cursor_error.rb

@@ -0,0 +1,11 @@
+module Mongoid
+  module Scroll
+    module Errors
+      class InvalidBase64CursorError < InvalidBaseCursorError
+        def initialize(opts = {})
+          super(compose_message('invalid_base64_cursor', opts))
+        end
+      end
+    end
+  end
+end

data/lib/mongoid/scroll/errors/invalid_base_cursor_error.rb

@@ -0,0 +1,8 @@
+module Mongoid
+  module Scroll
+    module Errors
+      class InvalidBaseCursorError < Mongoid::Scroll::Errors::Base
+      end
+    end
+  end
+end

data/lib/mongoid/scroll/errors/invalid_cursor_error.rb

@@ -1,7 +1,7 @@
 module Mongoid
   module Scroll
     module Errors
-      class InvalidCursorError < Mongoid::Scroll::Errors::Base
+      class InvalidCursorError < InvalidBaseCursorError
         def initialize(opts = {})
           super(compose_message('invalid_cursor', opts))
         end

data/lib/mongoid/scroll/errors/mismatched_sort_fields_error.rb

@@ -0,0 +1,15 @@
+module Mongoid
+  module Scroll
+    module Errors
+      # Raised when the original sort params and the cursor sort params are different
+      class MismatchedSortFieldsError < Mongoid::Scroll::Errors::Base
+        def initialize(opts = {})
+          if opts[:diff] && opts[:diff].is_a?(Hash)
+            opts = opts.merge(diff: opts[:diff].keys.join(', '))
+          end
+          super(compose_message('mismatched_sort_fields', opts))
+        end
+      end
+    end
+  end
+end

data/lib/mongoid/scroll/errors.rb

@@ -1,5 +1,8 @@
 require 'mongoid/scroll/errors/base'
 require 'mongoid/scroll/errors/multiple_sort_fields_error'
+require 'mongoid/scroll/errors/mismatched_sort_fields_error'
+require 'mongoid/scroll/errors/invalid_base_cursor_error'
 require 'mongoid/scroll/errors/invalid_cursor_error'
+require 'mongoid/scroll/errors/invalid_base64_cursor_error'
 require 'mongoid/scroll/errors/no_such_field_error'
 require 'mongoid/scroll/errors/unsupported_field_type_error'

data/lib/mongoid/scroll/version.rb

@@ -1,5 +1,5 @@
 module Mongoid
   module Scroll
-    VERSION = '0.3.6'.freeze
+    VERSION = '1.0.0'.freeze
   end
 end

data/lib/mongoid-scroll.rb

@@ -6,7 +6,10 @@ require 'mongoid'
 require 'mongoid-compatibility'
 require 'mongoid/scroll/version'
 require 'mongoid/scroll/errors'
+require 'mongoid/scroll/base_cursor'
 require 'mongoid/scroll/cursor'
-require 'moped/scrollable' if Object.const_defined?(:Moped)
+require 'mongoid/scroll/base64_encoded_cursor'
+require 'mongoid/criteria/scrollable/fields'
+require 'mongoid/criteria/scrollable/cursors'
 require 'mongo/scrollable' if Object.const_defined?(:Mongo)
 require 'mongoid/criteria/scrollable'

data/spec/mongo/collection_view_spec.rb

@@ -2,123 +2,140 @@ require 'spec_helper'
 
 if Object.const_defined?(:Mongo)
   describe Mongo::Collection::View do
-    context 'scrollable' do
-      subject do
-        Mongoid.default_client['feed_items'].find
-      end
-      it ':scroll' do
-        expect(subject).to respond_to(:scroll)
-      end
-    end
-    context 'with multiple sort fields' do
-      subject do
-        Mongoid.default_client['feed_items'].find.sort(name: 1, value: -1)
-      end
-      it 'raises Mongoid::Scroll::Errors::MultipleSortFieldsError' do
-        expect { subject.scroll }.to raise_error Mongoid::Scroll::Errors::MultipleSortFieldsError,
-                                                 /You're attempting to scroll over data with a sort order that includes multiple fields: name, value./
-      end
-    end
-    context 'with no sort' do
-      subject do
-        Mongoid.default_client['feed_items'].find
-      end
-      it 'adds a default sort by _id' do
-        expect(subject.scroll.sort).to eq('_id' => 1)
-      end
-    end
-    context 'with data' do
-      before :each do
-        10.times do |i|
-          Mongoid.default_client['feed_items'].insert_one(
-            a_string: i.to_s,
-            a_integer: i,
-            a_datetime: DateTime.mongoize(DateTime.new(2013, i + 1, 21, 1, 42, 3, 'UTC')),
-            a_date: Date.mongoize(Date.new(2013, i + 1, 21)),
-            a_time: Time.mongoize(Time.at(Time.now.to_i + i))
-          )
+    [Mongoid::Scroll::Cursor, Mongoid::Scroll::Base64EncodedCursor].each do |cursor_type|
+      context cursor_type do
+        context 'scrollable' do
+          subject do
+            Mongoid.default_client['feed_items'].find
+          end
+          it ':scroll' do
+            expect(subject).to respond_to(:scroll)
+          end
         end
-      end
-      context 'default' do
-        it 'scrolls all' do
-          records = []
-          Mongoid.default_client['feed_items'].find.scroll do |record, _next_cursor|
-            records << record
+        context 'with multiple sort fields' do
+          subject do
+            Mongoid.default_client['feed_items'].find.sort(name: 1, value: -1)
+          end
+          it 'raises Mongoid::Scroll::Errors::MultipleSortFieldsError' do
+            expect { subject.scroll(cursor_type) }.to raise_error Mongoid::Scroll::Errors::MultipleSortFieldsError,
+                                                                  /You're attempting to scroll over data with a sort order that includes multiple fields: name, value./
           end
-          expect(records.size).to eq 10
-          expect(records).to eq Mongoid.default_client['feed_items'].find.to_a
         end
-      end
-      { a_string: String, a_integer: Integer, a_date: Date, a_datetime: DateTime }.each_pair do |field_name, field_type|
-        context field_type do
-          it 'scrolls all with a block' do
-            records = []
-            Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(nil, field_type: field_type) do |record, _next_cursor|
-              records << record
-            end
-            expect(records.size).to eq 10
-            expect(records).to eq Mongoid.default_client['feed_items'].find.to_a
+        context 'with different sort fields between the cursor and the criteria' do
+          subject do
+            Mongoid.default_client['feed_items'].find.sort(name: -1)
           end
-          it 'scrolls all with a break' do
-            records = []
-            cursor = nil
-            Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(5).scroll(nil, field_type: field_type) do |record, next_cursor|
-              records << record
-              cursor = next_cursor
-            end
-            expect(records.size).to eq 5
-            Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(cursor, field_type: field_type) do |record, next_cursor|
-              records << record
-              cursor = next_cursor
+
+          it 'raises Mongoid::Scroll::Errors::MismatchedSortFieldsError' do
+            record = Feed::Item.create!
+            cursor = cursor_type.from_record(record, field: record.fields['a_string'])
+            expect(cursor).to be_a cursor_type
+            error_string = /You're attempting to scroll over data with a sort order that differs between the cursor and the original criteria: field_name, direction./
+            expect { subject.scroll(cursor, field_type: String) }.to raise_error Mongoid::Scroll::Errors::MismatchedSortFieldsError, error_string
+          end
+        end
+        context 'with no sort' do
+          subject do
+            Mongoid.default_client['feed_items'].find
+          end
+          it 'adds a default sort by _id' do
+            expect(subject.scroll(cursor_type).sort).to eq('_id' => 1)
+          end
+        end
+        context 'with data' do
+          before :each do
+            10.times do |i|
+              Mongoid.default_client['feed_items'].insert_one(
+                a_string: i.to_s,
+                a_integer: i,
+                a_datetime: DateTime.mongoize(DateTime.new(2013, i + 1, 21, 1, 42, 3, 'UTC')),
+                a_date: Date.mongoize(Date.new(2013, i + 1, 21)),
+                a_time: Time.mongoize(Time.at(Time.now.to_i + i))
+              )
             end
-            expect(records.size).to eq 10
-            expect(records).to eq Mongoid.default_client['feed_items'].find.to_a
           end
-          it 'scrolls in descending order' do
-            records = []
-            Mongoid.default_client['feed_items'].find.sort(field_name => -1).limit(3).scroll(nil, field_type: field_type, field_name: field_name) do |record, _next_cursor|
-              records << record
+          context 'default' do
+            it 'scrolls all' do
+              records = []
+              Mongoid.default_client['feed_items'].find.scroll(cursor_type) do |record, _next_cursor|
+                records << record
+              end
+              expect(records.size).to eq 10
+              expect(records).to eq Mongoid.default_client['feed_items'].find.to_a
             end
-            expect(records.size).to eq 3
-            expect(records).to eq Mongoid.default_client['feed_items'].find.sort(field_name => -1).limit(3).to_a
           end
-          it 'map' do
-            record = Mongoid.default_client['feed_items'].find.limit(3).scroll(nil, field_type: field_type, field_name: field_name).map { |r| r }.last
-            cursor = Mongoid::Scroll::Cursor.from_record(record, field_type: field_type, field_name: field_name)
-            expect(cursor).to_not be nil
-            expect(cursor.to_s.split(':')).to eq [
-              Mongoid::Scroll::Cursor.transform_field_value(field_type, field_name, record[field_name.to_s]).to_s,
-              record['_id'].to_s
-            ]
+          { a_string: String, a_integer: Integer, a_date: Date, a_datetime: DateTime }.each_pair do |field_name, field_type|
+            context field_type do
+              it 'scrolls all with a block' do
+                records = []
+                Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(cursor_type, field_type: field_type) do |record, _next_cursor|
+                  records << record
+                end
+                expect(records.size).to eq 10
+                expect(records).to eq Mongoid.default_client['feed_items'].find.to_a
+              end
+              it 'scrolls all with a break' do
+                records = []
+                cursor = nil
+                Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(5).scroll(cursor_type, field_type: field_type) do |record, next_cursor|
+                  records << record
+                  cursor = next_cursor
+                  expect(cursor).to be_a cursor_type
+                end
+                expect(records.size).to eq 5
+                Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(cursor, field_type: field_type) do |record, next_cursor|
+                  records << record
+                  cursor = next_cursor
+                  expect(cursor).to be_a cursor_type
+                end
+                expect(records.size).to eq 10
+                expect(records).to eq Mongoid.default_client['feed_items'].find.to_a
+              end
+              it 'scrolls in descending order' do
+                records = []
+                Mongoid.default_client['feed_items'].find.sort(field_name => -1).limit(3).scroll(cursor_type, field_type: field_type, field_name: field_name) do |record, _next_cursor|
+                  records << record
+                end
+                expect(records.size).to eq 3
+                expect(records).to eq Mongoid.default_client['feed_items'].find.sort(field_name => -1).limit(3).to_a
+              end
+              it 'map' do
+                record = Mongoid.default_client['feed_items'].find.limit(3).scroll(cursor_type, field_type: field_type, field_name: field_name).map { |r| r }.last
+                cursor = cursor_type.from_record(record, field_type: field_type, field_name: field_name)
+                expect(cursor).to_not be nil
+                expect(cursor.value).to eq record[field_name.to_s]
+                expect(cursor.tiebreak_id).to eq record['_id']
+              end
+            end
           end
         end
-      end
-    end
-    context 'with overlapping data', if: MongoDB.mmapv1? do
-      before :each do
-        3.times { Feed::Item.create! a_integer: 5 }
-        Feed::Item.first.update_attributes!(name: Array(1000).join('a'))
-      end
-      it 'natural order is different from order by id' do
-        # natural order isn't necessarily going to be the same as _id order
-        # if a document is updated and grows in size, it may need to be relocated and
-        # thus cause the natural order to change
-        expect(Feed::Item.order_by('$natural' => 1).to_a).to_not eq Feed::Item.order_by(_id: 1).to_a
-      end
-      [{ a_integer: 1 }, { a_integer: -1 }].each do |sort_order|
-        it "scrolls by #{sort_order}" do
-          records = []
-          cursor = nil
-          Mongoid.default_client['feed_items'].find.sort(sort_order).limit(2).scroll do |record, next_cursor|
-            records << record
-            cursor = next_cursor
+        context 'with overlapping data', if: MongoDB.mmapv1? do
+          before :each do
+            3.times { Feed::Item.create! a_integer: 5 }
+            Feed::Item.first.update_attributes!(name: Array(1000).join('a'))
+          end
+          it 'natural order is different from order by id' do
+            # natural order isn't necessarily going to be the same as _id order
+            # if a document is updated and grows in size, it may need to be relocated and
+            # thus cause the natural order to change
+            expect(Feed::Item.order_by('$natural' => 1).to_a).to_not eq Feed::Item.order_by(_id: 1).to_a
           end
-          expect(records.size).to eq 2
-          Mongoid.default_client['feed_items'].find.sort(sort_order).scroll(cursor) do |record, _next_cursor|
-            records << record
+          [{ a_integer: 1 }, { a_integer: -1 }].each do |sort_order|
+            it "scrolls by #{sort_order}" do
+              records = []
+              cursor = nil
+              Mongoid.default_client['feed_items'].find.sort(sort_order).limit(2).scroll(cursor_type) do |record, next_cursor|
+                records << record
+                cursor = next_cursor
+              end
+              expect(records.size).to eq 2
+              Mongoid.default_client['feed_items'].find.sort(sort_order).scroll(cursor) do |record, _next_cursor|
+                records << record
+              end
+              expect(records.size).to eq 3
+              expect(records).to eq Mongoid.default_client['feed_items'].find.sort(sort_order.merge(_id: sort_order[:a_integer])).to_a
+            end
           end
-          expect(records.size).to eq 3
-          expect(records).to eq Mongoid.default_client['feed_items'].find.sort(sort_order.merge(_id: sort_order[:a_integer])).to_a
         end
       end
     end