mongoid-scroll 1.0.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +20 -2
- data/.gitignore +1 -0
- data/.rubocop.yml +5 -0
- data/.rubocop_todo.yml +153 -35
- data/CHANGELOG.md +15 -5
- data/Gemfile +4 -4
- data/README.md +24 -36
- data/RELEASING.md +1 -4
- data/Rakefile +3 -3
- data/UPGRADING.md +21 -1
- data/examples/{mongoid_scroll_feed.rb → feed.rb} +3 -3
- data/lib/config/locales/en.yml +4 -1
- data/lib/mongoid/criteria/scrollable/iterator.rb +19 -0
- data/lib/mongoid/criteria/scrollable.rb +28 -8
- data/lib/mongoid/scroll/base64_encoded_cursor.rb +8 -6
- data/lib/mongoid/scroll/base_cursor.rb +12 -11
- data/lib/mongoid/scroll/cursor.rb +5 -5
- data/lib/mongoid/scroll/errors/base.rb +4 -4
- data/lib/mongoid/scroll/errors/mismatched_sort_fields_error.rb +1 -3
- data/lib/mongoid/scroll/errors/multiple_sort_fields_error.rb +1 -3
- data/lib/mongoid/scroll/errors/unsupported_type_error.rb +11 -0
- data/lib/mongoid/scroll/errors.rb +1 -0
- data/lib/mongoid/scroll/version.rb +1 -1
- data/lib/mongoid-scroll.rb +1 -2
- data/mongoid-scroll.gemspec +3 -3
- data/spec/mongoid/base64_encoded_cursor_spec.rb +148 -77
- data/spec/mongoid/criteria_spec.rb +108 -37
- data/spec/mongoid/cursor_spec.rb +112 -63
- data/spec/mongoid/scroll_spec.rb +1 -1
- data/spec/spec_helper.rb +9 -2
- data/spec/support/feed/embedded_item.rb +2 -0
- data/spec/support/feed/item.rb +1 -1
- metadata +12 -26
- data/examples/mongo_ruby_driver_scroll_feed.rb +0 -45
- data/lib/mongo/scrollable.rb +0 -39
- data/spec/mongo/collection_view_spec.rb +0 -143
data/spec/support/feed/item.rb
CHANGED
@@ -13,7 +13,7 @@ module Feed
|
|
13
13
|
embeds_many :embedded_items, class_name: 'Feed::EmbeddedItem'
|
14
14
|
|
15
15
|
publisher_options = { class_name: 'Feed::Publisher' }
|
16
|
-
publisher_options[:optional] = true
|
16
|
+
publisher_options[:optional] = true
|
17
17
|
belongs_to :publisher, publisher_options
|
18
18
|
end
|
19
19
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mongoid-scroll
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Doubrovkine
|
@@ -9,24 +9,10 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2024-09-07 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
|
-
name:
|
16
|
-
requirement: !ruby/object:Gem::Requirement
|
17
|
-
requirements:
|
18
|
-
- - ">="
|
19
|
-
- !ruby/object:Gem::Version
|
20
|
-
version: '3.0'
|
21
|
-
type: :runtime
|
22
|
-
prerelease: false
|
23
|
-
version_requirements: !ruby/object:Gem::Requirement
|
24
|
-
requirements:
|
25
|
-
- - ">="
|
26
|
-
- !ruby/object:Gem::Version
|
27
|
-
version: '3.0'
|
28
|
-
- !ruby/object:Gem::Dependency
|
29
|
-
name: mongoid-compatibility
|
15
|
+
name: i18n
|
30
16
|
requirement: !ruby/object:Gem::Requirement
|
31
17
|
requirements:
|
32
18
|
- - ">="
|
@@ -40,19 +26,19 @@ dependencies:
|
|
40
26
|
- !ruby/object:Gem::Version
|
41
27
|
version: '0'
|
42
28
|
- !ruby/object:Gem::Dependency
|
43
|
-
name:
|
29
|
+
name: mongoid
|
44
30
|
requirement: !ruby/object:Gem::Requirement
|
45
31
|
requirements:
|
46
32
|
- - ">="
|
47
33
|
- !ruby/object:Gem::Version
|
48
|
-
version: '0'
|
34
|
+
version: '6.0'
|
49
35
|
type: :runtime
|
50
36
|
prerelease: false
|
51
37
|
version_requirements: !ruby/object:Gem::Requirement
|
52
38
|
requirements:
|
53
39
|
- - ">="
|
54
40
|
- !ruby/object:Gem::Version
|
55
|
-
version: '0'
|
41
|
+
version: '6.0'
|
56
42
|
description:
|
57
43
|
email: dblock@dblock.org
|
58
44
|
executables: []
|
@@ -73,14 +59,13 @@ files:
|
|
73
59
|
- RELEASING.md
|
74
60
|
- Rakefile
|
75
61
|
- UPGRADING.md
|
76
|
-
- examples/
|
77
|
-
- examples/mongoid_scroll_feed.rb
|
62
|
+
- examples/feed.rb
|
78
63
|
- lib/config/locales/en.yml
|
79
|
-
- lib/mongo/scrollable.rb
|
80
64
|
- lib/mongoid-scroll.rb
|
81
65
|
- lib/mongoid/criteria/scrollable.rb
|
82
66
|
- lib/mongoid/criteria/scrollable/cursors.rb
|
83
67
|
- lib/mongoid/criteria/scrollable/fields.rb
|
68
|
+
- lib/mongoid/criteria/scrollable/iterator.rb
|
84
69
|
- lib/mongoid/scroll/base64_encoded_cursor.rb
|
85
70
|
- lib/mongoid/scroll/base_cursor.rb
|
86
71
|
- lib/mongoid/scroll/cursor.rb
|
@@ -93,10 +78,10 @@ files:
|
|
93
78
|
- lib/mongoid/scroll/errors/multiple_sort_fields_error.rb
|
94
79
|
- lib/mongoid/scroll/errors/no_such_field_error.rb
|
95
80
|
- lib/mongoid/scroll/errors/unsupported_field_type_error.rb
|
81
|
+
- lib/mongoid/scroll/errors/unsupported_type_error.rb
|
96
82
|
- lib/mongoid/scroll/version.rb
|
97
83
|
- lib/mongoid_scroll.rb
|
98
84
|
- mongoid-scroll.gemspec
|
99
|
-
- spec/mongo/collection_view_spec.rb
|
100
85
|
- spec/mongoid/base64_encoded_cursor_spec.rb
|
101
86
|
- spec/mongoid/criteria_spec.rb
|
102
87
|
- spec/mongoid/cursor_spec.rb
|
@@ -109,7 +94,8 @@ files:
|
|
109
94
|
homepage: http://github.com/mongoid/mongoid-scroll
|
110
95
|
licenses:
|
111
96
|
- MIT
|
112
|
-
metadata:
|
97
|
+
metadata:
|
98
|
+
rubygems_mfa_required: 'true'
|
113
99
|
post_install_message:
|
114
100
|
rdoc_options: []
|
115
101
|
require_paths:
|
@@ -125,7 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
125
111
|
- !ruby/object:Gem::Version
|
126
112
|
version: 1.3.6
|
127
113
|
requirements: []
|
128
|
-
rubygems_version: 3.
|
114
|
+
rubygems_version: 3.4.6
|
129
115
|
signing_key:
|
130
116
|
specification_version: 4
|
131
117
|
summary: Mongoid extensions to enable infinite scroll.
|
@@ -1,45 +0,0 @@
|
|
1
|
-
require 'bundler'
|
2
|
-
Bundler.setup(:default, :development)
|
3
|
-
|
4
|
-
require 'mongoid-scroll'
|
5
|
-
require 'faker'
|
6
|
-
|
7
|
-
raise 'No Mongo Ruby Driver' unless Object.const_defined?(:Mongo)
|
8
|
-
|
9
|
-
Mongo::Logger.logger = Logger.new($stdout)
|
10
|
-
Mongo::Logger.logger.level = Logger::INFO
|
11
|
-
|
12
|
-
Mongoid.connect_to 'mongoid_scroll_demo'
|
13
|
-
Mongoid.purge!
|
14
|
-
|
15
|
-
# total items to insert
|
16
|
-
total_items = 20
|
17
|
-
# a MongoDB query will be executed every scroll_by items
|
18
|
-
scroll_by = 7
|
19
|
-
|
20
|
-
# insert items with a position out-of-order
|
21
|
-
rands = (0..total_items).to_a.sort { rand }[0..total_items]
|
22
|
-
total_items.times do
|
23
|
-
Mongoid.default_client['feed_items'].insert_one(title: Faker::Lorem.sentence, position: rands.pop)
|
24
|
-
end
|
25
|
-
|
26
|
-
Mongoid.default_client['feed_items'].indexes.create_one(position: 1, _id: 1)
|
27
|
-
|
28
|
-
total_shown = 0
|
29
|
-
next_cursor = nil
|
30
|
-
loop do
|
31
|
-
current_cursor = next_cursor
|
32
|
-
next_cursor = nil
|
33
|
-
Mongoid.default_client['feed_items'].find.limit(scroll_by).sort(position: 1).scroll(current_cursor, field_type: Integer, field_name: 'position') do |item, cursor|
|
34
|
-
puts "#{item['position']}: #{item['title']}"
|
35
|
-
next_cursor = cursor
|
36
|
-
total_shown += 1
|
37
|
-
end
|
38
|
-
break unless next_cursor
|
39
|
-
# destroy an item just for the heck of it, scroll is not affected
|
40
|
-
item = Mongoid.default_client['feed_items'].find.sort(position: 1).first
|
41
|
-
Mongoid.default_client['feed_items'].find(_id: item['_id']).delete_one
|
42
|
-
end
|
43
|
-
|
44
|
-
# this will be 20
|
45
|
-
puts "Shown #{total_shown} items."
|
data/lib/mongo/scrollable.rb
DELETED
@@ -1,39 +0,0 @@
|
|
1
|
-
module Mongo
|
2
|
-
module Scrollable
|
3
|
-
include Mongoid::Criteria::Scrollable::Fields
|
4
|
-
include Mongoid::Criteria::Scrollable::Cursors
|
5
|
-
|
6
|
-
def scroll(cursor_or_type = nil, options = nil, &_block)
|
7
|
-
cursor, cursor_type = cursor_and_type(cursor_or_type)
|
8
|
-
view = self
|
9
|
-
# we don't support scrolling over a view with multiple fields
|
10
|
-
raise Mongoid::Scroll::Errors::MultipleSortFieldsError.new(sort: view.sort) if view.sort && view.sort.keys.size != 1
|
11
|
-
# scroll field and direction
|
12
|
-
scroll_field = view.sort ? view.sort.keys.first : :_id
|
13
|
-
scroll_direction = view.sort ? view.sort.values.first.to_i : 1
|
14
|
-
# scroll cursor from the parameter, with value and tiebreak_id
|
15
|
-
options = { field_type: BSON::ObjectId } unless options
|
16
|
-
cursor_options = { field_name: scroll_field, direction: scroll_direction }.merge(options)
|
17
|
-
cursor = cursor && cursor.is_a?(cursor_type) ? cursor : cursor_type.new(cursor, cursor_options)
|
18
|
-
raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options)
|
19
|
-
# make a view
|
20
|
-
view = Mongo::Collection::View.new(
|
21
|
-
view.collection,
|
22
|
-
view.selector.merge(cursor.criteria),
|
23
|
-
sort: (view.sort || {}).merge(_id: scroll_direction),
|
24
|
-
skip: skip,
|
25
|
-
limit: limit
|
26
|
-
)
|
27
|
-
# scroll
|
28
|
-
if block_given?
|
29
|
-
view.each do |record|
|
30
|
-
yield record, cursor_type.from_record(record, cursor_options)
|
31
|
-
end
|
32
|
-
else
|
33
|
-
view
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
Mongo::Collection::View.send(:include, Mongo::Scrollable)
|
@@ -1,143 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
if Object.const_defined?(:Mongo)
|
4
|
-
describe Mongo::Collection::View do
|
5
|
-
[Mongoid::Scroll::Cursor, Mongoid::Scroll::Base64EncodedCursor].each do |cursor_type|
|
6
|
-
context cursor_type do
|
7
|
-
context 'scrollable' do
|
8
|
-
subject do
|
9
|
-
Mongoid.default_client['feed_items'].find
|
10
|
-
end
|
11
|
-
it ':scroll' do
|
12
|
-
expect(subject).to respond_to(:scroll)
|
13
|
-
end
|
14
|
-
end
|
15
|
-
context 'with multiple sort fields' do
|
16
|
-
subject do
|
17
|
-
Mongoid.default_client['feed_items'].find.sort(name: 1, value: -1)
|
18
|
-
end
|
19
|
-
it 'raises Mongoid::Scroll::Errors::MultipleSortFieldsError' do
|
20
|
-
expect { subject.scroll(cursor_type) }.to raise_error Mongoid::Scroll::Errors::MultipleSortFieldsError,
|
21
|
-
/You're attempting to scroll over data with a sort order that includes multiple fields: name, value./
|
22
|
-
end
|
23
|
-
end
|
24
|
-
context 'with different sort fields between the cursor and the criteria' do
|
25
|
-
subject do
|
26
|
-
Mongoid.default_client['feed_items'].find.sort(name: -1)
|
27
|
-
end
|
28
|
-
|
29
|
-
it 'raises Mongoid::Scroll::Errors::MismatchedSortFieldsError' do
|
30
|
-
record = Feed::Item.create!
|
31
|
-
cursor = cursor_type.from_record(record, field: record.fields['a_string'])
|
32
|
-
expect(cursor).to be_a cursor_type
|
33
|
-
error_string = /You're attempting to scroll over data with a sort order that differs between the cursor and the original criteria: field_name, direction./
|
34
|
-
expect { subject.scroll(cursor, field_type: String) }.to raise_error Mongoid::Scroll::Errors::MismatchedSortFieldsError, error_string
|
35
|
-
end
|
36
|
-
end
|
37
|
-
context 'with no sort' do
|
38
|
-
subject do
|
39
|
-
Mongoid.default_client['feed_items'].find
|
40
|
-
end
|
41
|
-
it 'adds a default sort by _id' do
|
42
|
-
expect(subject.scroll(cursor_type).sort).to eq('_id' => 1)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
context 'with data' do
|
46
|
-
before :each do
|
47
|
-
10.times do |i|
|
48
|
-
Mongoid.default_client['feed_items'].insert_one(
|
49
|
-
a_string: i.to_s,
|
50
|
-
a_integer: i,
|
51
|
-
a_datetime: DateTime.mongoize(DateTime.new(2013, i + 1, 21, 1, 42, 3, 'UTC')),
|
52
|
-
a_date: Date.mongoize(Date.new(2013, i + 1, 21)),
|
53
|
-
a_time: Time.mongoize(Time.at(Time.now.to_i + i))
|
54
|
-
)
|
55
|
-
end
|
56
|
-
end
|
57
|
-
context 'default' do
|
58
|
-
it 'scrolls all' do
|
59
|
-
records = []
|
60
|
-
Mongoid.default_client['feed_items'].find.scroll(cursor_type) do |record, _next_cursor|
|
61
|
-
records << record
|
62
|
-
end
|
63
|
-
expect(records.size).to eq 10
|
64
|
-
expect(records).to eq Mongoid.default_client['feed_items'].find.to_a
|
65
|
-
end
|
66
|
-
end
|
67
|
-
{ a_string: String, a_integer: Integer, a_date: Date, a_datetime: DateTime }.each_pair do |field_name, field_type|
|
68
|
-
context field_type do
|
69
|
-
it 'scrolls all with a block' do
|
70
|
-
records = []
|
71
|
-
Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(cursor_type, field_type: field_type) do |record, _next_cursor|
|
72
|
-
records << record
|
73
|
-
end
|
74
|
-
expect(records.size).to eq 10
|
75
|
-
expect(records).to eq Mongoid.default_client['feed_items'].find.to_a
|
76
|
-
end
|
77
|
-
it 'scrolls all with a break' do
|
78
|
-
records = []
|
79
|
-
cursor = nil
|
80
|
-
Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(5).scroll(cursor_type, field_type: field_type) do |record, next_cursor|
|
81
|
-
records << record
|
82
|
-
cursor = next_cursor
|
83
|
-
expect(cursor).to be_a cursor_type
|
84
|
-
end
|
85
|
-
expect(records.size).to eq 5
|
86
|
-
Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(cursor, field_type: field_type) do |record, next_cursor|
|
87
|
-
records << record
|
88
|
-
cursor = next_cursor
|
89
|
-
expect(cursor).to be_a cursor_type
|
90
|
-
end
|
91
|
-
expect(records.size).to eq 10
|
92
|
-
expect(records).to eq Mongoid.default_client['feed_items'].find.to_a
|
93
|
-
end
|
94
|
-
it 'scrolls in descending order' do
|
95
|
-
records = []
|
96
|
-
Mongoid.default_client['feed_items'].find.sort(field_name => -1).limit(3).scroll(cursor_type, field_type: field_type, field_name: field_name) do |record, _next_cursor|
|
97
|
-
records << record
|
98
|
-
end
|
99
|
-
expect(records.size).to eq 3
|
100
|
-
expect(records).to eq Mongoid.default_client['feed_items'].find.sort(field_name => -1).limit(3).to_a
|
101
|
-
end
|
102
|
-
it 'map' do
|
103
|
-
record = Mongoid.default_client['feed_items'].find.limit(3).scroll(cursor_type, field_type: field_type, field_name: field_name).map { |r| r }.last
|
104
|
-
cursor = cursor_type.from_record(record, field_type: field_type, field_name: field_name)
|
105
|
-
expect(cursor).to_not be nil
|
106
|
-
expect(cursor.value).to eq record[field_name.to_s]
|
107
|
-
expect(cursor.tiebreak_id).to eq record['_id']
|
108
|
-
end
|
109
|
-
end
|
110
|
-
end
|
111
|
-
end
|
112
|
-
context 'with overlapping data', if: MongoDB.mmapv1? do
|
113
|
-
before :each do
|
114
|
-
3.times { Feed::Item.create! a_integer: 5 }
|
115
|
-
Feed::Item.first.update_attributes!(name: Array(1000).join('a'))
|
116
|
-
end
|
117
|
-
it 'natural order is different from order by id' do
|
118
|
-
# natural order isn't necessarily going to be the same as _id order
|
119
|
-
# if a document is updated and grows in size, it may need to be relocated and
|
120
|
-
# thus cause the natural order to change
|
121
|
-
expect(Feed::Item.order_by('$natural' => 1).to_a).to_not eq Feed::Item.order_by(_id: 1).to_a
|
122
|
-
end
|
123
|
-
[{ a_integer: 1 }, { a_integer: -1 }].each do |sort_order|
|
124
|
-
it "scrolls by #{sort_order}" do
|
125
|
-
records = []
|
126
|
-
cursor = nil
|
127
|
-
Mongoid.default_client['feed_items'].find.sort(sort_order).limit(2).scroll(cursor_type) do |record, next_cursor|
|
128
|
-
records << record
|
129
|
-
cursor = next_cursor
|
130
|
-
end
|
131
|
-
expect(records.size).to eq 2
|
132
|
-
Mongoid.default_client['feed_items'].find.sort(sort_order).scroll(cursor) do |record, _next_cursor|
|
133
|
-
records << record
|
134
|
-
end
|
135
|
-
expect(records.size).to eq 3
|
136
|
-
expect(records).to eq Mongoid.default_client['feed_items'].find.sort(sort_order.merge(_id: sort_order[:a_integer])).to_a
|
137
|
-
end
|
138
|
-
end
|
139
|
-
end
|
140
|
-
end
|
141
|
-
end
|
142
|
-
end
|
143
|
-
end
|