mongoid-scroll 0.3.6 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/ci.yml +36 -0
- data/.github/workflows/danger.yml +17 -0
- data/.rubocop_todo.yml +24 -19
- data/CHANGELOG.md +14 -0
- data/Gemfile +10 -16
- data/LICENSE.md +1 -1
- data/README.md +59 -46
- data/RELEASING.md +4 -4
- data/UPGRADING.md +14 -0
- data/examples/mongoid_scroll_feed.rb +2 -8
- data/lib/config/locales/en.yml +8 -0
- data/lib/mongo/scrollable.rb +8 -3
- data/lib/mongoid/criteria/scrollable/cursors.rb +18 -0
- data/lib/mongoid/criteria/scrollable/fields.rb +21 -0
- data/lib/mongoid/criteria/scrollable.rb +13 -12
- data/lib/mongoid/scroll/base64_encoded_cursor.rb +40 -0
- data/lib/mongoid/scroll/base_cursor.rb +123 -0
- data/lib/mongoid/scroll/cursor.rb +24 -87
- data/lib/mongoid/scroll/errors/base.rb +1 -1
- data/lib/mongoid/scroll/errors/invalid_base64_cursor_error.rb +11 -0
- data/lib/mongoid/scroll/errors/invalid_base_cursor_error.rb +8 -0
- data/lib/mongoid/scroll/errors/invalid_cursor_error.rb +1 -1
- data/lib/mongoid/scroll/errors/mismatched_sort_fields_error.rb +15 -0
- data/lib/mongoid/scroll/errors.rb +3 -0
- data/lib/mongoid/scroll/version.rb +1 -1
- data/lib/mongoid-scroll.rb +4 -1
- data/spec/mongo/collection_view_spec.rb +122 -105
- data/spec/mongoid/base64_encoded_cursor_spec.rb +233 -0
- data/spec/mongoid/criteria_spec.rb +236 -197
- data/spec/mongoid/{scroll_cursor_spec.rb → cursor_spec.rb} +48 -12
- data/spec/spec_helper.rb +1 -1
- data/spec/support/feed/item.rb +1 -1
- metadata +19 -13
- data/.travis.yml +0 -35
- data/examples/moped_scroll_feed.rb +0 -45
- data/lib/moped/scrollable.rb +0 -38
- data/spec/moped/query_spec.rb +0 -126
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 73c44f9f9dcdc554e3f0ab2701cb4c87822bdf905bd7212e6bb59ea66e6a454d
|
|
4
|
+
data.tar.gz: 6e2abaa7d08e7bf0dfb914fc26b51b1d835526c8d0b1bc75fb0577adf06edb8d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ea72567b1edb2d8df27dc309d685804a8f199ff060ad93cb64ef01b6fd098a4ae7f677242375a89875bbdb8f8104f238cb6d79fb667799b5e90c297ce96d0e6f
|
|
7
|
+
data.tar.gz: ab22160efc873bf42430292047661de3b532a0629326dbfdc93a2ef49bc64207725b405472410171e96e6f997e5a23fc5914698b488a4fd1cacb29021bd70261
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
on: [push, pull_request]
|
|
3
|
+
|
|
4
|
+
jobs:
|
|
5
|
+
test:
|
|
6
|
+
runs-on: ubuntu-latest
|
|
7
|
+
strategy:
|
|
8
|
+
matrix:
|
|
9
|
+
entry:
|
|
10
|
+
- { ruby: '2.6', mongodb: '4.4', mongoid: '5' }
|
|
11
|
+
- { ruby: '3.2', mongodb: '6.0', mongoid: '5' }
|
|
12
|
+
- { ruby: '2.7', mongodb: '4.4', mongoid: '6' }
|
|
13
|
+
- { ruby: '2.7', mongodb: '4.4', mongoid: '7' }
|
|
14
|
+
- { ruby: '3.0', mongodb: '4.4', mongoid: '6' }
|
|
15
|
+
- { ruby: '3.0', mongodb: '4.4', mongoid: '7' }
|
|
16
|
+
- { ruby: '3.2', mongodb: '5.0', mongoid: '7' }
|
|
17
|
+
- { ruby: '3.2', mongodb: '6.0', mongoid: '7' }
|
|
18
|
+
- { ruby: '3.1', mongodb: '4.4', mongoid: '8' }
|
|
19
|
+
- { ruby: '3.2', mongodb: '5.0', mongoid: '8' }
|
|
20
|
+
- { ruby: '3.2', mongodb: '6.0', mongoid: '8' }
|
|
21
|
+
name: test (ruby=${{ matrix.entry.ruby }}, mongodb=${{ matrix.entry.mongodb }}), mongoid=${{ matrix.entry.mongoid }})
|
|
22
|
+
env:
|
|
23
|
+
MONGOID_VERSION: ${{ matrix.entry.mongoid }}
|
|
24
|
+
steps:
|
|
25
|
+
- name: Set up MongoDB ${{ matrix.entry.mongodb }}
|
|
26
|
+
uses: supercharge/mongodb-github-action@1.8.0
|
|
27
|
+
with:
|
|
28
|
+
mongodb-version: ${{ matrix.entry.mongodb }}
|
|
29
|
+
- uses: actions/checkout@v3
|
|
30
|
+
- name: Set up Ruby
|
|
31
|
+
uses: ruby/setup-ruby@v1
|
|
32
|
+
with:
|
|
33
|
+
ruby-version: ${{ matrix.entry.ruby }}
|
|
34
|
+
bundler-cache: true
|
|
35
|
+
- name: Run tests
|
|
36
|
+
run: bundle exec rspec
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
name: PR Linter
|
|
2
|
+
on: [pull_request]
|
|
3
|
+
jobs:
|
|
4
|
+
danger:
|
|
5
|
+
runs-on: ubuntu-latest
|
|
6
|
+
steps:
|
|
7
|
+
- uses: actions/checkout@v3
|
|
8
|
+
with:
|
|
9
|
+
fetch-depth: 0
|
|
10
|
+
- uses: ruby/setup-ruby@v1
|
|
11
|
+
with:
|
|
12
|
+
ruby-version: 2.7
|
|
13
|
+
bundler-cache: true
|
|
14
|
+
- run: |
|
|
15
|
+
# Personal access token for dangerpr-bot - public, but base64 encoded to avoid tripping up GitHub
|
|
16
|
+
TOKEN=$(echo -n Z2hwX0xNQ3VmanBFeTBvYkZVTWh6NVNqVFFBOEUxU25abzBqRUVuaAo= | base64 --decode)
|
|
17
|
+
DANGER_GITHUB_API_TOKEN=$TOKEN bundle exec danger --verbose
|
data/.rubocop_todo.yml
CHANGED
|
@@ -1,47 +1,52 @@
|
|
|
1
1
|
# This configuration was generated by
|
|
2
2
|
# `rubocop --auto-gen-config`
|
|
3
|
-
# on
|
|
3
|
+
# on 2023-03-07 08:58:13 -0500 using RuboCop version 0.49.1.
|
|
4
4
|
# The point is for the user to remove these configuration records
|
|
5
5
|
# one by one as the offenses are removed from the code base.
|
|
6
6
|
# Note that changes in the inspected code, or installation of new
|
|
7
7
|
# versions of RuboCop, may require this file to be generated again.
|
|
8
8
|
|
|
9
|
-
# Offense count:
|
|
9
|
+
# Offense count: 5
|
|
10
10
|
# Configuration parameters: Include.
|
|
11
11
|
# Include: **/Gemfile, **/gems.rb
|
|
12
12
|
Bundler/DuplicatedGem:
|
|
13
13
|
Exclude:
|
|
14
14
|
- 'Gemfile'
|
|
15
15
|
|
|
16
|
-
# Offense count:
|
|
16
|
+
# Offense count: 8
|
|
17
17
|
Metrics/AbcSize:
|
|
18
|
-
Max:
|
|
18
|
+
Max: 38
|
|
19
19
|
|
|
20
|
-
# Offense count:
|
|
20
|
+
# Offense count: 18
|
|
21
21
|
# Configuration parameters: CountComments, ExcludedMethods.
|
|
22
22
|
Metrics/BlockLength:
|
|
23
|
-
Max:
|
|
23
|
+
Max: 258
|
|
24
|
+
|
|
25
|
+
# Offense count: 1
|
|
26
|
+
# Configuration parameters: CountComments.
|
|
27
|
+
Metrics/ClassLength:
|
|
28
|
+
Max: 103
|
|
24
29
|
|
|
25
30
|
# Offense count: 5
|
|
26
31
|
Metrics/CyclomaticComplexity:
|
|
27
|
-
Max:
|
|
32
|
+
Max: 13
|
|
28
33
|
|
|
29
|
-
# Offense count:
|
|
34
|
+
# Offense count: 146
|
|
30
35
|
# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
|
31
36
|
# URISchemes: http, https
|
|
32
37
|
Metrics/LineLength:
|
|
33
|
-
Max:
|
|
38
|
+
Max: 252
|
|
34
39
|
|
|
35
|
-
# Offense count:
|
|
40
|
+
# Offense count: 7
|
|
36
41
|
# Configuration parameters: CountComments.
|
|
37
42
|
Metrics/MethodLength:
|
|
38
|
-
Max:
|
|
43
|
+
Max: 23
|
|
39
44
|
|
|
40
|
-
# Offense count:
|
|
45
|
+
# Offense count: 4
|
|
41
46
|
Metrics/PerceivedComplexity:
|
|
42
|
-
Max:
|
|
47
|
+
Max: 12
|
|
43
48
|
|
|
44
|
-
# Offense count:
|
|
49
|
+
# Offense count: 11
|
|
45
50
|
Style/Documentation:
|
|
46
51
|
Exclude:
|
|
47
52
|
- 'spec/**/*'
|
|
@@ -49,13 +54,14 @@ Style/Documentation:
|
|
|
49
54
|
- 'examples/mongoid_scroll_feed.rb'
|
|
50
55
|
- 'lib/mongo/scrollable.rb'
|
|
51
56
|
- 'lib/mongoid/criteria/scrollable.rb'
|
|
57
|
+
- 'lib/mongoid/scroll/base_cursor.rb'
|
|
52
58
|
- 'lib/mongoid/scroll/cursor.rb'
|
|
53
59
|
- 'lib/mongoid/scroll/errors/base.rb'
|
|
60
|
+
- 'lib/mongoid/scroll/errors/invalid_base64_cursor_error.rb'
|
|
54
61
|
- 'lib/mongoid/scroll/errors/invalid_cursor_error.rb'
|
|
55
62
|
- 'lib/mongoid/scroll/errors/multiple_sort_fields_error.rb'
|
|
56
63
|
- 'lib/mongoid/scroll/errors/no_such_field_error.rb'
|
|
57
64
|
- 'lib/mongoid/scroll/errors/unsupported_field_type_error.rb'
|
|
58
|
-
- 'lib/moped/scrollable.rb'
|
|
59
65
|
|
|
60
66
|
# Offense count: 1
|
|
61
67
|
# Configuration parameters: ExpectMatchingDefinition, Regex, IgnoreExecutableScripts, AllowedAcronyms.
|
|
@@ -65,12 +71,11 @@ Style/FileName:
|
|
|
65
71
|
- 'lib/mongoid-scroll.rb'
|
|
66
72
|
|
|
67
73
|
# Offense count: 1
|
|
68
|
-
|
|
69
|
-
Style/GuardClause:
|
|
74
|
+
Style/MultilineTernaryOperator:
|
|
70
75
|
Exclude:
|
|
71
|
-
- 'lib/
|
|
76
|
+
- 'lib/mongoid/scroll/cursor.rb'
|
|
72
77
|
|
|
73
|
-
# Offense count:
|
|
78
|
+
# Offense count: 3
|
|
74
79
|
# Cop supports --auto-correct.
|
|
75
80
|
# Configuration parameters: SupportedStyles.
|
|
76
81
|
# SupportedStyles: compact, exploded
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
### 1.0.0 (2023/03/08)
|
|
2
|
+
|
|
3
|
+
* [#25](https://github.com/mongoid/mongoid-scroll/pull/25): Compatibility with Ruby 3 - [@leamotta](https://github.com/leamotta).
|
|
4
|
+
* [#25](https://github.com/mongoid/mongoid-scroll/pull/25): Replace Travis CI with GHA - [@leamotta](https://github.com/leamotta).
|
|
5
|
+
* [#29](https://github.com/mongoid/mongoid-scroll/pull/29): Add ability to include the current record to the cursor - [@FabienChaynes](https://github.com/FabienChaynes).
|
|
6
|
+
* [#30](https://github.com/mongoid/mongoid-scroll/pull/30): Prevent discrepancy between the original sort and the cursor sort - [@FabienChaynes](https://github.com/FabienChaynes).
|
|
7
|
+
* [#32](https://github.com/mongoid/mongoid-scroll/pull/32): Add Base64 serialization for cursors - [@FabienChaynes](https://github.com/FabienChaynes).
|
|
8
|
+
* [#33](https://github.com/mongoid/mongoid-scroll/pull/33): Removed support for Mongoid 3, 4 and Moped - [@dblock](https://github.com/dblock).
|
|
9
|
+
* [#34](https://github.com/mongoid/mongoid-scroll/pull/34): Added support for Mongoid 8 - [@dblock](https://github.com/dblock).
|
|
10
|
+
|
|
11
|
+
### 0.3.7 (2021/06/01)
|
|
12
|
+
|
|
13
|
+
* [#22](https://github.com/mongoid/mongoid-scroll/pull/22): Compatibility with Mongoid 7 - [@asgerb](https://github.com/asgerb).
|
|
14
|
+
|
|
1
15
|
### 0.3.6 (2018/05/01)
|
|
2
16
|
|
|
3
17
|
* [#18](https://github.com/mongoid/mongoid-scroll/pull/18): Fix mongoid scroll without block returning wrong criteria - [@asgerb](https://github.com/asgerb).
|
data/Gemfile
CHANGED
|
@@ -2,28 +2,22 @@ source 'http://rubygems.org'
|
|
|
2
2
|
|
|
3
3
|
gemspec
|
|
4
4
|
|
|
5
|
-
case version = ENV['MONGOID_VERSION'] || '~>
|
|
6
|
-
when 'HEAD'
|
|
7
|
-
|
|
8
|
-
when /7/
|
|
9
|
-
|
|
10
|
-
when /
|
|
11
|
-
gem '
|
|
12
|
-
when /5/
|
|
5
|
+
case version = ENV['MONGOID_VERSION'] || '~> 7.0'
|
|
6
|
+
when 'HEAD' then gem 'mongoid', github: 'mongodb/mongoid'
|
|
7
|
+
when /8/ then gem 'mongoid', '~> 8.0'
|
|
8
|
+
when /7/ then gem 'mongoid', '~> 7.0'
|
|
9
|
+
when /6/ then gem 'mongoid', '~> 6.0'
|
|
10
|
+
when /5/ then
|
|
11
|
+
gem 'bigdecimal', '1.3.5'
|
|
13
12
|
gem 'mongoid', '~> 5.0'
|
|
14
|
-
|
|
15
|
-
gem 'mongoid', '~> 4.0'
|
|
16
|
-
when /3/
|
|
17
|
-
gem 'mongoid', '~> 3.1'
|
|
18
|
-
else
|
|
19
|
-
gem 'mongoid', version
|
|
13
|
+
else gem 'mongoid', version
|
|
20
14
|
end
|
|
21
15
|
|
|
22
16
|
group :development, :test do
|
|
23
17
|
gem 'bundler'
|
|
24
|
-
gem 'database_cleaner'
|
|
18
|
+
gem 'database_cleaner', '~> 1.8.5'
|
|
25
19
|
gem 'faker'
|
|
26
|
-
gem 'mongoid-danger', '~> 0.
|
|
20
|
+
gem 'mongoid-danger', '~> 0.2.0', require: false
|
|
27
21
|
gem 'rake'
|
|
28
22
|
gem 'rspec', '~> 3.0'
|
|
29
23
|
gem 'rspec-its'
|
data/LICENSE.md
CHANGED
data/README.md
CHANGED
|
@@ -1,27 +1,38 @@
|
|
|
1
|
-
Mongoid::Scroll
|
|
2
|
-
|
|
1
|
+
- [Mongoid::Scroll](#mongoidscroll)
|
|
2
|
+
- [Compatibility](#compatibility)
|
|
3
|
+
- [Demo](#demo)
|
|
4
|
+
- [The Problem](#the-problem)
|
|
5
|
+
- [Installation](#installation)
|
|
6
|
+
- [Usage](#usage)
|
|
7
|
+
- [Mongoid](#mongoid)
|
|
8
|
+
- [Mongo-Ruby-Driver (Mongoid 5)](#mongo-ruby-driver-mongoid-5)
|
|
9
|
+
- [Indexes and Performance](#indexes-and-performance)
|
|
10
|
+
- [Cursors](#cursors)
|
|
11
|
+
- [Standard Cursor](#standard-cursor)
|
|
12
|
+
- [Base64 Encoded Cursor](#base64-encoded-cursor)
|
|
13
|
+
- [Contributing](#contributing)
|
|
14
|
+
- [Copyright and License](#copyright-and-license)
|
|
15
|
+
|
|
16
|
+
# Mongoid::Scroll
|
|
3
17
|
|
|
4
18
|
[](https://badge.fury.io/rb/mongoid-scroll)
|
|
5
|
-
[](https://github.com/mongoid/mongoid-scroll/actions/workflows/ci.yml)
|
|
6
20
|
[](https://gemnasium.com/mongoid/mongoid-scroll)
|
|
7
21
|
[](https://codeclimate.com/github/mongoid/mongoid-scroll)
|
|
8
22
|
|
|
9
|
-
Mongoid extension that enables infinite scrolling for `Mongoid::Criteria
|
|
23
|
+
Mongoid extension that enables infinite scrolling for `Mongoid::Criteria` and `Mongo::Collection::View`.
|
|
10
24
|
|
|
11
|
-
Compatibility
|
|
12
|
-
-------------
|
|
25
|
+
## Compatibility
|
|
13
26
|
|
|
14
|
-
This gem supports Mongoid
|
|
27
|
+
This gem supports Mongoid 5, 6, 7 and 8.
|
|
15
28
|
|
|
16
|
-
Demo
|
|
17
|
-
----
|
|
29
|
+
## Demo
|
|
18
30
|
|
|
19
31
|
Check out [shows on artsy.net](http://artsy.net/shows). Keep scrolling down.
|
|
20
32
|
|
|
21
|
-
There're also two code samples for Mongoid
|
|
33
|
+
There're also two code samples for Mongoid in [examples](examples). Run `bundle exec ruby examples/mongoid_scroll_feed.rb`.
|
|
22
34
|
|
|
23
|
-
The Problem
|
|
24
|
-
-----------
|
|
35
|
+
## The Problem
|
|
25
36
|
|
|
26
37
|
Traditional pagination does not work when data changes between paginated requests, which makes it unsuitable for infinite scroll behaviors.
|
|
27
38
|
|
|
@@ -30,8 +41,7 @@ Traditional pagination does not work when data changes between paginated request
|
|
|
30
41
|
|
|
31
42
|
The solution implemented by the `scroll` extension paginates data using a cursor, giving you the ability to restart pagination where you left it off. This is a non-trivial problem when combined with sorting over non-unique record fields, such as timestamps.
|
|
32
43
|
|
|
33
|
-
Installation
|
|
34
|
-
------------
|
|
44
|
+
## Installation
|
|
35
45
|
|
|
36
46
|
Add the gem to your Gemfile and run `bundle install`.
|
|
37
47
|
|
|
@@ -39,8 +49,7 @@ Add the gem to your Gemfile and run `bundle install`.
|
|
|
39
49
|
gem 'mongoid-scroll'
|
|
40
50
|
```
|
|
41
51
|
|
|
42
|
-
Usage
|
|
43
|
-
-----
|
|
52
|
+
## Usage
|
|
44
53
|
|
|
45
54
|
### Mongoid
|
|
46
55
|
|
|
@@ -84,27 +93,6 @@ Feed::Item.desc(:position).scroll(saved_cursor) do |record, next_cursor|
|
|
|
84
93
|
end
|
|
85
94
|
```
|
|
86
95
|
|
|
87
|
-
### Moped (Mongoid 3 and 4)
|
|
88
|
-
|
|
89
|
-
Scroll a `Moped::Query` and save a cursor to the last item. You must also supply a `field_type` of the sort criteria.
|
|
90
|
-
|
|
91
|
-
```ruby
|
|
92
|
-
saved_cursor = nil
|
|
93
|
-
session[:feed_items].find.sort(position: -1).limit(5).scroll(nil, { field_type: DateTime }) do |record, next_cursor|
|
|
94
|
-
# each record, one-by-one
|
|
95
|
-
saved_cursor = next_cursor
|
|
96
|
-
end
|
|
97
|
-
```
|
|
98
|
-
|
|
99
|
-
Resume iterating using the previously saved cursor.
|
|
100
|
-
|
|
101
|
-
```ruby
|
|
102
|
-
session[:feed_items].find.sort(position: -1).limit(5).scroll(saved_cursor, { field_type: DateTime }) do |record, next_cursor|
|
|
103
|
-
# each record, one-by-one
|
|
104
|
-
saved_cursor = next_cursor
|
|
105
|
-
end
|
|
106
|
-
```
|
|
107
|
-
|
|
108
96
|
### Mongo-Ruby-Driver (Mongoid 5)
|
|
109
97
|
|
|
110
98
|
Scroll a `Mongo::Collection::View` and save a cursor to the last item. You must also supply a `field_type` of the sort criteria.
|
|
@@ -126,8 +114,7 @@ session[:feed_items].find.sort(position: -1).limit(5).scroll(saved_cursor, { fie
|
|
|
126
114
|
end
|
|
127
115
|
```
|
|
128
116
|
|
|
129
|
-
Indexes and Performance
|
|
130
|
-
-----------------------
|
|
117
|
+
## Indexes and Performance
|
|
131
118
|
|
|
132
119
|
A query without a cursor is identical to a query without a scroll.
|
|
133
120
|
|
|
@@ -159,8 +146,7 @@ module Feed
|
|
|
159
146
|
end
|
|
160
147
|
```
|
|
161
148
|
|
|
162
|
-
Cursors
|
|
163
|
-
-------
|
|
149
|
+
## Cursors
|
|
164
150
|
|
|
165
151
|
You can use `Mongoid::Scroll::Cursor.from_record` to generate a cursor. A cursor points at the last record of the previous iteration and unlike MongoDB cursors will not expire.
|
|
166
152
|
|
|
@@ -176,14 +162,41 @@ You can also a `field_name` and `field_type` instead of a Mongoid field.
|
|
|
176
162
|
cursor = Mongoid::Scroll::Cursor.from_record(record, { field_type: DateTime, field_name: "position" })
|
|
177
163
|
```
|
|
178
164
|
|
|
179
|
-
|
|
180
|
-
|
|
165
|
+
When the `include_current` option is set to `true`, the cursor will include the record it points to:
|
|
166
|
+
|
|
167
|
+
```ruby
|
|
168
|
+
record = Feed::Item.desc(:position).limit(3).last
|
|
169
|
+
cursor = Mongoid::Scroll::Cursor.from_record(record, { field: Feed::Item.fields["position"], include_current: true })
|
|
170
|
+
Feed::Item.asc(:position).limit(1).scroll(cursor).first # record
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
If the `field_name`, `field_type` or `direction` options you specify when creating the cursor are different from the original criteria, a `Mongoid::Scroll::Errors::MismatchedSortFieldsError` will be raised.
|
|
174
|
+
|
|
175
|
+
```ruby
|
|
176
|
+
cursor = Mongoid::Scroll::Cursor.from_record(record, { field_type: DateTime, field_name: "position" })
|
|
177
|
+
Feed::Item.desc(:created_at).scroll(cursor) # Raises a Mongoid::Scroll::Errors::MismatchedSortFieldsError
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
### Standard Cursor
|
|
181
|
+
|
|
182
|
+
The `Mongoid::Scroll::Cursor` encodes a value and a tiebreak ID separated by `:`, and does not include other options, such as scroll direction. Take extra care not to pass a cursor into a scroll with different options.
|
|
183
|
+
|
|
184
|
+
### Base64 Encoded Cursor
|
|
185
|
+
|
|
186
|
+
The `Mongoid::Scroll::Base64EncodedCursor` can be used instead of `Mongoid::Scroll::Cursor` to generate a base64-encoded string (using RFC 4648) containing all the information needed to rebuild a cursor.
|
|
187
|
+
|
|
188
|
+
```ruby
|
|
189
|
+
Feed::Item.desc(:position).limit(5).scroll(Mongoid::Scroll::Base64EncodedCursor) do |record, next_cursor|
|
|
190
|
+
# next_cursor is of type Mongoid::Scroll::Base64EncodedCursor
|
|
191
|
+
end
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
## Contributing
|
|
181
195
|
|
|
182
196
|
Fork the project. Make your feature addition or bug fix with tests. Send a pull request. Bonus points for topic branches.
|
|
183
197
|
|
|
184
|
-
Copyright and License
|
|
185
|
-
---------------------
|
|
198
|
+
## Copyright and License
|
|
186
199
|
|
|
187
200
|
MIT License, see [LICENSE](http://github.com/mongoid/mongoid-scroll/raw/master/LICENSE.md) for details.
|
|
188
201
|
|
|
189
|
-
(c) 2013-
|
|
202
|
+
(c) 2013-2023 [Daniel Doubrovkine](http://github.com/dblock), based on code by [Frank Macreery](http://github.com/macreery), [Artsy Inc.](http://artsy.net)
|
data/RELEASING.md
CHANGED
|
@@ -12,12 +12,12 @@ bundle install
|
|
|
12
12
|
rake
|
|
13
13
|
```
|
|
14
14
|
|
|
15
|
-
Check that the last build succeeded in [
|
|
15
|
+
Check that the last build succeeded in [Github Actions](https://github.com/mongoid/mongoid-scroll/actions) for all supported platforms.
|
|
16
16
|
|
|
17
17
|
Increment the version, modify [lib/mongoid/scroll/version.rb](lib/mongoid/scroll/version.rb).
|
|
18
18
|
|
|
19
|
-
* Increment the third number if the release has bug fixes and/or very minor features, only (eg. change `0.
|
|
20
|
-
* Increment the second number if the release contains major features or breaking API changes (eg. change `0.
|
|
19
|
+
* Increment the third number if the release has bug fixes and/or very minor features, only (eg. change `0.4.1` to `0.4.2`).
|
|
20
|
+
* Increment the second number if the release contains major features or breaking API changes (eg. change `0.4.1` to `0.5.0`).
|
|
21
21
|
|
|
22
22
|
Change "Next Release" in [CHANGELOG.md](CHANGELOG.md) to the new version.
|
|
23
23
|
|
|
@@ -51,7 +51,7 @@ Pushed mongoid-scroll 0.4.0 to rubygems.org.
|
|
|
51
51
|
Add the next release to [CHANGELOG.md](CHANGELOG.md).
|
|
52
52
|
|
|
53
53
|
```
|
|
54
|
-
### Next
|
|
54
|
+
### 0.4.1 (Next)
|
|
55
55
|
|
|
56
56
|
* Your contribution here.
|
|
57
57
|
```
|
data/UPGRADING.md
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Upgrading
|
|
2
|
+
|
|
3
|
+
## Upgrading to >= 1.0.0
|
|
4
|
+
|
|
5
|
+
### Mismatched Sort Fields
|
|
6
|
+
|
|
7
|
+
Both `Mongoid::Criteria::Scrollable#scroll` and `Mongo::Scrollable` now raise a `Mongoid::Scroll::Errors::MismatchedSortFieldsError` when there are discrepancies between the cursor sort options and the original sort options.
|
|
8
|
+
|
|
9
|
+
For example, the following code will now raise a `MismatchedSortFieldsError` because we set a different field name (`position`) from the `created_at` field used to sort in `scroll`.
|
|
10
|
+
|
|
11
|
+
```ruby
|
|
12
|
+
cursor.field_name = "position"
|
|
13
|
+
Feed::Item.desc(:created_at).scroll(cursor)
|
|
14
|
+
```
|
|
@@ -4,14 +4,8 @@ Bundler.setup(:default, :development)
|
|
|
4
4
|
require 'mongoid-scroll'
|
|
5
5
|
require 'faker'
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
Moped.logger.level = Logger::DEBUG
|
|
10
|
-
else
|
|
11
|
-
Mongoid.logger.level = Logger::INFO
|
|
12
|
-
Mongo::Logger.logger.level = Logger::INFO if Mongoid::Compatibility::Version.mongoid5?
|
|
13
|
-
end
|
|
14
|
-
|
|
7
|
+
Mongoid.logger.level = Logger::INFO
|
|
8
|
+
Mongo::Logger.logger.level = Logger::INFO if Mongoid::Compatibility::Version.mongoid5?
|
|
15
9
|
Mongoid.connect_to 'mongoid_scroll_demo'
|
|
16
10
|
Mongoid.purge!
|
|
17
11
|
|
data/lib/config/locales/en.yml
CHANGED
|
@@ -7,10 +7,18 @@ en:
|
|
|
7
7
|
message: "Scrolling over a criteria with multiple fields is not supported."
|
|
8
8
|
summary: "You're attempting to scroll over data with a sort order that includes multiple fields: %{sort}."
|
|
9
9
|
resolution: "Simplify the sort order to a single field."
|
|
10
|
+
mismatched_sort_fields:
|
|
11
|
+
message: "Specifying different sort fields than the original sort is not supported."
|
|
12
|
+
summary: "You're attempting to scroll over data with a sort order that differs between the cursor and the original criteria: %{diff}."
|
|
13
|
+
resolution: "Don't update the cursor sort options."
|
|
10
14
|
invalid_cursor:
|
|
11
15
|
message: "The cursor supplied is invalid."
|
|
12
16
|
summary: "The cursor supplied is invalid: %{cursor}."
|
|
13
17
|
resolution: "Cursors must be in the form 'value:tiebreak_id'."
|
|
18
|
+
invalid_base64_cursor:
|
|
19
|
+
message: "The cursor supplied is invalid."
|
|
20
|
+
summary: "The cursor supplied is invalid: %{cursor}."
|
|
21
|
+
resolution: "Cursors must be a base64-encoded string."
|
|
14
22
|
no_such_field:
|
|
15
23
|
message: "Invalid field."
|
|
16
24
|
summary: "The field supplied in the cursor does not exist: %{field}."
|
data/lib/mongo/scrollable.rb
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
module Mongo
|
|
2
2
|
module Scrollable
|
|
3
|
-
|
|
3
|
+
include Mongoid::Criteria::Scrollable::Fields
|
|
4
|
+
include Mongoid::Criteria::Scrollable::Cursors
|
|
5
|
+
|
|
6
|
+
def scroll(cursor_or_type = nil, options = nil, &_block)
|
|
7
|
+
cursor, cursor_type = cursor_and_type(cursor_or_type)
|
|
4
8
|
view = self
|
|
5
9
|
# we don't support scrolling over a view with multiple fields
|
|
6
10
|
raise Mongoid::Scroll::Errors::MultipleSortFieldsError.new(sort: view.sort) if view.sort && view.sort.keys.size != 1
|
|
@@ -10,7 +14,8 @@ module Mongo
|
|
|
10
14
|
# scroll cursor from the parameter, with value and tiebreak_id
|
|
11
15
|
options = { field_type: BSON::ObjectId } unless options
|
|
12
16
|
cursor_options = { field_name: scroll_field, direction: scroll_direction }.merge(options)
|
|
13
|
-
cursor = cursor.is_a?(
|
|
17
|
+
cursor = cursor && cursor.is_a?(cursor_type) ? cursor : cursor_type.new(cursor, cursor_options)
|
|
18
|
+
raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options)
|
|
14
19
|
# make a view
|
|
15
20
|
view = Mongo::Collection::View.new(
|
|
16
21
|
view.collection,
|
|
@@ -22,7 +27,7 @@ module Mongo
|
|
|
22
27
|
# scroll
|
|
23
28
|
if block_given?
|
|
24
29
|
view.each do |record|
|
|
25
|
-
yield record,
|
|
30
|
+
yield record, cursor_type.from_record(record, cursor_options)
|
|
26
31
|
end
|
|
27
32
|
else
|
|
28
33
|
view
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
module Mongoid
|
|
2
|
+
class Criteria
|
|
3
|
+
module Scrollable
|
|
4
|
+
# Shared by *::Scrollable modules
|
|
5
|
+
module Cursors
|
|
6
|
+
private
|
|
7
|
+
|
|
8
|
+
def cursor_and_type(cursor_or_type)
|
|
9
|
+
cursor = cursor_or_type.is_a?(Class) ? nil : cursor_or_type
|
|
10
|
+
cursor_type = cursor_or_type.is_a?(Class) ? cursor_or_type : nil
|
|
11
|
+
cursor_type ||= cursor.class if cursor.is_a?(Mongoid::Scroll::BaseCursor)
|
|
12
|
+
cursor_type ||= Mongoid::Scroll::Cursor
|
|
13
|
+
[cursor, cursor_type]
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
module Mongoid
|
|
2
|
+
class Criteria
|
|
3
|
+
module Scrollable
|
|
4
|
+
# Shared by *::Scrollable modules
|
|
5
|
+
module Fields
|
|
6
|
+
private
|
|
7
|
+
|
|
8
|
+
def raise_mismatched_sort_fields_error!(cursor, criteria_cursor_options)
|
|
9
|
+
diff = cursor.sort_options.reject { |k, v| criteria_cursor_options[k] == v }
|
|
10
|
+
raise Mongoid::Scroll::Errors::MismatchedSortFieldsError.new(diff: diff)
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def different_sort_fields?(cursor, criteria_cursor_options)
|
|
14
|
+
criteria_cursor_options[:field_type] = criteria_cursor_options[:field_type].to_s
|
|
15
|
+
criteria_cursor_options[:field_name] = criteria_cursor_options[:field_name].to_s
|
|
16
|
+
criteria_cursor_options != cursor.sort_options
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
module Mongoid
|
|
2
2
|
class Criteria
|
|
3
3
|
module Scrollable
|
|
4
|
-
|
|
4
|
+
include Mongoid::Criteria::Scrollable::Fields
|
|
5
|
+
include Mongoid::Criteria::Scrollable::Cursors
|
|
6
|
+
|
|
7
|
+
def scroll(cursor_or_type = nil, &_block)
|
|
8
|
+
cursor, cursor_type = cursor_and_type(cursor_or_type)
|
|
5
9
|
raise_multiple_sort_fields_error if multiple_sort_fields?
|
|
6
10
|
criteria = dup
|
|
7
11
|
criteria.merge!(default_sort) if no_sort_option?
|
|
8
12
|
cursor_options = build_cursor_options(criteria)
|
|
9
|
-
cursor = cursor.is_a?(
|
|
13
|
+
cursor = cursor.is_a?(cursor_type) ? cursor : new_cursor(cursor_type, cursor, cursor_options)
|
|
14
|
+
raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options)
|
|
10
15
|
cursor_criteria = build_cursor_criteria(criteria, cursor)
|
|
11
16
|
if block_given?
|
|
12
17
|
cursor_criteria.order_by(_id: scroll_direction(criteria)).each do |record|
|
|
13
|
-
yield record, cursor_from_record(record, cursor_options)
|
|
18
|
+
yield record, cursor_from_record(cursor_type, record, cursor_options)
|
|
14
19
|
end
|
|
15
20
|
else
|
|
16
21
|
cursor_criteria
|
|
@@ -51,8 +56,8 @@ module Mongoid
|
|
|
51
56
|
}
|
|
52
57
|
end
|
|
53
58
|
|
|
54
|
-
def new_cursor(cursor, cursor_options)
|
|
55
|
-
|
|
59
|
+
def new_cursor(cursor_type, cursor, cursor_options)
|
|
60
|
+
cursor_type.new(cursor, cursor_options)
|
|
56
61
|
end
|
|
57
62
|
|
|
58
63
|
def build_cursor_criteria(criteria, cursor)
|
|
@@ -61,18 +66,14 @@ module Mongoid
|
|
|
61
66
|
cursor_criteria
|
|
62
67
|
end
|
|
63
68
|
|
|
64
|
-
def cursor_from_record(record, cursor_options)
|
|
65
|
-
|
|
69
|
+
def cursor_from_record(cursor_type, record, cursor_options)
|
|
70
|
+
cursor_type.from_record(record, cursor_options)
|
|
66
71
|
end
|
|
67
72
|
|
|
68
73
|
def scroll_field_type(criteria)
|
|
69
74
|
scroll_field = scroll_field(criteria)
|
|
70
75
|
field = criteria.klass.fields[scroll_field.to_s]
|
|
71
|
-
field.foreign_key? && field.object_id_field? ?
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
def bson_type
|
|
75
|
-
Mongoid::Compatibility::Version.mongoid3? ? Moped::BSON::ObjectId : BSON::ObjectId
|
|
76
|
+
field.foreign_key? && field.object_id_field? ? BSON::ObjectId : field.type
|
|
76
77
|
end
|
|
77
78
|
end
|
|
78
79
|
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
require 'base64'
|
|
2
|
+
require 'json'
|
|
3
|
+
|
|
4
|
+
module Mongoid
|
|
5
|
+
module Scroll
|
|
6
|
+
# Allows to serializer/deserialize the cursor using RFC 4648
|
|
7
|
+
class Base64EncodedCursor < BaseCursor
|
|
8
|
+
def initialize(value, options = {})
|
|
9
|
+
options = extract_field_options(options)
|
|
10
|
+
if value
|
|
11
|
+
begin
|
|
12
|
+
parsed = ::JSON.parse(::Base64.strict_decode64(value))
|
|
13
|
+
rescue
|
|
14
|
+
raise Mongoid::Scroll::Errors::InvalidBase64CursorError.new(cursor: value)
|
|
15
|
+
end
|
|
16
|
+
super parse_field_value(parsed['field_type'], parsed['field_name'], parsed['value']), {
|
|
17
|
+
field_type: parsed['field_type'],
|
|
18
|
+
field_name: parsed['field_name'],
|
|
19
|
+
direction: parsed['direction'],
|
|
20
|
+
include_current: parsed['include_current'],
|
|
21
|
+
tiebreak_id: parsed['tiebreak_id'] && !parsed['tiebreak_id'].empty? ? BSON::ObjectId.from_string(parsed['tiebreak_id']) : nil
|
|
22
|
+
}
|
|
23
|
+
else
|
|
24
|
+
super nil, options
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def to_s
|
|
29
|
+
Base64.strict_encode64({
|
|
30
|
+
value: transform_field_value(field_type, field_name, value),
|
|
31
|
+
field_type: field_type.to_s,
|
|
32
|
+
field_name: field_name,
|
|
33
|
+
direction: direction,
|
|
34
|
+
include_current: include_current,
|
|
35
|
+
tiebreak_id: tiebreak_id && tiebreak_id.to_s
|
|
36
|
+
}.to_json)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|