mongoid-scroll 0.3.6 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/ci.yml +36 -0
- data/.github/workflows/danger.yml +17 -0
- data/.rubocop_todo.yml +24 -19
- data/CHANGELOG.md +14 -0
- data/Gemfile +10 -16
- data/LICENSE.md +1 -1
- data/README.md +59 -46
- data/RELEASING.md +4 -4
- data/UPGRADING.md +14 -0
- data/examples/mongoid_scroll_feed.rb +2 -8
- data/lib/config/locales/en.yml +8 -0
- data/lib/mongo/scrollable.rb +8 -3
- data/lib/mongoid/criteria/scrollable/cursors.rb +18 -0
- data/lib/mongoid/criteria/scrollable/fields.rb +21 -0
- data/lib/mongoid/criteria/scrollable.rb +13 -12
- data/lib/mongoid/scroll/base64_encoded_cursor.rb +40 -0
- data/lib/mongoid/scroll/base_cursor.rb +123 -0
- data/lib/mongoid/scroll/cursor.rb +24 -87
- data/lib/mongoid/scroll/errors/base.rb +1 -1
- data/lib/mongoid/scroll/errors/invalid_base64_cursor_error.rb +11 -0
- data/lib/mongoid/scroll/errors/invalid_base_cursor_error.rb +8 -0
- data/lib/mongoid/scroll/errors/invalid_cursor_error.rb +1 -1
- data/lib/mongoid/scroll/errors/mismatched_sort_fields_error.rb +15 -0
- data/lib/mongoid/scroll/errors.rb +3 -0
- data/lib/mongoid/scroll/version.rb +1 -1
- data/lib/mongoid-scroll.rb +4 -1
- data/spec/mongo/collection_view_spec.rb +122 -105
- data/spec/mongoid/base64_encoded_cursor_spec.rb +233 -0
- data/spec/mongoid/criteria_spec.rb +236 -197
- data/spec/mongoid/{scroll_cursor_spec.rb → cursor_spec.rb} +48 -12
- data/spec/spec_helper.rb +1 -1
- data/spec/support/feed/item.rb +1 -1
- metadata +19 -13
- data/.travis.yml +0 -35
- data/examples/moped_scroll_feed.rb +0 -45
- data/lib/moped/scrollable.rb +0 -38
- data/spec/moped/query_spec.rb +0 -126
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 73c44f9f9dcdc554e3f0ab2701cb4c87822bdf905bd7212e6bb59ea66e6a454d
|
|
4
|
+
data.tar.gz: 6e2abaa7d08e7bf0dfb914fc26b51b1d835526c8d0b1bc75fb0577adf06edb8d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ea72567b1edb2d8df27dc309d685804a8f199ff060ad93cb64ef01b6fd098a4ae7f677242375a89875bbdb8f8104f238cb6d79fb667799b5e90c297ce96d0e6f
|
|
7
|
+
data.tar.gz: ab22160efc873bf42430292047661de3b532a0629326dbfdc93a2ef49bc64207725b405472410171e96e6f997e5a23fc5914698b488a4fd1cacb29021bd70261
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
on: [push, pull_request]
|
|
3
|
+
|
|
4
|
+
jobs:
|
|
5
|
+
test:
|
|
6
|
+
runs-on: ubuntu-latest
|
|
7
|
+
strategy:
|
|
8
|
+
matrix:
|
|
9
|
+
entry:
|
|
10
|
+
- { ruby: '2.6', mongodb: '4.4', mongoid: '5' }
|
|
11
|
+
- { ruby: '3.2', mongodb: '6.0', mongoid: '5' }
|
|
12
|
+
- { ruby: '2.7', mongodb: '4.4', mongoid: '6' }
|
|
13
|
+
- { ruby: '2.7', mongodb: '4.4', mongoid: '7' }
|
|
14
|
+
- { ruby: '3.0', mongodb: '4.4', mongoid: '6' }
|
|
15
|
+
- { ruby: '3.0', mongodb: '4.4', mongoid: '7' }
|
|
16
|
+
- { ruby: '3.2', mongodb: '5.0', mongoid: '7' }
|
|
17
|
+
- { ruby: '3.2', mongodb: '6.0', mongoid: '7' }
|
|
18
|
+
- { ruby: '3.1', mongodb: '4.4', mongoid: '8' }
|
|
19
|
+
- { ruby: '3.2', mongodb: '5.0', mongoid: '8' }
|
|
20
|
+
- { ruby: '3.2', mongodb: '6.0', mongoid: '8' }
|
|
21
|
+
name: test (ruby=${{ matrix.entry.ruby }}, mongodb=${{ matrix.entry.mongodb }}), mongoid=${{ matrix.entry.mongoid }})
|
|
22
|
+
env:
|
|
23
|
+
MONGOID_VERSION: ${{ matrix.entry.mongoid }}
|
|
24
|
+
steps:
|
|
25
|
+
- name: Set up MongoDB ${{ matrix.entry.mongodb }}
|
|
26
|
+
uses: supercharge/mongodb-github-action@1.8.0
|
|
27
|
+
with:
|
|
28
|
+
mongodb-version: ${{ matrix.entry.mongodb }}
|
|
29
|
+
- uses: actions/checkout@v3
|
|
30
|
+
- name: Set up Ruby
|
|
31
|
+
uses: ruby/setup-ruby@v1
|
|
32
|
+
with:
|
|
33
|
+
ruby-version: ${{ matrix.entry.ruby }}
|
|
34
|
+
bundler-cache: true
|
|
35
|
+
- name: Run tests
|
|
36
|
+
run: bundle exec rspec
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
name: PR Linter
|
|
2
|
+
on: [pull_request]
|
|
3
|
+
jobs:
|
|
4
|
+
danger:
|
|
5
|
+
runs-on: ubuntu-latest
|
|
6
|
+
steps:
|
|
7
|
+
- uses: actions/checkout@v3
|
|
8
|
+
with:
|
|
9
|
+
fetch-depth: 0
|
|
10
|
+
- uses: ruby/setup-ruby@v1
|
|
11
|
+
with:
|
|
12
|
+
ruby-version: 2.7
|
|
13
|
+
bundler-cache: true
|
|
14
|
+
- run: |
|
|
15
|
+
# Personal access token for dangerpr-bot - public, but base64 encoded to avoid tripping up GitHub
|
|
16
|
+
TOKEN=$(echo -n Z2hwX0xNQ3VmanBFeTBvYkZVTWh6NVNqVFFBOEUxU25abzBqRUVuaAo= | base64 --decode)
|
|
17
|
+
DANGER_GITHUB_API_TOKEN=$TOKEN bundle exec danger --verbose
|
data/.rubocop_todo.yml
CHANGED
|
@@ -1,47 +1,52 @@
|
|
|
1
1
|
# This configuration was generated by
|
|
2
2
|
# `rubocop --auto-gen-config`
|
|
3
|
-
# on
|
|
3
|
+
# on 2023-03-07 08:58:13 -0500 using RuboCop version 0.49.1.
|
|
4
4
|
# The point is for the user to remove these configuration records
|
|
5
5
|
# one by one as the offenses are removed from the code base.
|
|
6
6
|
# Note that changes in the inspected code, or installation of new
|
|
7
7
|
# versions of RuboCop, may require this file to be generated again.
|
|
8
8
|
|
|
9
|
-
# Offense count:
|
|
9
|
+
# Offense count: 5
|
|
10
10
|
# Configuration parameters: Include.
|
|
11
11
|
# Include: **/Gemfile, **/gems.rb
|
|
12
12
|
Bundler/DuplicatedGem:
|
|
13
13
|
Exclude:
|
|
14
14
|
- 'Gemfile'
|
|
15
15
|
|
|
16
|
-
# Offense count:
|
|
16
|
+
# Offense count: 8
|
|
17
17
|
Metrics/AbcSize:
|
|
18
|
-
Max:
|
|
18
|
+
Max: 38
|
|
19
19
|
|
|
20
|
-
# Offense count:
|
|
20
|
+
# Offense count: 18
|
|
21
21
|
# Configuration parameters: CountComments, ExcludedMethods.
|
|
22
22
|
Metrics/BlockLength:
|
|
23
|
-
Max:
|
|
23
|
+
Max: 258
|
|
24
|
+
|
|
25
|
+
# Offense count: 1
|
|
26
|
+
# Configuration parameters: CountComments.
|
|
27
|
+
Metrics/ClassLength:
|
|
28
|
+
Max: 103
|
|
24
29
|
|
|
25
30
|
# Offense count: 5
|
|
26
31
|
Metrics/CyclomaticComplexity:
|
|
27
|
-
Max:
|
|
32
|
+
Max: 13
|
|
28
33
|
|
|
29
|
-
# Offense count:
|
|
34
|
+
# Offense count: 146
|
|
30
35
|
# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
|
31
36
|
# URISchemes: http, https
|
|
32
37
|
Metrics/LineLength:
|
|
33
|
-
Max:
|
|
38
|
+
Max: 252
|
|
34
39
|
|
|
35
|
-
# Offense count:
|
|
40
|
+
# Offense count: 7
|
|
36
41
|
# Configuration parameters: CountComments.
|
|
37
42
|
Metrics/MethodLength:
|
|
38
|
-
Max:
|
|
43
|
+
Max: 23
|
|
39
44
|
|
|
40
|
-
# Offense count:
|
|
45
|
+
# Offense count: 4
|
|
41
46
|
Metrics/PerceivedComplexity:
|
|
42
|
-
Max:
|
|
47
|
+
Max: 12
|
|
43
48
|
|
|
44
|
-
# Offense count:
|
|
49
|
+
# Offense count: 11
|
|
45
50
|
Style/Documentation:
|
|
46
51
|
Exclude:
|
|
47
52
|
- 'spec/**/*'
|
|
@@ -49,13 +54,14 @@ Style/Documentation:
|
|
|
49
54
|
- 'examples/mongoid_scroll_feed.rb'
|
|
50
55
|
- 'lib/mongo/scrollable.rb'
|
|
51
56
|
- 'lib/mongoid/criteria/scrollable.rb'
|
|
57
|
+
- 'lib/mongoid/scroll/base_cursor.rb'
|
|
52
58
|
- 'lib/mongoid/scroll/cursor.rb'
|
|
53
59
|
- 'lib/mongoid/scroll/errors/base.rb'
|
|
60
|
+
- 'lib/mongoid/scroll/errors/invalid_base64_cursor_error.rb'
|
|
54
61
|
- 'lib/mongoid/scroll/errors/invalid_cursor_error.rb'
|
|
55
62
|
- 'lib/mongoid/scroll/errors/multiple_sort_fields_error.rb'
|
|
56
63
|
- 'lib/mongoid/scroll/errors/no_such_field_error.rb'
|
|
57
64
|
- 'lib/mongoid/scroll/errors/unsupported_field_type_error.rb'
|
|
58
|
-
- 'lib/moped/scrollable.rb'
|
|
59
65
|
|
|
60
66
|
# Offense count: 1
|
|
61
67
|
# Configuration parameters: ExpectMatchingDefinition, Regex, IgnoreExecutableScripts, AllowedAcronyms.
|
|
@@ -65,12 +71,11 @@ Style/FileName:
|
|
|
65
71
|
- 'lib/mongoid-scroll.rb'
|
|
66
72
|
|
|
67
73
|
# Offense count: 1
|
|
68
|
-
|
|
69
|
-
Style/GuardClause:
|
|
74
|
+
Style/MultilineTernaryOperator:
|
|
70
75
|
Exclude:
|
|
71
|
-
- 'lib/
|
|
76
|
+
- 'lib/mongoid/scroll/cursor.rb'
|
|
72
77
|
|
|
73
|
-
# Offense count:
|
|
78
|
+
# Offense count: 3
|
|
74
79
|
# Cop supports --auto-correct.
|
|
75
80
|
# Configuration parameters: SupportedStyles.
|
|
76
81
|
# SupportedStyles: compact, exploded
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
### 1.0.0 (2023/03/08)
|
|
2
|
+
|
|
3
|
+
* [#25](https://github.com/mongoid/mongoid-scroll/pull/25): Compatibility with Ruby 3 - [@leamotta](https://github.com/leamotta).
|
|
4
|
+
* [#25](https://github.com/mongoid/mongoid-scroll/pull/25): Replace Travis CI with GHA - [@leamotta](https://github.com/leamotta).
|
|
5
|
+
* [#29](https://github.com/mongoid/mongoid-scroll/pull/29): Add ability to include the current record to the cursor - [@FabienChaynes](https://github.com/FabienChaynes).
|
|
6
|
+
* [#30](https://github.com/mongoid/mongoid-scroll/pull/30): Prevent discrepancy between the original sort and the cursor sort - [@FabienChaynes](https://github.com/FabienChaynes).
|
|
7
|
+
* [#32](https://github.com/mongoid/mongoid-scroll/pull/32): Add Base64 serialization for cursors - [@FabienChaynes](https://github.com/FabienChaynes).
|
|
8
|
+
* [#33](https://github.com/mongoid/mongoid-scroll/pull/33): Removed support for Mongoid 3, 4 and Moped - [@dblock](https://github.com/dblock).
|
|
9
|
+
* [#34](https://github.com/mongoid/mongoid-scroll/pull/34): Added support for Mongoid 8 - [@dblock](https://github.com/dblock).
|
|
10
|
+
|
|
11
|
+
### 0.3.7 (2021/06/01)
|
|
12
|
+
|
|
13
|
+
* [#22](https://github.com/mongoid/mongoid-scroll/pull/22): Compatibility with Mongoid 7 - [@asgerb](https://github.com/asgerb).
|
|
14
|
+
|
|
1
15
|
### 0.3.6 (2018/05/01)
|
|
2
16
|
|
|
3
17
|
* [#18](https://github.com/mongoid/mongoid-scroll/pull/18): Fix mongoid scroll without block returning wrong criteria - [@asgerb](https://github.com/asgerb).
|
data/Gemfile
CHANGED
|
@@ -2,28 +2,22 @@ source 'http://rubygems.org'
|
|
|
2
2
|
|
|
3
3
|
gemspec
|
|
4
4
|
|
|
5
|
-
case version = ENV['MONGOID_VERSION'] || '~>
|
|
6
|
-
when 'HEAD'
|
|
7
|
-
|
|
8
|
-
when /7/
|
|
9
|
-
|
|
10
|
-
when /
|
|
11
|
-
gem '
|
|
12
|
-
when /5/
|
|
5
|
+
case version = ENV['MONGOID_VERSION'] || '~> 7.0'
|
|
6
|
+
when 'HEAD' then gem 'mongoid', github: 'mongodb/mongoid'
|
|
7
|
+
when /8/ then gem 'mongoid', '~> 8.0'
|
|
8
|
+
when /7/ then gem 'mongoid', '~> 7.0'
|
|
9
|
+
when /6/ then gem 'mongoid', '~> 6.0'
|
|
10
|
+
when /5/ then
|
|
11
|
+
gem 'bigdecimal', '1.3.5'
|
|
13
12
|
gem 'mongoid', '~> 5.0'
|
|
14
|
-
|
|
15
|
-
gem 'mongoid', '~> 4.0'
|
|
16
|
-
when /3/
|
|
17
|
-
gem 'mongoid', '~> 3.1'
|
|
18
|
-
else
|
|
19
|
-
gem 'mongoid', version
|
|
13
|
+
else gem 'mongoid', version
|
|
20
14
|
end
|
|
21
15
|
|
|
22
16
|
group :development, :test do
|
|
23
17
|
gem 'bundler'
|
|
24
|
-
gem 'database_cleaner'
|
|
18
|
+
gem 'database_cleaner', '~> 1.8.5'
|
|
25
19
|
gem 'faker'
|
|
26
|
-
gem 'mongoid-danger', '~> 0.
|
|
20
|
+
gem 'mongoid-danger', '~> 0.2.0', require: false
|
|
27
21
|
gem 'rake'
|
|
28
22
|
gem 'rspec', '~> 3.0'
|
|
29
23
|
gem 'rspec-its'
|
data/LICENSE.md
CHANGED
data/README.md
CHANGED
|
@@ -1,27 +1,38 @@
|
|
|
1
|
-
Mongoid::Scroll
|
|
2
|
-
|
|
1
|
+
- [Mongoid::Scroll](#mongoidscroll)
|
|
2
|
+
- [Compatibility](#compatibility)
|
|
3
|
+
- [Demo](#demo)
|
|
4
|
+
- [The Problem](#the-problem)
|
|
5
|
+
- [Installation](#installation)
|
|
6
|
+
- [Usage](#usage)
|
|
7
|
+
- [Mongoid](#mongoid)
|
|
8
|
+
- [Mongo-Ruby-Driver (Mongoid 5)](#mongo-ruby-driver-mongoid-5)
|
|
9
|
+
- [Indexes and Performance](#indexes-and-performance)
|
|
10
|
+
- [Cursors](#cursors)
|
|
11
|
+
- [Standard Cursor](#standard-cursor)
|
|
12
|
+
- [Base64 Encoded Cursor](#base64-encoded-cursor)
|
|
13
|
+
- [Contributing](#contributing)
|
|
14
|
+
- [Copyright and License](#copyright-and-license)
|
|
15
|
+
|
|
16
|
+
# Mongoid::Scroll
|
|
3
17
|
|
|
4
18
|
[](https://badge.fury.io/rb/mongoid-scroll)
|
|
5
|
-
[](https://github.com/mongoid/mongoid-scroll/actions/workflows/ci.yml)
|
|
6
20
|
[](https://gemnasium.com/mongoid/mongoid-scroll)
|
|
7
21
|
[](https://codeclimate.com/github/mongoid/mongoid-scroll)
|
|
8
22
|
|
|
9
|
-
Mongoid extension that enables infinite scrolling for `Mongoid::Criteria
|
|
23
|
+
Mongoid extension that enables infinite scrolling for `Mongoid::Criteria` and `Mongo::Collection::View`.
|
|
10
24
|
|
|
11
|
-
Compatibility
|
|
12
|
-
-------------
|
|
25
|
+
## Compatibility
|
|
13
26
|
|
|
14
|
-
This gem supports Mongoid
|
|
27
|
+
This gem supports Mongoid 5, 6, 7 and 8.
|
|
15
28
|
|
|
16
|
-
Demo
|
|
17
|
-
----
|
|
29
|
+
## Demo
|
|
18
30
|
|
|
19
31
|
Check out [shows on artsy.net](http://artsy.net/shows). Keep scrolling down.
|
|
20
32
|
|
|
21
|
-
There're also two code samples for Mongoid
|
|
33
|
+
There're also two code samples for Mongoid in [examples](examples). Run `bundle exec ruby examples/mongoid_scroll_feed.rb`.
|
|
22
34
|
|
|
23
|
-
The Problem
|
|
24
|
-
-----------
|
|
35
|
+
## The Problem
|
|
25
36
|
|
|
26
37
|
Traditional pagination does not work when data changes between paginated requests, which makes it unsuitable for infinite scroll behaviors.
|
|
27
38
|
|
|
@@ -30,8 +41,7 @@ Traditional pagination does not work when data changes between paginated request
|
|
|
30
41
|
|
|
31
42
|
The solution implemented by the `scroll` extension paginates data using a cursor, giving you the ability to restart pagination where you left it off. This is a non-trivial problem when combined with sorting over non-unique record fields, such as timestamps.
|
|
32
43
|
|
|
33
|
-
Installation
|
|
34
|
-
------------
|
|
44
|
+
## Installation
|
|
35
45
|
|
|
36
46
|
Add the gem to your Gemfile and run `bundle install`.
|
|
37
47
|
|
|
@@ -39,8 +49,7 @@ Add the gem to your Gemfile and run `bundle install`.
|
|
|
39
49
|
gem 'mongoid-scroll'
|
|
40
50
|
```
|
|
41
51
|
|
|
42
|
-
Usage
|
|
43
|
-
-----
|
|
52
|
+
## Usage
|
|
44
53
|
|
|
45
54
|
### Mongoid
|
|
46
55
|
|
|
@@ -84,27 +93,6 @@ Feed::Item.desc(:position).scroll(saved_cursor) do |record, next_cursor|
|
|
|
84
93
|
end
|
|
85
94
|
```
|
|
86
95
|
|
|
87
|
-
### Moped (Mongoid 3 and 4)
|
|
88
|
-
|
|
89
|
-
Scroll a `Moped::Query` and save a cursor to the last item. You must also supply a `field_type` of the sort criteria.
|
|
90
|
-
|
|
91
|
-
```ruby
|
|
92
|
-
saved_cursor = nil
|
|
93
|
-
session[:feed_items].find.sort(position: -1).limit(5).scroll(nil, { field_type: DateTime }) do |record, next_cursor|
|
|
94
|
-
# each record, one-by-one
|
|
95
|
-
saved_cursor = next_cursor
|
|
96
|
-
end
|
|
97
|
-
```
|
|
98
|
-
|
|
99
|
-
Resume iterating using the previously saved cursor.
|
|
100
|
-
|
|
101
|
-
```ruby
|
|
102
|
-
session[:feed_items].find.sort(position: -1).limit(5).scroll(saved_cursor, { field_type: DateTime }) do |record, next_cursor|
|
|
103
|
-
# each record, one-by-one
|
|
104
|
-
saved_cursor = next_cursor
|
|
105
|
-
end
|
|
106
|
-
```
|
|
107
|
-
|
|
108
96
|
### Mongo-Ruby-Driver (Mongoid 5)
|
|
109
97
|
|
|
110
98
|
Scroll a `Mongo::Collection::View` and save a cursor to the last item. You must also supply a `field_type` of the sort criteria.
|
|
@@ -126,8 +114,7 @@ session[:feed_items].find.sort(position: -1).limit(5).scroll(saved_cursor, { fie
|
|
|
126
114
|
end
|
|
127
115
|
```
|
|
128
116
|
|
|
129
|
-
Indexes and Performance
|
|
130
|
-
-----------------------
|
|
117
|
+
## Indexes and Performance
|
|
131
118
|
|
|
132
119
|
A query without a cursor is identical to a query without a scroll.
|
|
133
120
|
|
|
@@ -159,8 +146,7 @@ module Feed
|
|
|
159
146
|
end
|
|
160
147
|
```
|
|
161
148
|
|
|
162
|
-
Cursors
|
|
163
|
-
-------
|
|
149
|
+
## Cursors
|
|
164
150
|
|
|
165
151
|
You can use `Mongoid::Scroll::Cursor.from_record` to generate a cursor. A cursor points at the last record of the previous iteration and unlike MongoDB cursors will not expire.
|
|
166
152
|
|
|
@@ -176,14 +162,41 @@ You can also a `field_name` and `field_type` instead of a Mongoid field.
|
|
|
176
162
|
cursor = Mongoid::Scroll::Cursor.from_record(record, { field_type: DateTime, field_name: "position" })
|
|
177
163
|
```
|
|
178
164
|
|
|
179
|
-
|
|
180
|
-
|
|
165
|
+
When the `include_current` option is set to `true`, the cursor will include the record it points to:
|
|
166
|
+
|
|
167
|
+
```ruby
|
|
168
|
+
record = Feed::Item.desc(:position).limit(3).last
|
|
169
|
+
cursor = Mongoid::Scroll::Cursor.from_record(record, { field: Feed::Item.fields["position"], include_current: true })
|
|
170
|
+
Feed::Item.asc(:position).limit(1).scroll(cursor).first # record
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
If the `field_name`, `field_type` or `direction` options you specify when creating the cursor are different from the original criteria, a `Mongoid::Scroll::Errors::MismatchedSortFieldsError` will be raised.
|
|
174
|
+
|
|
175
|
+
```ruby
|
|
176
|
+
cursor = Mongoid::Scroll::Cursor.from_record(record, { field_type: DateTime, field_name: "position" })
|
|
177
|
+
Feed::Item.desc(:created_at).scroll(cursor) # Raises a Mongoid::Scroll::Errors::MismatchedSortFieldsError
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
### Standard Cursor
|
|
181
|
+
|
|
182
|
+
The `Mongoid::Scroll::Cursor` encodes a value and a tiebreak ID separated by `:`, and does not include other options, such as scroll direction. Take extra care not to pass a cursor into a scroll with different options.
|
|
183
|
+
|
|
184
|
+
### Base64 Encoded Cursor
|
|
185
|
+
|
|
186
|
+
The `Mongoid::Scroll::Base64EncodedCursor` can be used instead of `Mongoid::Scroll::Cursor` to generate a base64-encoded string (using RFC 4648) containing all the information needed to rebuild a cursor.
|
|
187
|
+
|
|
188
|
+
```ruby
|
|
189
|
+
Feed::Item.desc(:position).limit(5).scroll(Mongoid::Scroll::Base64EncodedCursor) do |record, next_cursor|
|
|
190
|
+
# next_cursor is of type Mongoid::Scroll::Base64EncodedCursor
|
|
191
|
+
end
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
## Contributing
|
|
181
195
|
|
|
182
196
|
Fork the project. Make your feature addition or bug fix with tests. Send a pull request. Bonus points for topic branches.
|
|
183
197
|
|
|
184
|
-
Copyright and License
|
|
185
|
-
---------------------
|
|
198
|
+
## Copyright and License
|
|
186
199
|
|
|
187
200
|
MIT License, see [LICENSE](http://github.com/mongoid/mongoid-scroll/raw/master/LICENSE.md) for details.
|
|
188
201
|
|
|
189
|
-
(c) 2013-
|
|
202
|
+
(c) 2013-2023 [Daniel Doubrovkine](http://github.com/dblock), based on code by [Frank Macreery](http://github.com/macreery), [Artsy Inc.](http://artsy.net)
|
data/RELEASING.md
CHANGED
|
@@ -12,12 +12,12 @@ bundle install
|
|
|
12
12
|
rake
|
|
13
13
|
```
|
|
14
14
|
|
|
15
|
-
Check that the last build succeeded in [
|
|
15
|
+
Check that the last build succeeded in [Github Actions](https://github.com/mongoid/mongoid-scroll/actions) for all supported platforms.
|
|
16
16
|
|
|
17
17
|
Increment the version, modify [lib/mongoid/scroll/version.rb](lib/mongoid/scroll/version.rb).
|
|
18
18
|
|
|
19
|
-
* Increment the third number if the release has bug fixes and/or very minor features, only (eg. change `0.
|
|
20
|
-
* Increment the second number if the release contains major features or breaking API changes (eg. change `0.
|
|
19
|
+
* Increment the third number if the release has bug fixes and/or very minor features, only (eg. change `0.4.1` to `0.4.2`).
|
|
20
|
+
* Increment the second number if the release contains major features or breaking API changes (eg. change `0.4.1` to `0.5.0`).
|
|
21
21
|
|
|
22
22
|
Change "Next Release" in [CHANGELOG.md](CHANGELOG.md) to the new version.
|
|
23
23
|
|
|
@@ -51,7 +51,7 @@ Pushed mongoid-scroll 0.4.0 to rubygems.org.
|
|
|
51
51
|
Add the next release to [CHANGELOG.md](CHANGELOG.md).
|
|
52
52
|
|
|
53
53
|
```
|
|
54
|
-
### Next
|
|
54
|
+
### 0.4.1 (Next)
|
|
55
55
|
|
|
56
56
|
* Your contribution here.
|
|
57
57
|
```
|
data/UPGRADING.md
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Upgrading
|
|
2
|
+
|
|
3
|
+
## Upgrading to >= 1.0.0
|
|
4
|
+
|
|
5
|
+
### Mismatched Sort Fields
|
|
6
|
+
|
|
7
|
+
Both `Mongoid::Criteria::Scrollable#scroll` and `Mongo::Scrollable` now raise a `Mongoid::Scroll::Errors::MismatchedSortFieldsError` when there are discrepancies between the cursor sort options and the original sort options.
|
|
8
|
+
|
|
9
|
+
For example, the following code will now raise a `MismatchedSortFieldsError` because we set a different field name (`position`) from the `created_at` field used to sort in `scroll`.
|
|
10
|
+
|
|
11
|
+
```ruby
|
|
12
|
+
cursor.field_name = "position"
|
|
13
|
+
Feed::Item.desc(:created_at).scroll(cursor)
|
|
14
|
+
```
|
|
@@ -4,14 +4,8 @@ Bundler.setup(:default, :development)
|
|
|
4
4
|
require 'mongoid-scroll'
|
|
5
5
|
require 'faker'
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
Moped.logger.level = Logger::DEBUG
|
|
10
|
-
else
|
|
11
|
-
Mongoid.logger.level = Logger::INFO
|
|
12
|
-
Mongo::Logger.logger.level = Logger::INFO if Mongoid::Compatibility::Version.mongoid5?
|
|
13
|
-
end
|
|
14
|
-
|
|
7
|
+
Mongoid.logger.level = Logger::INFO
|
|
8
|
+
Mongo::Logger.logger.level = Logger::INFO if Mongoid::Compatibility::Version.mongoid5?
|
|
15
9
|
Mongoid.connect_to 'mongoid_scroll_demo'
|
|
16
10
|
Mongoid.purge!
|
|
17
11
|
|
data/lib/config/locales/en.yml
CHANGED
|
@@ -7,10 +7,18 @@ en:
|
|
|
7
7
|
message: "Scrolling over a criteria with multiple fields is not supported."
|
|
8
8
|
summary: "You're attempting to scroll over data with a sort order that includes multiple fields: %{sort}."
|
|
9
9
|
resolution: "Simplify the sort order to a single field."
|
|
10
|
+
mismatched_sort_fields:
|
|
11
|
+
message: "Specifying different sort fields than the original sort is not supported."
|
|
12
|
+
summary: "You're attempting to scroll over data with a sort order that differs between the cursor and the original criteria: %{diff}."
|
|
13
|
+
resolution: "Don't update the cursor sort options."
|
|
10
14
|
invalid_cursor:
|
|
11
15
|
message: "The cursor supplied is invalid."
|
|
12
16
|
summary: "The cursor supplied is invalid: %{cursor}."
|
|
13
17
|
resolution: "Cursors must be in the form 'value:tiebreak_id'."
|
|
18
|
+
invalid_base64_cursor:
|
|
19
|
+
message: "The cursor supplied is invalid."
|
|
20
|
+
summary: "The cursor supplied is invalid: %{cursor}."
|
|
21
|
+
resolution: "Cursors must be a base64-encoded string."
|
|
14
22
|
no_such_field:
|
|
15
23
|
message: "Invalid field."
|
|
16
24
|
summary: "The field supplied in the cursor does not exist: %{field}."
|
data/lib/mongo/scrollable.rb
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
module Mongo
|
|
2
2
|
module Scrollable
|
|
3
|
-
|
|
3
|
+
include Mongoid::Criteria::Scrollable::Fields
|
|
4
|
+
include Mongoid::Criteria::Scrollable::Cursors
|
|
5
|
+
|
|
6
|
+
def scroll(cursor_or_type = nil, options = nil, &_block)
|
|
7
|
+
cursor, cursor_type = cursor_and_type(cursor_or_type)
|
|
4
8
|
view = self
|
|
5
9
|
# we don't support scrolling over a view with multiple fields
|
|
6
10
|
raise Mongoid::Scroll::Errors::MultipleSortFieldsError.new(sort: view.sort) if view.sort && view.sort.keys.size != 1
|
|
@@ -10,7 +14,8 @@ module Mongo
|
|
|
10
14
|
# scroll cursor from the parameter, with value and tiebreak_id
|
|
11
15
|
options = { field_type: BSON::ObjectId } unless options
|
|
12
16
|
cursor_options = { field_name: scroll_field, direction: scroll_direction }.merge(options)
|
|
13
|
-
cursor = cursor.is_a?(
|
|
17
|
+
cursor = cursor && cursor.is_a?(cursor_type) ? cursor : cursor_type.new(cursor, cursor_options)
|
|
18
|
+
raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options)
|
|
14
19
|
# make a view
|
|
15
20
|
view = Mongo::Collection::View.new(
|
|
16
21
|
view.collection,
|
|
@@ -22,7 +27,7 @@ module Mongo
|
|
|
22
27
|
# scroll
|
|
23
28
|
if block_given?
|
|
24
29
|
view.each do |record|
|
|
25
|
-
yield record,
|
|
30
|
+
yield record, cursor_type.from_record(record, cursor_options)
|
|
26
31
|
end
|
|
27
32
|
else
|
|
28
33
|
view
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
module Mongoid
|
|
2
|
+
class Criteria
|
|
3
|
+
module Scrollable
|
|
4
|
+
# Shared by *::Scrollable modules
|
|
5
|
+
module Cursors
|
|
6
|
+
private
|
|
7
|
+
|
|
8
|
+
def cursor_and_type(cursor_or_type)
|
|
9
|
+
cursor = cursor_or_type.is_a?(Class) ? nil : cursor_or_type
|
|
10
|
+
cursor_type = cursor_or_type.is_a?(Class) ? cursor_or_type : nil
|
|
11
|
+
cursor_type ||= cursor.class if cursor.is_a?(Mongoid::Scroll::BaseCursor)
|
|
12
|
+
cursor_type ||= Mongoid::Scroll::Cursor
|
|
13
|
+
[cursor, cursor_type]
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
module Mongoid
|
|
2
|
+
class Criteria
|
|
3
|
+
module Scrollable
|
|
4
|
+
# Shared by *::Scrollable modules
|
|
5
|
+
module Fields
|
|
6
|
+
private
|
|
7
|
+
|
|
8
|
+
def raise_mismatched_sort_fields_error!(cursor, criteria_cursor_options)
|
|
9
|
+
diff = cursor.sort_options.reject { |k, v| criteria_cursor_options[k] == v }
|
|
10
|
+
raise Mongoid::Scroll::Errors::MismatchedSortFieldsError.new(diff: diff)
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def different_sort_fields?(cursor, criteria_cursor_options)
|
|
14
|
+
criteria_cursor_options[:field_type] = criteria_cursor_options[:field_type].to_s
|
|
15
|
+
criteria_cursor_options[:field_name] = criteria_cursor_options[:field_name].to_s
|
|
16
|
+
criteria_cursor_options != cursor.sort_options
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
module Mongoid
|
|
2
2
|
class Criteria
|
|
3
3
|
module Scrollable
|
|
4
|
-
|
|
4
|
+
include Mongoid::Criteria::Scrollable::Fields
|
|
5
|
+
include Mongoid::Criteria::Scrollable::Cursors
|
|
6
|
+
|
|
7
|
+
def scroll(cursor_or_type = nil, &_block)
|
|
8
|
+
cursor, cursor_type = cursor_and_type(cursor_or_type)
|
|
5
9
|
raise_multiple_sort_fields_error if multiple_sort_fields?
|
|
6
10
|
criteria = dup
|
|
7
11
|
criteria.merge!(default_sort) if no_sort_option?
|
|
8
12
|
cursor_options = build_cursor_options(criteria)
|
|
9
|
-
cursor = cursor.is_a?(
|
|
13
|
+
cursor = cursor.is_a?(cursor_type) ? cursor : new_cursor(cursor_type, cursor, cursor_options)
|
|
14
|
+
raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options)
|
|
10
15
|
cursor_criteria = build_cursor_criteria(criteria, cursor)
|
|
11
16
|
if block_given?
|
|
12
17
|
cursor_criteria.order_by(_id: scroll_direction(criteria)).each do |record|
|
|
13
|
-
yield record, cursor_from_record(record, cursor_options)
|
|
18
|
+
yield record, cursor_from_record(cursor_type, record, cursor_options)
|
|
14
19
|
end
|
|
15
20
|
else
|
|
16
21
|
cursor_criteria
|
|
@@ -51,8 +56,8 @@ module Mongoid
|
|
|
51
56
|
}
|
|
52
57
|
end
|
|
53
58
|
|
|
54
|
-
def new_cursor(cursor, cursor_options)
|
|
55
|
-
|
|
59
|
+
def new_cursor(cursor_type, cursor, cursor_options)
|
|
60
|
+
cursor_type.new(cursor, cursor_options)
|
|
56
61
|
end
|
|
57
62
|
|
|
58
63
|
def build_cursor_criteria(criteria, cursor)
|
|
@@ -61,18 +66,14 @@ module Mongoid
|
|
|
61
66
|
cursor_criteria
|
|
62
67
|
end
|
|
63
68
|
|
|
64
|
-
def cursor_from_record(record, cursor_options)
|
|
65
|
-
|
|
69
|
+
def cursor_from_record(cursor_type, record, cursor_options)
|
|
70
|
+
cursor_type.from_record(record, cursor_options)
|
|
66
71
|
end
|
|
67
72
|
|
|
68
73
|
def scroll_field_type(criteria)
|
|
69
74
|
scroll_field = scroll_field(criteria)
|
|
70
75
|
field = criteria.klass.fields[scroll_field.to_s]
|
|
71
|
-
field.foreign_key? && field.object_id_field? ?
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
def bson_type
|
|
75
|
-
Mongoid::Compatibility::Version.mongoid3? ? Moped::BSON::ObjectId : BSON::ObjectId
|
|
76
|
+
field.foreign_key? && field.object_id_field? ? BSON::ObjectId : field.type
|
|
76
77
|
end
|
|
77
78
|
end
|
|
78
79
|
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
require 'base64'
|
|
2
|
+
require 'json'
|
|
3
|
+
|
|
4
|
+
module Mongoid
|
|
5
|
+
module Scroll
|
|
6
|
+
# Allows to serializer/deserialize the cursor using RFC 4648
|
|
7
|
+
class Base64EncodedCursor < BaseCursor
|
|
8
|
+
def initialize(value, options = {})
|
|
9
|
+
options = extract_field_options(options)
|
|
10
|
+
if value
|
|
11
|
+
begin
|
|
12
|
+
parsed = ::JSON.parse(::Base64.strict_decode64(value))
|
|
13
|
+
rescue
|
|
14
|
+
raise Mongoid::Scroll::Errors::InvalidBase64CursorError.new(cursor: value)
|
|
15
|
+
end
|
|
16
|
+
super parse_field_value(parsed['field_type'], parsed['field_name'], parsed['value']), {
|
|
17
|
+
field_type: parsed['field_type'],
|
|
18
|
+
field_name: parsed['field_name'],
|
|
19
|
+
direction: parsed['direction'],
|
|
20
|
+
include_current: parsed['include_current'],
|
|
21
|
+
tiebreak_id: parsed['tiebreak_id'] && !parsed['tiebreak_id'].empty? ? BSON::ObjectId.from_string(parsed['tiebreak_id']) : nil
|
|
22
|
+
}
|
|
23
|
+
else
|
|
24
|
+
super nil, options
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def to_s
|
|
29
|
+
Base64.strict_encode64({
|
|
30
|
+
value: transform_field_value(field_type, field_name, value),
|
|
31
|
+
field_type: field_type.to_s,
|
|
32
|
+
field_name: field_name,
|
|
33
|
+
direction: direction,
|
|
34
|
+
include_current: include_current,
|
|
35
|
+
tiebreak_id: tiebreak_id && tiebreak_id.to_s
|
|
36
|
+
}.to_json)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|