dhs 1.0.1 → 1.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/rubocop.yml +2 -13
- data/.github/workflows/test.yml +2 -13
- data/lib/dhs/concerns/record/batch.rb +6 -5
- data/lib/dhs/version.rb +1 -1
- data/spec/record/find_each_spec.rb +6 -6
- data/spec/record/find_in_batches_spec.rb +50 -12
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f42a1fdc00478fe3e24e797f52396c953442e4e0d2da738b28fb7cc55b908ea3
|
4
|
+
data.tar.gz: 1465403fb48c2dca7f2b955550309a7983a4d264ef10cbb35d620df88f01f1fe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bd6412647a124e42350e185d60a6a0040aa9ef30db503fddf529db0979f4c6b7151bc3e9cdad8378a111512b4f481868a39839c89e42f7ea420c3a27e2539050
|
7
|
+
data.tar.gz: a6936544068aa3b6591163f768a6e3c96ce38317abc183ca3bec8e62c50da10d25518f215b939945acd990f449615865d19dec5886a8cd2e88e59965c1fcb5d8
|
@@ -8,20 +8,9 @@ jobs:
|
|
8
8
|
|
9
9
|
steps:
|
10
10
|
- uses: actions/checkout@v2
|
11
|
-
- uses:
|
11
|
+
- uses: ruby/setup-ruby@master
|
12
12
|
with:
|
13
|
-
|
14
|
-
- name: Cache Ruby Gems
|
15
|
-
uses: actions/cache@v2
|
16
|
-
with:
|
17
|
-
path: /.tmp/vendor/bundle
|
18
|
-
key: ${{ runner.os }}-gems-latest-${{ hashFiles('**/Gemfile.lock') }}
|
19
|
-
restore-keys: |
|
20
|
-
${{ runner.os }}-gems-latest-
|
21
|
-
- name: Bundle Install
|
22
|
-
run: |
|
23
|
-
bundle config path /.tmp/vendor/bundle
|
24
|
-
bundle install --jobs 4 --retry 3
|
13
|
+
bundler-cache: true
|
25
14
|
- name: Run Rubocop
|
26
15
|
run: |
|
27
16
|
bundle exec rubocop
|
data/.github/workflows/test.yml
CHANGED
@@ -8,20 +8,9 @@ jobs:
|
|
8
8
|
|
9
9
|
steps:
|
10
10
|
- uses: actions/checkout@v2
|
11
|
-
- uses:
|
11
|
+
- uses: ruby/setup-ruby@master
|
12
12
|
with:
|
13
|
-
|
14
|
-
- name: Cache Ruby Gems
|
15
|
-
uses: actions/cache@v2
|
16
|
-
with:
|
17
|
-
path: /.tmp/vendor/bundle
|
18
|
-
key: ${{ runner.os }}-gems-latest-${{ hashFiles('**/Gemfile.lock') }}
|
19
|
-
restore-keys: |
|
20
|
-
${{ runner.os }}-gems-latest-
|
21
|
-
- name: Bundle Install
|
22
|
-
run: |
|
23
|
-
bundle config path /.tmp/vendor/bundle
|
24
|
-
bundle install --jobs 4 --retry 3
|
13
|
+
bundler-cache: true
|
25
14
|
- name: Run Tests
|
26
15
|
run: |
|
27
16
|
bundle exec rspec
|
@@ -22,17 +22,18 @@ class DHS::Record
|
|
22
22
|
def find_in_batches(options = {})
|
23
23
|
raise 'No block given' unless block_given?
|
24
24
|
options = options.dup
|
25
|
-
start = options
|
26
|
-
batch_size = options.delete(:batch_size) ||
|
25
|
+
start = options[:start] || self.pagination_class::DEFAULT_OFFSET
|
26
|
+
batch_size = options.delete(:batch_size) || self.pagination_class::DEFAULT_LIMIT
|
27
27
|
loop do # as suggested by Matz
|
28
28
|
options = options.dup
|
29
29
|
options[:params] = (options[:params] || {}).merge(limit_key(:parameter) => batch_size, pagination_key(:parameter) => start)
|
30
30
|
data = request(options)
|
31
|
-
|
32
|
-
|
31
|
+
pagination = self.pagination(data)
|
32
|
+
batch_size = pagination.limit
|
33
|
+
left = pagination.pages_left
|
33
34
|
yield new(data)
|
34
35
|
break if left <= 0
|
35
|
-
start
|
36
|
+
start = pagination.next_offset
|
36
37
|
end
|
37
38
|
end
|
38
39
|
end
|
data/lib/dhs/version.rb
CHANGED
@@ -29,11 +29,11 @@ describe DHS::Collection do
|
|
29
29
|
|
30
30
|
context 'find_each' do
|
31
31
|
it 'processes each record by fetching records in batches' do
|
32
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
33
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
34
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
35
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
36
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
32
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=0").to_return(status: 200, body: api_response((1..100).to_a, 0))
|
33
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=100").to_return(status: 200, body: api_response((101..200).to_a, 100))
|
34
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=200").to_return(status: 200, body: api_response((201..300).to_a, 200))
|
35
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=300").to_return(status: 200, body: api_response((301..400).to_a, 300))
|
36
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=400").to_return(status: 200, body: api_response((401..total).to_a, 400))
|
37
37
|
count = 0
|
38
38
|
Record.find_each do |record|
|
39
39
|
count += 1
|
@@ -45,7 +45,7 @@ describe DHS::Collection do
|
|
45
45
|
end
|
46
46
|
|
47
47
|
it 'passes options to the requests made' do
|
48
|
-
request = stub_request(:get, 'http://depay.fi/v2/feedbacks?limit=100&offset=
|
48
|
+
request = stub_request(:get, 'http://depay.fi/v2/feedbacks?limit=100&offset=0')
|
49
49
|
.with(headers: { 'Authorization' => 'Bearer 123' })
|
50
50
|
.to_return(body: {
|
51
51
|
items: []
|
@@ -29,11 +29,11 @@ describe DHS::Collection do
|
|
29
29
|
|
30
30
|
context 'find_batches' do
|
31
31
|
it 'processes records in batches' do
|
32
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
33
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
34
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
35
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
36
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
32
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=0").to_return(status: 200, body: api_response((1..100).to_a, 0))
|
33
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=100").to_return(status: 200, body: api_response((101..200).to_a, 100))
|
34
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=200").to_return(status: 200, body: api_response((201..300).to_a, 200))
|
35
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=300").to_return(status: 200, body: api_response((301..400).to_a, 300))
|
36
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=400").to_return(status: 200, body: api_response((401..total).to_a, 400))
|
37
37
|
length = 0
|
38
38
|
Record.find_in_batches do |records|
|
39
39
|
length += records.length
|
@@ -44,11 +44,11 @@ describe DHS::Collection do
|
|
44
44
|
end
|
45
45
|
|
46
46
|
it 'adapts to backend max limit' do
|
47
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=230&offset=
|
48
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
49
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
50
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
51
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
47
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=230&offset=0").to_return(status: 200, body: api_response((1..100).to_a, 0))
|
48
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=100").to_return(status: 200, body: api_response((101..200).to_a, 100))
|
49
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=200").to_return(status: 200, body: api_response((201..300).to_a, 200))
|
50
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=300").to_return(status: 200, body: api_response((301..400).to_a, 300))
|
51
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=400").to_return(status: 200, body: api_response((401..total).to_a, 400))
|
52
52
|
length = 0
|
53
53
|
Record.find_in_batches(batch_size: 230) do |records|
|
54
54
|
length += records.length
|
@@ -59,8 +59,8 @@ describe DHS::Collection do
|
|
59
59
|
end
|
60
60
|
|
61
61
|
it 'forwards offset' do
|
62
|
-
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=
|
63
|
-
Record.find_in_batches(start:
|
62
|
+
stub_request(:get, "#{datastore}/feedbacks?limit=100&offset=400").to_return(status: 200, body: api_response((401..total).to_a, 400))
|
63
|
+
Record.find_in_batches(start: 400) do |records|
|
64
64
|
expect(records.length).to eq(total - 400)
|
65
65
|
end
|
66
66
|
end
|
@@ -119,4 +119,42 @@ describe DHS::Collection do
|
|
119
119
|
expect(length).to eq total
|
120
120
|
end
|
121
121
|
end
|
122
|
+
|
123
|
+
context 'different pagination strategy' do
|
124
|
+
before do
|
125
|
+
class Transaction < DHS::Record
|
126
|
+
endpoint 'https://api/transactions'
|
127
|
+
configuration limit_key: :items_on_page, pagination_strategy: :total_pages, pagination_key: :page, items_key: :transactions, total_key: :total_pages
|
128
|
+
end
|
129
|
+
|
130
|
+
stub_request(:get, 'https://api/transactions?items_on_page=50&page=1')
|
131
|
+
.to_return(body: {
|
132
|
+
page: 1,
|
133
|
+
total_pages: 2,
|
134
|
+
items_on_page: 50,
|
135
|
+
transactions: 50.times.map { |index| { id: index } }
|
136
|
+
}.to_json)
|
137
|
+
|
138
|
+
stub_request(:get, 'https://api/transactions?items_on_page=50&page=2')
|
139
|
+
.to_return(body: {
|
140
|
+
page: 2,
|
141
|
+
total_pages: 2,
|
142
|
+
items_on_page: 50,
|
143
|
+
transactions: 22.times.map { |index| { id: 50 + index } }
|
144
|
+
}.to_json)
|
145
|
+
end
|
146
|
+
|
147
|
+
it 'find in batches and paginates automatically even for different pagination strategies' do
|
148
|
+
total = 0
|
149
|
+
transactions = []
|
150
|
+
|
151
|
+
Transaction.find_in_batches(batch_size: 50) do |batch|
|
152
|
+
total += batch.length
|
153
|
+
transactions << batch
|
154
|
+
end
|
155
|
+
|
156
|
+
expect(total).to eq(72)
|
157
|
+
expect(transactions.flatten.as_json).to eq(72.times.map { |index| { id: index }.as_json })
|
158
|
+
end
|
159
|
+
end
|
122
160
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dhs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- https://github.com/DePayFi/dhs/graphs/contributors
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-04-
|
11
|
+
date: 2021-04-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activemodel
|