dhs 1.3.0 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +46 -1
- data/lib/dhs/concerns/record/pagination.rb +4 -1
- data/lib/dhs/concerns/record/request.rb +11 -8
- data/lib/dhs/pagination/link.rb +4 -0
- data/lib/dhs/pagination/next_offset.rb +28 -0
- data/lib/dhs/version.rb +1 -1
- data/lib/dhs.rb +1 -0
- data/spec/graphql/main_spec.rb +3 -1
- data/spec/pagination/next_offset_spec.rb +33 -0
- data/spec/record/includes_spec.rb +1 -1
- data/spec/record/paginatable_collection_spec.rb +1 -1
- metadata +6 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c31dbca2778853f9bafb3a74e2c7ef830d3897c0caa78b66e5bb58203fc8bb11
|
4
|
+
data.tar.gz: 8e5968134c395c0b4e6196506ef5502f5941f68754423bc885bc536a44c2ba33
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 54e1c9ec3c0b71566a8b6831690250d0eecc00f00155617dc39873c20d49a26ef57051ae04d7d3a1bbb3733f75fc3ccc0024b96c3b4bd086dafe48a1634cb8cf
|
7
|
+
data.tar.gz: 1bea7f70ccd0cba8dc488604d5f4a668d1bc602a2cb30d19058b195a17b2d9f1aaa5d72e21286e910570a89be0e78c25adac4bb45186a6f3f11ffcc81128b39c
|
data/README.md
CHANGED
@@ -164,7 +164,7 @@ You can use DHS also to fetch records from GraphQL Endpoints:
|
|
164
164
|
|
165
165
|
class Record < DHS::Record
|
166
166
|
|
167
|
-
configuration
|
167
|
+
configuration items_key: [:data, :ethereum, :address, 0, :balances]
|
168
168
|
|
169
169
|
endpoint 'https://graphql.bitquery.io/',
|
170
170
|
graphql: {
|
@@ -1386,6 +1386,51 @@ Sequentially:
|
|
1386
1386
|
GET https://service.example.com/records?from_record_id=xcaoXBmuMyFFEcFDSgNgDQ&limit=100
|
1387
1387
|
```
|
1388
1388
|
|
1389
|
+
##### Pagination strategy: next_offset
|
1390
|
+
|
1391
|
+
The `next_offset` strategy continuously follows in-response offset information to following pages until the last page is reached (indicated by next offset being either empty or 0).
|
1392
|
+
|
1393
|
+
*WARNING*
|
1394
|
+
|
1395
|
+
Loading all pages from a resource paginated with next_offset only can result in very poor performance, as pages can only be loaded sequentially!
|
1396
|
+
|
1397
|
+
```ruby
|
1398
|
+
# app/models/record.rb
|
1399
|
+
|
1400
|
+
class Search < DHS::Record
|
1401
|
+
configuration pagination_strategy: 'next_offset'
|
1402
|
+
|
1403
|
+
endpoint '{+service}/assets'
|
1404
|
+
end
|
1405
|
+
```
|
1406
|
+
|
1407
|
+
```ruby
|
1408
|
+
# app/controllers/some_controller.rb
|
1409
|
+
|
1410
|
+
Record.all
|
1411
|
+
|
1412
|
+
```
|
1413
|
+
```
|
1414
|
+
GET https://service.example.com/assets?limit=100
|
1415
|
+
{
|
1416
|
+
items: [{...}, ...],
|
1417
|
+
limit: 10,
|
1418
|
+
next_offset: 29
|
1419
|
+
}
|
1420
|
+
GET https://service.example.com/assets?offset=29
|
1421
|
+
{
|
1422
|
+
items: [{...}, ...],
|
1423
|
+
limit: 10,
|
1424
|
+
next_offset: 39
|
1425
|
+
}
|
1426
|
+
GET https://service.example.com/assets?offset=39
|
1427
|
+
{
|
1428
|
+
items: [{...}, ...],
|
1429
|
+
limit: 10,
|
1430
|
+
next_offset: 0
|
1431
|
+
}
|
1432
|
+
```
|
1433
|
+
|
1389
1434
|
#### Pagination keys
|
1390
1435
|
|
1391
1436
|
##### limit_key
|
@@ -30,6 +30,8 @@ class DHS::Record
|
|
30
30
|
DHS::Pagination::Start
|
31
31
|
when :link
|
32
32
|
DHS::Pagination::Link
|
33
|
+
when :next_offset
|
34
|
+
DHS::Pagination::NextOffset
|
33
35
|
else
|
34
36
|
DHS::Pagination::Offset
|
35
37
|
end
|
@@ -43,7 +45,8 @@ class DHS::Record
|
|
43
45
|
def paginated?(raw)
|
44
46
|
raw.is_a?(Hash) && (
|
45
47
|
raw.dig(*total_key).present? ||
|
46
|
-
raw.dig(*limit_key(:body)).present?
|
48
|
+
raw.dig(*limit_key(:body)).present? ||
|
49
|
+
raw.dig(*pagination_key(:body)).present?
|
47
50
|
)
|
48
51
|
end
|
49
52
|
end
|
@@ -237,7 +237,7 @@ class DHS::Record
|
|
237
237
|
if pagination.parallel?
|
238
238
|
load_and_merge_parallel_requests!(record, data, pagination, options)
|
239
239
|
else
|
240
|
-
load_and_merge_sequential_requests!(record, data, options,
|
240
|
+
load_and_merge_sequential_requests!(record, data, options, pagination)
|
241
241
|
end
|
242
242
|
end
|
243
243
|
|
@@ -249,13 +249,16 @@ class DHS::Record
|
|
249
249
|
end
|
250
250
|
end
|
251
251
|
|
252
|
-
def load_and_merge_sequential_requests!(record, data, options,
|
253
|
-
warn '[WARNING] You are loading all pages from a resource paginated with
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
252
|
+
def load_and_merge_sequential_requests!(record, data, options, pagination)
|
253
|
+
warn '[WARNING] You are loading all pages from a resource paginated with sequential pagination.'
|
254
|
+
next_value = pagination.next(data._raw)
|
255
|
+
while next_value.present?
|
256
|
+
page_data = if next_value.is_a?(String) && next_value.match(/^http/)
|
257
|
+
record.request(options.except(:all).merge(url: next_value))
|
258
|
+
else
|
259
|
+
record.request(options.except(:all).merge(params: (options.dig(:params) || {}).merge(next_value) ))
|
260
|
+
end
|
261
|
+
next_value = pagination.next(page_data._raw)
|
259
262
|
merge_batch_data_with_parent!(page_data, data, pagination)
|
260
263
|
end
|
261
264
|
end
|
data/lib/dhs/pagination/link.rb
CHANGED
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class DHS::Pagination::NextOffset < DHS::Pagination::Base
|
4
|
+
|
5
|
+
DEFAULT_OFFSET = 0
|
6
|
+
|
7
|
+
def total
|
8
|
+
data._raw.dig(*_record.items_key).count || 0
|
9
|
+
end
|
10
|
+
alias count total
|
11
|
+
|
12
|
+
def parallel?
|
13
|
+
false
|
14
|
+
end
|
15
|
+
|
16
|
+
def pages_left?
|
17
|
+
next_offset = data._raw.dig(*_record.pagination_key(:body))
|
18
|
+
next_offset.present? && !next_offset.zero?
|
19
|
+
end
|
20
|
+
|
21
|
+
def next(current)
|
22
|
+
next_value = current.dig(*_record.pagination_key(:body))
|
23
|
+
return if next_value.blank? || next_value.zero?
|
24
|
+
{
|
25
|
+
_record.pagination_key(:parameter) => current.dig(*_record.pagination_key(:body))
|
26
|
+
}
|
27
|
+
end
|
28
|
+
end
|
data/lib/dhs/version.rb
CHANGED
data/lib/dhs.rb
CHANGED
@@ -39,6 +39,7 @@ module DHS
|
|
39
39
|
autoload :Page, 'dhs/pagination/page'
|
40
40
|
autoload :TotalPages, 'dhs/pagination/total_pages'
|
41
41
|
autoload :OffsetPage, 'dhs/pagination/offset_page'
|
42
|
+
autoload :NextOffset, 'dhs/pagination/next_offset'
|
42
43
|
autoload :Start, 'dhs/pagination/start'
|
43
44
|
autoload :Link, 'dhs/pagination/link'
|
44
45
|
end
|
data/spec/graphql/main_spec.rb
CHANGED
@@ -88,11 +88,13 @@ describe 'main graphql support' do
|
|
88
88
|
end
|
89
89
|
|
90
90
|
before do
|
91
|
+
DHC.config.placeholder('bitquery', 'https://graphql.bitquery.io/')
|
92
|
+
|
91
93
|
class Record < DHS::Record
|
92
94
|
|
93
95
|
configuration items_key: [:data, :ethereum, :address, 0, :balances]
|
94
96
|
|
95
|
-
endpoint '
|
97
|
+
endpoint '{+bitquery}',
|
96
98
|
graphql: {
|
97
99
|
query: %{
|
98
100
|
query ($network: EthereumNetwork!, $address: String!) {
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rails_helper'
|
4
|
+
|
5
|
+
describe DHS::Record do
|
6
|
+
|
7
|
+
context 'pagination' do
|
8
|
+
|
9
|
+
def stub_api_request(items: [], offset: nil, next_offset:)
|
10
|
+
stub_request(:get, ["http://depay.fi/v2/transactions?limit=100", offset ? "offset=#{offset}" : nil].compact.join('&'))
|
11
|
+
.to_return(body: { items: items, next_offset: next_offset }.to_json)
|
12
|
+
end
|
13
|
+
|
14
|
+
let!(:requests) do
|
15
|
+
stub_api_request(items: (0...100).to_a, next_offset: 99)
|
16
|
+
stub_api_request(items: (100...200).to_a, offset: 99, next_offset: 199)
|
17
|
+
stub_api_request(items: (200...300).to_a, offset: 199, next_offset: 0)
|
18
|
+
end
|
19
|
+
|
20
|
+
before do
|
21
|
+
class Transaction < DHS::Record
|
22
|
+
configuration pagination_strategy: :next_offset, pagination_key: { body: :next_offset, parameter: :offset }
|
23
|
+
|
24
|
+
endpoint 'http://depay.fi/v2/transactions'
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'fetches all the pages' do
|
29
|
+
transactions = Transaction.all.fetch
|
30
|
+
expect(transactions.to_a).to eq (0...300).to_a
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -167,7 +167,7 @@ describe DHS::Record do
|
|
167
167
|
.includes(:users, contracts: :products)
|
168
168
|
.find(1)
|
169
169
|
end).to output(
|
170
|
-
%r{\[WARNING\] You are loading all pages from a resource paginated with
|
170
|
+
%r{\[WARNING\] You are loading all pages from a resource paginated with sequential pagination.}
|
171
171
|
).to_stderr
|
172
172
|
|
173
173
|
expect(customer.users.length).to eq amount_of_users
|
@@ -348,7 +348,7 @@ describe DHS::Record do
|
|
348
348
|
expect(lambda do
|
349
349
|
all = Record.all.fetch
|
350
350
|
end).to output(
|
351
|
-
%r{\[WARNING\] You are loading all pages from a resource paginated with
|
351
|
+
%r{\[WARNING\] You are loading all pages from a resource paginated with sequential pagination.}
|
352
352
|
).to_stderr
|
353
353
|
|
354
354
|
expect(all).to be_kind_of Record
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dhs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- https://github.com/DePayFi/dhs/graphs/contributors
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-04-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activemodel
|
@@ -327,6 +327,7 @@ files:
|
|
327
327
|
- lib/dhs/item.rb
|
328
328
|
- lib/dhs/pagination/base.rb
|
329
329
|
- lib/dhs/pagination/link.rb
|
330
|
+
- lib/dhs/pagination/next_offset.rb
|
330
331
|
- lib/dhs/pagination/offset.rb
|
331
332
|
- lib/dhs/pagination/offset_page.rb
|
332
333
|
- lib/dhs/pagination/page.rb
|
@@ -466,6 +467,7 @@ files:
|
|
466
467
|
- spec/pagination/link/pages_left_spec.rb
|
467
468
|
- spec/pagination/link/parallel_spec.rb
|
468
469
|
- spec/pagination/link/total_spec.rb
|
470
|
+
- spec/pagination/next_offset_spec.rb
|
469
471
|
- spec/pagination/offset/pages_left_spec.rb
|
470
472
|
- spec/pagination/offset_page_spec.rb
|
471
473
|
- spec/pagination/parameters_spec.rb
|
@@ -578,7 +580,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
578
580
|
version: '0'
|
579
581
|
requirements:
|
580
582
|
- Ruby >= 2.7.2
|
581
|
-
rubygems_version: 3.2.
|
583
|
+
rubygems_version: 3.2.33
|
582
584
|
signing_key:
|
583
585
|
specification_version: 4
|
584
586
|
summary: 'REST services accelerator: Rails gem providing an easy, active-record-like
|
@@ -703,6 +705,7 @@ test_files:
|
|
703
705
|
- spec/pagination/link/pages_left_spec.rb
|
704
706
|
- spec/pagination/link/parallel_spec.rb
|
705
707
|
- spec/pagination/link/total_spec.rb
|
708
|
+
- spec/pagination/next_offset_spec.rb
|
706
709
|
- spec/pagination/offset/pages_left_spec.rb
|
707
710
|
- spec/pagination/offset_page_spec.rb
|
708
711
|
- spec/pagination/parameters_spec.rb
|