dhs 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +46 -1
- data/lib/dhs/concerns/record/pagination.rb +4 -1
- data/lib/dhs/concerns/record/request.rb +11 -8
- data/lib/dhs/pagination/link.rb +4 -0
- data/lib/dhs/pagination/next_offset.rb +28 -0
- data/lib/dhs/version.rb +1 -1
- data/lib/dhs.rb +1 -0
- data/spec/graphql/main_spec.rb +3 -1
- data/spec/pagination/next_offset_spec.rb +33 -0
- data/spec/record/includes_spec.rb +1 -1
- data/spec/record/paginatable_collection_spec.rb +1 -1
- metadata +6 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c31dbca2778853f9bafb3a74e2c7ef830d3897c0caa78b66e5bb58203fc8bb11
|
4
|
+
data.tar.gz: 8e5968134c395c0b4e6196506ef5502f5941f68754423bc885bc536a44c2ba33
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 54e1c9ec3c0b71566a8b6831690250d0eecc00f00155617dc39873c20d49a26ef57051ae04d7d3a1bbb3733f75fc3ccc0024b96c3b4bd086dafe48a1634cb8cf
|
7
|
+
data.tar.gz: 1bea7f70ccd0cba8dc488604d5f4a668d1bc602a2cb30d19058b195a17b2d9f1aaa5d72e21286e910570a89be0e78c25adac4bb45186a6f3f11ffcc81128b39c
|
data/README.md
CHANGED
@@ -164,7 +164,7 @@ You can use DHS also to fetch records from GraphQL Endpoints:
|
|
164
164
|
|
165
165
|
class Record < DHS::Record
|
166
166
|
|
167
|
-
configuration
|
167
|
+
configuration items_key: [:data, :ethereum, :address, 0, :balances]
|
168
168
|
|
169
169
|
endpoint 'https://graphql.bitquery.io/',
|
170
170
|
graphql: {
|
@@ -1386,6 +1386,51 @@ Sequentially:
|
|
1386
1386
|
GET https://service.example.com/records?from_record_id=xcaoXBmuMyFFEcFDSgNgDQ&limit=100
|
1387
1387
|
```
|
1388
1388
|
|
1389
|
+
##### Pagination strategy: next_offset
|
1390
|
+
|
1391
|
+
The `next_offset` strategy continuously follows in-response offset information to following pages until the last page is reached (indicated by next offset being either empty or 0).
|
1392
|
+
|
1393
|
+
*WARNING*
|
1394
|
+
|
1395
|
+
Loading all pages from a resource paginated with next_offset only can result in very poor performance, as pages can only be loaded sequentially!
|
1396
|
+
|
1397
|
+
```ruby
|
1398
|
+
# app/models/record.rb
|
1399
|
+
|
1400
|
+
class Search < DHS::Record
|
1401
|
+
configuration pagination_strategy: 'next_offset'
|
1402
|
+
|
1403
|
+
endpoint '{+service}/assets'
|
1404
|
+
end
|
1405
|
+
```
|
1406
|
+
|
1407
|
+
```ruby
|
1408
|
+
# app/controllers/some_controller.rb
|
1409
|
+
|
1410
|
+
Record.all
|
1411
|
+
|
1412
|
+
```
|
1413
|
+
```
|
1414
|
+
GET https://service.example.com/assets?limit=100
|
1415
|
+
{
|
1416
|
+
items: [{...}, ...],
|
1417
|
+
limit: 10,
|
1418
|
+
next_offset: 29
|
1419
|
+
}
|
1420
|
+
GET https://service.example.com/assets?offset=29
|
1421
|
+
{
|
1422
|
+
items: [{...}, ...],
|
1423
|
+
limit: 10,
|
1424
|
+
next_offset: 39
|
1425
|
+
}
|
1426
|
+
GET https://service.example.com/assets?offset=39
|
1427
|
+
{
|
1428
|
+
items: [{...}, ...],
|
1429
|
+
limit: 10,
|
1430
|
+
next_offset: 0
|
1431
|
+
}
|
1432
|
+
```
|
1433
|
+
|
1389
1434
|
#### Pagination keys
|
1390
1435
|
|
1391
1436
|
##### limit_key
|
@@ -30,6 +30,8 @@ class DHS::Record
|
|
30
30
|
DHS::Pagination::Start
|
31
31
|
when :link
|
32
32
|
DHS::Pagination::Link
|
33
|
+
when :next_offset
|
34
|
+
DHS::Pagination::NextOffset
|
33
35
|
else
|
34
36
|
DHS::Pagination::Offset
|
35
37
|
end
|
@@ -43,7 +45,8 @@ class DHS::Record
|
|
43
45
|
def paginated?(raw)
|
44
46
|
raw.is_a?(Hash) && (
|
45
47
|
raw.dig(*total_key).present? ||
|
46
|
-
raw.dig(*limit_key(:body)).present?
|
48
|
+
raw.dig(*limit_key(:body)).present? ||
|
49
|
+
raw.dig(*pagination_key(:body)).present?
|
47
50
|
)
|
48
51
|
end
|
49
52
|
end
|
@@ -237,7 +237,7 @@ class DHS::Record
|
|
237
237
|
if pagination.parallel?
|
238
238
|
load_and_merge_parallel_requests!(record, data, pagination, options)
|
239
239
|
else
|
240
|
-
load_and_merge_sequential_requests!(record, data, options,
|
240
|
+
load_and_merge_sequential_requests!(record, data, options, pagination)
|
241
241
|
end
|
242
242
|
end
|
243
243
|
|
@@ -249,13 +249,16 @@ class DHS::Record
|
|
249
249
|
end
|
250
250
|
end
|
251
251
|
|
252
|
-
def load_and_merge_sequential_requests!(record, data, options,
|
253
|
-
warn '[WARNING] You are loading all pages from a resource paginated with
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
252
|
+
def load_and_merge_sequential_requests!(record, data, options, pagination)
|
253
|
+
warn '[WARNING] You are loading all pages from a resource paginated with sequential pagination.'
|
254
|
+
next_value = pagination.next(data._raw)
|
255
|
+
while next_value.present?
|
256
|
+
page_data = if next_value.is_a?(String) && next_value.match(/^http/)
|
257
|
+
record.request(options.except(:all).merge(url: next_value))
|
258
|
+
else
|
259
|
+
record.request(options.except(:all).merge(params: (options.dig(:params) || {}).merge(next_value) ))
|
260
|
+
end
|
261
|
+
next_value = pagination.next(page_data._raw)
|
259
262
|
merge_batch_data_with_parent!(page_data, data, pagination)
|
260
263
|
end
|
261
264
|
end
|
data/lib/dhs/pagination/link.rb
CHANGED
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class DHS::Pagination::NextOffset < DHS::Pagination::Base
|
4
|
+
|
5
|
+
DEFAULT_OFFSET = 0
|
6
|
+
|
7
|
+
def total
|
8
|
+
data._raw.dig(*_record.items_key).count || 0
|
9
|
+
end
|
10
|
+
alias count total
|
11
|
+
|
12
|
+
def parallel?
|
13
|
+
false
|
14
|
+
end
|
15
|
+
|
16
|
+
def pages_left?
|
17
|
+
next_offset = data._raw.dig(*_record.pagination_key(:body))
|
18
|
+
next_offset.present? && !next_offset.zero?
|
19
|
+
end
|
20
|
+
|
21
|
+
def next(current)
|
22
|
+
next_value = current.dig(*_record.pagination_key(:body))
|
23
|
+
return if next_value.blank? || next_value.zero?
|
24
|
+
{
|
25
|
+
_record.pagination_key(:parameter) => current.dig(*_record.pagination_key(:body))
|
26
|
+
}
|
27
|
+
end
|
28
|
+
end
|
data/lib/dhs/version.rb
CHANGED
data/lib/dhs.rb
CHANGED
@@ -39,6 +39,7 @@ module DHS
|
|
39
39
|
autoload :Page, 'dhs/pagination/page'
|
40
40
|
autoload :TotalPages, 'dhs/pagination/total_pages'
|
41
41
|
autoload :OffsetPage, 'dhs/pagination/offset_page'
|
42
|
+
autoload :NextOffset, 'dhs/pagination/next_offset'
|
42
43
|
autoload :Start, 'dhs/pagination/start'
|
43
44
|
autoload :Link, 'dhs/pagination/link'
|
44
45
|
end
|
data/spec/graphql/main_spec.rb
CHANGED
@@ -88,11 +88,13 @@ describe 'main graphql support' do
|
|
88
88
|
end
|
89
89
|
|
90
90
|
before do
|
91
|
+
DHC.config.placeholder('bitquery', 'https://graphql.bitquery.io/')
|
92
|
+
|
91
93
|
class Record < DHS::Record
|
92
94
|
|
93
95
|
configuration items_key: [:data, :ethereum, :address, 0, :balances]
|
94
96
|
|
95
|
-
endpoint '
|
97
|
+
endpoint '{+bitquery}',
|
96
98
|
graphql: {
|
97
99
|
query: %{
|
98
100
|
query ($network: EthereumNetwork!, $address: String!) {
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rails_helper'
|
4
|
+
|
5
|
+
describe DHS::Record do
|
6
|
+
|
7
|
+
context 'pagination' do
|
8
|
+
|
9
|
+
def stub_api_request(items: [], offset: nil, next_offset:)
|
10
|
+
stub_request(:get, ["http://depay.fi/v2/transactions?limit=100", offset ? "offset=#{offset}" : nil].compact.join('&'))
|
11
|
+
.to_return(body: { items: items, next_offset: next_offset }.to_json)
|
12
|
+
end
|
13
|
+
|
14
|
+
let!(:requests) do
|
15
|
+
stub_api_request(items: (0...100).to_a, next_offset: 99)
|
16
|
+
stub_api_request(items: (100...200).to_a, offset: 99, next_offset: 199)
|
17
|
+
stub_api_request(items: (200...300).to_a, offset: 199, next_offset: 0)
|
18
|
+
end
|
19
|
+
|
20
|
+
before do
|
21
|
+
class Transaction < DHS::Record
|
22
|
+
configuration pagination_strategy: :next_offset, pagination_key: { body: :next_offset, parameter: :offset }
|
23
|
+
|
24
|
+
endpoint 'http://depay.fi/v2/transactions'
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'fetches all the pages' do
|
29
|
+
transactions = Transaction.all.fetch
|
30
|
+
expect(transactions.to_a).to eq (0...300).to_a
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -167,7 +167,7 @@ describe DHS::Record do
|
|
167
167
|
.includes(:users, contracts: :products)
|
168
168
|
.find(1)
|
169
169
|
end).to output(
|
170
|
-
%r{\[WARNING\] You are loading all pages from a resource paginated with
|
170
|
+
%r{\[WARNING\] You are loading all pages from a resource paginated with sequential pagination.}
|
171
171
|
).to_stderr
|
172
172
|
|
173
173
|
expect(customer.users.length).to eq amount_of_users
|
@@ -348,7 +348,7 @@ describe DHS::Record do
|
|
348
348
|
expect(lambda do
|
349
349
|
all = Record.all.fetch
|
350
350
|
end).to output(
|
351
|
-
%r{\[WARNING\] You are loading all pages from a resource paginated with
|
351
|
+
%r{\[WARNING\] You are loading all pages from a resource paginated with sequential pagination.}
|
352
352
|
).to_stderr
|
353
353
|
|
354
354
|
expect(all).to be_kind_of Record
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dhs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- https://github.com/DePayFi/dhs/graphs/contributors
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-04-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activemodel
|
@@ -327,6 +327,7 @@ files:
|
|
327
327
|
- lib/dhs/item.rb
|
328
328
|
- lib/dhs/pagination/base.rb
|
329
329
|
- lib/dhs/pagination/link.rb
|
330
|
+
- lib/dhs/pagination/next_offset.rb
|
330
331
|
- lib/dhs/pagination/offset.rb
|
331
332
|
- lib/dhs/pagination/offset_page.rb
|
332
333
|
- lib/dhs/pagination/page.rb
|
@@ -466,6 +467,7 @@ files:
|
|
466
467
|
- spec/pagination/link/pages_left_spec.rb
|
467
468
|
- spec/pagination/link/parallel_spec.rb
|
468
469
|
- spec/pagination/link/total_spec.rb
|
470
|
+
- spec/pagination/next_offset_spec.rb
|
469
471
|
- spec/pagination/offset/pages_left_spec.rb
|
470
472
|
- spec/pagination/offset_page_spec.rb
|
471
473
|
- spec/pagination/parameters_spec.rb
|
@@ -578,7 +580,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
578
580
|
version: '0'
|
579
581
|
requirements:
|
580
582
|
- Ruby >= 2.7.2
|
581
|
-
rubygems_version: 3.2.
|
583
|
+
rubygems_version: 3.2.33
|
582
584
|
signing_key:
|
583
585
|
specification_version: 4
|
584
586
|
summary: 'REST services accelerator: Rails gem providing an easy, active-record-like
|
@@ -703,6 +705,7 @@ test_files:
|
|
703
705
|
- spec/pagination/link/pages_left_spec.rb
|
704
706
|
- spec/pagination/link/parallel_spec.rb
|
705
707
|
- spec/pagination/link/total_spec.rb
|
708
|
+
- spec/pagination/next_offset_spec.rb
|
706
709
|
- spec/pagination/offset/pages_left_spec.rb
|
707
710
|
- spec/pagination/offset_page_spec.rb
|
708
711
|
- spec/pagination/parameters_spec.rb
|