dhs 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1ed15754a20492dd70e8d58fba832f6db2c747d4f13d04ad23457ab72fd81eda
4
- data.tar.gz: 2626f410568a801f540f592e32c8f2f6ca5f9c21fa9e1a74229d9f0866139f21
3
+ metadata.gz: c31dbca2778853f9bafb3a74e2c7ef830d3897c0caa78b66e5bb58203fc8bb11
4
+ data.tar.gz: 8e5968134c395c0b4e6196506ef5502f5941f68754423bc885bc536a44c2ba33
5
5
  SHA512:
6
- metadata.gz: 86b1ecd19fe674a0464ad2032337bd5801c9a4a6ef5209515e11b4f38b16d14a896f1b577e0cd37d2399159a7bb7c7a1cb2b0c2cd0d4dac1925616940d22bcb1
7
- data.tar.gz: a663b02f76a6571a9f50ee00af2268be77c16d8c7606d01391e2d36c20d5e81fd0ba66b38f71226c9f643a403d8bd84b5162fa581447c68713c2a1748be6b324
6
+ metadata.gz: 54e1c9ec3c0b71566a8b6831690250d0eecc00f00155617dc39873c20d49a26ef57051ae04d7d3a1bbb3733f75fc3ccc0024b96c3b4bd086dafe48a1634cb8cf
7
+ data.tar.gz: 1bea7f70ccd0cba8dc488604d5f4a668d1bc602a2cb30d19058b195a17b2d9f1aaa5d72e21286e910570a89be0e78c25adac4bb45186a6f3f11ffcc81128b39c
data/README.md CHANGED
@@ -164,7 +164,7 @@ You can use DHS also to fetch records from GraphQL Endpoints:
164
164
 
165
165
  class Record < DHS::Record
166
166
 
167
- configuration items_keys: [:data, :ethereum, :address, 0, :balances]
167
+ configuration items_key: [:data, :ethereum, :address, 0, :balances]
168
168
 
169
169
  endpoint 'https://graphql.bitquery.io/',
170
170
  graphql: {
@@ -1386,6 +1386,51 @@ Sequentially:
1386
1386
  GET https://service.example.com/records?from_record_id=xcaoXBmuMyFFEcFDSgNgDQ&limit=100
1387
1387
  ```
1388
1388
 
1389
+ ##### Pagination strategy: next_offset
1390
+
1391
+ The `next_offset` strategy continuously follows in-response offset information to following pages until the last page is reached (indicated by next offset being either empty or 0).
1392
+
1393
+ *WARNING*
1394
+
1395
+ Loading all pages from a resource paginated with next_offset only can result in very poor performance, as pages can only be loaded sequentially!
1396
+
1397
+ ```ruby
1398
+ # app/models/record.rb
1399
+
1400
+ class Search < DHS::Record
1401
+ configuration pagination_strategy: 'next_offset'
1402
+
1403
+ endpoint '{+service}/assets'
1404
+ end
1405
+ ```
1406
+
1407
+ ```ruby
1408
+ # app/controllers/some_controller.rb
1409
+
1410
+ Record.all
1411
+
1412
+ ```
1413
+ ```
1414
+ GET https://service.example.com/assets?limit=100
1415
+ {
1416
+ items: [{...}, ...],
1417
+ limit: 10,
1418
+ next_offset: 29
1419
+ }
1420
+ GET https://service.example.com/assets?offset=29
1421
+ {
1422
+ items: [{...}, ...],
1423
+ limit: 10,
1424
+ next_offset: 39
1425
+ }
1426
+ GET https://service.example.com/assets?offset=39
1427
+ {
1428
+ items: [{...}, ...],
1429
+ limit: 10,
1430
+ next_offset: 0
1431
+ }
1432
+ ```
1433
+
1389
1434
  #### Pagination keys
1390
1435
 
1391
1436
  ##### limit_key
@@ -30,6 +30,8 @@ class DHS::Record
30
30
  DHS::Pagination::Start
31
31
  when :link
32
32
  DHS::Pagination::Link
33
+ when :next_offset
34
+ DHS::Pagination::NextOffset
33
35
  else
34
36
  DHS::Pagination::Offset
35
37
  end
@@ -43,7 +45,8 @@ class DHS::Record
43
45
  def paginated?(raw)
44
46
  raw.is_a?(Hash) && (
45
47
  raw.dig(*total_key).present? ||
46
- raw.dig(*limit_key(:body)).present?
48
+ raw.dig(*limit_key(:body)).present? ||
49
+ raw.dig(*pagination_key(:body)).present?
47
50
  )
48
51
  end
49
52
  end
@@ -237,7 +237,7 @@ class DHS::Record
237
237
  if pagination.parallel?
238
238
  load_and_merge_parallel_requests!(record, data, pagination, options)
239
239
  else
240
- load_and_merge_sequential_requests!(record, data, options, data._raw.dig(:next, :href), pagination)
240
+ load_and_merge_sequential_requests!(record, data, options, pagination)
241
241
  end
242
242
  end
243
243
 
@@ -249,13 +249,16 @@ class DHS::Record
249
249
  end
250
250
  end
251
251
 
252
- def load_and_merge_sequential_requests!(record, data, options, next_link, pagination)
253
- warn '[WARNING] You are loading all pages from a resource paginated with links only. As this is performed sequentially, it can result in very poor performance! (https://github.com/DePayFi/dhs#pagination-strategy-link).'
254
- while next_link.present?
255
- page_data = record.request(
256
- options.except(:all).merge(url: next_link)
257
- )
258
- next_link = page_data._raw.dig(:next, :href)
252
+ def load_and_merge_sequential_requests!(record, data, options, pagination)
253
+ warn '[WARNING] You are loading all pages from a resource paginated with sequential pagination.'
254
+ next_value = pagination.next(data._raw)
255
+ while next_value.present?
256
+ page_data = if next_value.is_a?(String) && next_value.match(/^http/)
257
+ record.request(options.except(:all).merge(url: next_value))
258
+ else
259
+ record.request(options.except(:all).merge(params: (options.dig(:params) || {}).merge(next_value) ))
260
+ end
261
+ next_value = pagination.next(page_data._raw)
259
262
  merge_batch_data_with_parent!(page_data, data, pagination)
260
263
  end
261
264
  end
@@ -7,6 +7,10 @@ class DHS::Pagination::Link < DHS::Pagination::Base
7
7
 
8
8
  alias count total
9
9
 
10
+ def next(current)
11
+ current.dig(:next, :href)
12
+ end
13
+
10
14
  def pages_left
11
15
  pages_left? ? 1 : 0
12
16
  end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ class DHS::Pagination::NextOffset < DHS::Pagination::Base
4
+
5
+ DEFAULT_OFFSET = 0
6
+
7
+ def total
8
+ data._raw.dig(*_record.items_key).count || 0
9
+ end
10
+ alias count total
11
+
12
+ def parallel?
13
+ false
14
+ end
15
+
16
+ def pages_left?
17
+ next_offset = data._raw.dig(*_record.pagination_key(:body))
18
+ next_offset.present? && !next_offset.zero?
19
+ end
20
+
21
+ def next(current)
22
+ next_value = current.dig(*_record.pagination_key(:body))
23
+ return if next_value.blank? || next_value.zero?
24
+ {
25
+ _record.pagination_key(:parameter) => current.dig(*_record.pagination_key(:body))
26
+ }
27
+ end
28
+ end
data/lib/dhs/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module DHS
4
- VERSION = '1.3.0'
4
+ VERSION = '1.4.0'
5
5
  end
data/lib/dhs.rb CHANGED
@@ -39,6 +39,7 @@ module DHS
39
39
  autoload :Page, 'dhs/pagination/page'
40
40
  autoload :TotalPages, 'dhs/pagination/total_pages'
41
41
  autoload :OffsetPage, 'dhs/pagination/offset_page'
42
+ autoload :NextOffset, 'dhs/pagination/next_offset'
42
43
  autoload :Start, 'dhs/pagination/start'
43
44
  autoload :Link, 'dhs/pagination/link'
44
45
  end
@@ -88,11 +88,13 @@ describe 'main graphql support' do
88
88
  end
89
89
 
90
90
  before do
91
+ DHC.config.placeholder('bitquery', 'https://graphql.bitquery.io/')
92
+
91
93
  class Record < DHS::Record
92
94
 
93
95
  configuration items_key: [:data, :ethereum, :address, 0, :balances]
94
96
 
95
- endpoint 'https://graphql.bitquery.io/',
97
+ endpoint '{+bitquery}',
96
98
  graphql: {
97
99
  query: %{
98
100
  query ($network: EthereumNetwork!, $address: String!) {
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rails_helper'
4
+
5
+ describe DHS::Record do
6
+
7
+ context 'pagination' do
8
+
9
+ def stub_api_request(items: [], offset: nil, next_offset:)
10
+ stub_request(:get, ["http://depay.fi/v2/transactions?limit=100", offset ? "offset=#{offset}" : nil].compact.join('&'))
11
+ .to_return(body: { items: items, next_offset: next_offset }.to_json)
12
+ end
13
+
14
+ let!(:requests) do
15
+ stub_api_request(items: (0...100).to_a, next_offset: 99)
16
+ stub_api_request(items: (100...200).to_a, offset: 99, next_offset: 199)
17
+ stub_api_request(items: (200...300).to_a, offset: 199, next_offset: 0)
18
+ end
19
+
20
+ before do
21
+ class Transaction < DHS::Record
22
+ configuration pagination_strategy: :next_offset, pagination_key: { body: :next_offset, parameter: :offset }
23
+
24
+ endpoint 'http://depay.fi/v2/transactions'
25
+ end
26
+ end
27
+
28
+ it 'fetches all the pages' do
29
+ transactions = Transaction.all.fetch
30
+ expect(transactions.to_a).to eq (0...300).to_a
31
+ end
32
+ end
33
+ end
@@ -167,7 +167,7 @@ describe DHS::Record do
167
167
  .includes(:users, contracts: :products)
168
168
  .find(1)
169
169
  end).to output(
170
- %r{\[WARNING\] You are loading all pages from a resource paginated with links only. As this is performed sequentially, it can result in very poor performance! \(https://github.com/DePayFi/dhs#pagination-strategy-link\).}
170
+ %r{\[WARNING\] You are loading all pages from a resource paginated with sequential pagination.}
171
171
  ).to_stderr
172
172
 
173
173
  expect(customer.users.length).to eq amount_of_users
@@ -348,7 +348,7 @@ describe DHS::Record do
348
348
  expect(lambda do
349
349
  all = Record.all.fetch
350
350
  end).to output(
351
- %r{\[WARNING\] You are loading all pages from a resource paginated with links only. As this is performed sequentially, it can result in very poor performance! \(https://github.com/DePayFi/dhs#pagination-strategy-link\).}
351
+ %r{\[WARNING\] You are loading all pages from a resource paginated with sequential pagination.}
352
352
  ).to_stderr
353
353
 
354
354
  expect(all).to be_kind_of Record
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dhs
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.0
4
+ version: 1.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - https://github.com/DePayFi/dhs/graphs/contributors
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-12-21 00:00:00.000000000 Z
11
+ date: 2023-04-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activemodel
@@ -327,6 +327,7 @@ files:
327
327
  - lib/dhs/item.rb
328
328
  - lib/dhs/pagination/base.rb
329
329
  - lib/dhs/pagination/link.rb
330
+ - lib/dhs/pagination/next_offset.rb
330
331
  - lib/dhs/pagination/offset.rb
331
332
  - lib/dhs/pagination/offset_page.rb
332
333
  - lib/dhs/pagination/page.rb
@@ -466,6 +467,7 @@ files:
466
467
  - spec/pagination/link/pages_left_spec.rb
467
468
  - spec/pagination/link/parallel_spec.rb
468
469
  - spec/pagination/link/total_spec.rb
470
+ - spec/pagination/next_offset_spec.rb
469
471
  - spec/pagination/offset/pages_left_spec.rb
470
472
  - spec/pagination/offset_page_spec.rb
471
473
  - spec/pagination/parameters_spec.rb
@@ -578,7 +580,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
578
580
  version: '0'
579
581
  requirements:
580
582
  - Ruby >= 2.7.2
581
- rubygems_version: 3.2.22
583
+ rubygems_version: 3.2.33
582
584
  signing_key:
583
585
  specification_version: 4
584
586
  summary: 'REST services accelerator: Rails gem providing an easy, active-record-like
@@ -703,6 +705,7 @@ test_files:
703
705
  - spec/pagination/link/pages_left_spec.rb
704
706
  - spec/pagination/link/parallel_spec.rb
705
707
  - spec/pagination/link/total_spec.rb
708
+ - spec/pagination/next_offset_spec.rb
706
709
  - spec/pagination/offset/pages_left_spec.rb
707
710
  - spec/pagination/offset_page_spec.rb
708
711
  - spec/pagination/parameters_spec.rb