dhs 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1ed15754a20492dd70e8d58fba832f6db2c747d4f13d04ad23457ab72fd81eda
-  data.tar.gz: 2626f410568a801f540f592e32c8f2f6ca5f9c21fa9e1a74229d9f0866139f21
+  metadata.gz: c31dbca2778853f9bafb3a74e2c7ef830d3897c0caa78b66e5bb58203fc8bb11
+  data.tar.gz: 8e5968134c395c0b4e6196506ef5502f5941f68754423bc885bc536a44c2ba33
 SHA512:
-  metadata.gz: 86b1ecd19fe674a0464ad2032337bd5801c9a4a6ef5209515e11b4f38b16d14a896f1b577e0cd37d2399159a7bb7c7a1cb2b0c2cd0d4dac1925616940d22bcb1
-  data.tar.gz: a663b02f76a6571a9f50ee00af2268be77c16d8c7606d01391e2d36c20d5e81fd0ba66b38f71226c9f643a403d8bd84b5162fa581447c68713c2a1748be6b324
+  metadata.gz: 54e1c9ec3c0b71566a8b6831690250d0eecc00f00155617dc39873c20d49a26ef57051ae04d7d3a1bbb3733f75fc3ccc0024b96c3b4bd086dafe48a1634cb8cf
+  data.tar.gz: 1bea7f70ccd0cba8dc488604d5f4a668d1bc602a2cb30d19058b195a17b2d9f1aaa5d72e21286e910570a89be0e78c25adac4bb45186a6f3f11ffcc81128b39c

data/README.md

@@ -164,7 +164,7 @@ You can use DHS also to fetch records from GraphQL Endpoints:
 
 class Record < DHS::Record
 
-  configuration items_keys: [:data, :ethereum, :address, 0, :balances]
+  configuration items_key: [:data, :ethereum, :address, 0, :balances]
 
   endpoint 'https://graphql.bitquery.io/',
     graphql: {
@@ -1386,6 +1386,51 @@ Sequentially:
   GET https://service.example.com/records?from_record_id=xcaoXBmuMyFFEcFDSgNgDQ&limit=100
 ```
 
+##### Pagination strategy: next_offset
+
+The `next_offset` strategy continuously follows in-response offset information to following pages until the last page is reached (indicated by next offset being either empty or 0).
+
+*WARNING*
+
+Loading all pages from a resource paginated with next_offset only can result in very poor performance, as pages can only be loaded sequentially!
+
+```ruby
+# app/models/record.rb
+
+class Search < DHS::Record
+  configuration pagination_strategy: 'next_offset'
+
+  endpoint '{+service}/assets'
+end
+```
+
+```ruby
+# app/controllers/some_controller.rb
+
+Record.all
+
+```
+```
+GET https://service.example.com/assets?limit=100
+{
+  items: [{...}, ...],
+  limit: 10,
+  next_offset: 29
+}
+GET https://service.example.com/assets?offset=29
+{
+  items: [{...}, ...],
+  limit: 10,
+  next_offset: 39
+}
+GET https://service.example.com/assets?offset=39
+{
+  items: [{...}, ...],
+  limit: 10,
+  next_offset: 0
+}
+```
+
 #### Pagination keys
 
 ##### limit_key

data/lib/dhs/concerns/record/pagination.rb

@@ -30,6 +30,8 @@ class DHS::Record
           DHS::Pagination::Start
         when :link
           DHS::Pagination::Link
+        when :next_offset
+          DHS::Pagination::NextOffset
         else
           DHS::Pagination::Offset
         end
@@ -43,7 +45,8 @@ class DHS::Record
       def paginated?(raw)
         raw.is_a?(Hash) && (
           raw.dig(*total_key).present? ||
-          raw.dig(*limit_key(:body)).present?
+          raw.dig(*limit_key(:body)).present? ||
+          raw.dig(*pagination_key(:body)).present?
         )
       end
     end

data/lib/dhs/concerns/record/request.rb

@@ -237,7 +237,7 @@ class DHS::Record
         if pagination.parallel?
           load_and_merge_parallel_requests!(record, data, pagination, options)
         else
-          load_and_merge_sequential_requests!(record, data, options, data._raw.dig(:next, :href), pagination)
+          load_and_merge_sequential_requests!(record, data, options, pagination)
         end
       end
 
@@ -249,13 +249,16 @@ class DHS::Record
         end
       end
 
-      def load_and_merge_sequential_requests!(record, data, options, next_link, pagination)
-        warn '[WARNING] You are loading all pages from a resource paginated with links only. As this is performed sequentially, it can result in very poor performance! (https://github.com/DePayFi/dhs#pagination-strategy-link).'
-        while next_link.present?
-          page_data = record.request(
-            options.except(:all).merge(url: next_link)
-          )
-          next_link = page_data._raw.dig(:next, :href)
+      def load_and_merge_sequential_requests!(record, data, options, pagination)
+        warn '[WARNING] You are loading all pages from a resource paginated with sequential pagination.'
+        next_value = pagination.next(data._raw)
+        while next_value.present?
+          page_data = if next_value.is_a?(String) && next_value.match(/^http/)
+             record.request(options.except(:all).merge(url: next_value))
+          else
+            record.request(options.except(:all).merge(params: (options.dig(:params) || {}).merge(next_value) ))
+          end
+          next_value = pagination.next(page_data._raw)
           merge_batch_data_with_parent!(page_data, data, pagination)
         end
       end

data/lib/dhs/pagination/link.rb

@@ -7,6 +7,10 @@ class DHS::Pagination::Link < DHS::Pagination::Base
 
   alias count total
 
+  def next(current)
+    current.dig(:next, :href)
+  end
+
   def pages_left
     pages_left? ? 1 : 0
   end

data/lib/dhs/pagination/next_offset.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class DHS::Pagination::NextOffset < DHS::Pagination::Base
+
+  DEFAULT_OFFSET = 0
+  
+  def total
+    data._raw.dig(*_record.items_key).count || 0
+  end
+  alias count total
+
+  def parallel?
+    false
+  end
+
+  def pages_left?
+    next_offset = data._raw.dig(*_record.pagination_key(:body))
+    next_offset.present? && !next_offset.zero?
+  end
+
+  def next(current)
+    next_value = current.dig(*_record.pagination_key(:body))
+    return if next_value.blank? || next_value.zero?
+    {
+      _record.pagination_key(:parameter) => current.dig(*_record.pagination_key(:body))
+    }
+  end
+end

data/lib/dhs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DHS
-  VERSION = '1.3.0'
+  VERSION = '1.4.0'
 end

data/lib/dhs.rb

@@ -39,6 +39,7 @@ module DHS
     autoload :Page, 'dhs/pagination/page'
     autoload :TotalPages, 'dhs/pagination/total_pages'
     autoload :OffsetPage, 'dhs/pagination/offset_page'
+    autoload :NextOffset, 'dhs/pagination/next_offset'
     autoload :Start, 'dhs/pagination/start'
     autoload :Link, 'dhs/pagination/link'
   end

data/spec/graphql/main_spec.rb

@@ -88,11 +88,13 @@ describe 'main graphql support' do
   end
 
   before do
+    DHC.config.placeholder('bitquery', 'https://graphql.bitquery.io/')
+
     class Record < DHS::Record
 
       configuration items_key: [:data, :ethereum, :address, 0, :balances]
 
-      endpoint 'https://graphql.bitquery.io/',
+      endpoint '{+bitquery}',
         graphql: {
           query: %{
             query ($network: EthereumNetwork!, $address: String!) {

data/spec/pagination/next_offset_spec.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe DHS::Record do
+
+  context 'pagination' do
+
+    def stub_api_request(items: [], offset: nil, next_offset:)
+      stub_request(:get, ["http://depay.fi/v2/transactions?limit=100", offset ? "offset=#{offset}" : nil].compact.join('&'))
+        .to_return(body: { items: items, next_offset: next_offset }.to_json)
+    end
+
+    let!(:requests) do
+      stub_api_request(items: (0...100).to_a, next_offset: 99)
+      stub_api_request(items: (100...200).to_a, offset: 99, next_offset: 199)
+      stub_api_request(items: (200...300).to_a, offset: 199, next_offset: 0)
+    end
+
+    before do
+      class Transaction < DHS::Record
+        configuration pagination_strategy: :next_offset, pagination_key: { body: :next_offset, parameter: :offset }
+
+        endpoint 'http://depay.fi/v2/transactions'
+      end
+    end
+
+    it 'fetches all the pages' do
+      transactions = Transaction.all.fetch
+      expect(transactions.to_a).to eq (0...300).to_a
+    end
+  end
+end

data/spec/record/includes_spec.rb

@@ -167,7 +167,7 @@ describe DHS::Record do
           .includes(:users, contracts: :products)
           .find(1)
       end).to output(
-        %r{\[WARNING\] You are loading all pages from a resource paginated with links only. As this is performed sequentially, it can result in very poor performance! \(https://github.com/DePayFi/dhs#pagination-strategy-link\).}
+        %r{\[WARNING\] You are loading all pages from a resource paginated with sequential pagination.}
       ).to_stderr
 
       expect(customer.users.length).to eq amount_of_users

data/spec/record/paginatable_collection_spec.rb

@@ -348,7 +348,7 @@ describe DHS::Record do
       expect(lambda do
         all = Record.all.fetch
       end).to output(
-        %r{\[WARNING\] You are loading all pages from a resource paginated with links only. As this is performed sequentially, it can result in very poor performance! \(https://github.com/DePayFi/dhs#pagination-strategy-link\).}
+        %r{\[WARNING\] You are loading all pages from a resource paginated with sequential pagination.}
       ).to_stderr
 
       expect(all).to be_kind_of Record

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dhs
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - https://github.com/DePayFi/dhs/graphs/contributors
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-21 00:00:00.000000000 Z
+date: 2023-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -327,6 +327,7 @@ files:
 - lib/dhs/item.rb
 - lib/dhs/pagination/base.rb
 - lib/dhs/pagination/link.rb
+- lib/dhs/pagination/next_offset.rb
 - lib/dhs/pagination/offset.rb
 - lib/dhs/pagination/offset_page.rb
 - lib/dhs/pagination/page.rb
@@ -466,6 +467,7 @@ files:
 - spec/pagination/link/pages_left_spec.rb
 - spec/pagination/link/parallel_spec.rb
 - spec/pagination/link/total_spec.rb
+- spec/pagination/next_offset_spec.rb
 - spec/pagination/offset/pages_left_spec.rb
 - spec/pagination/offset_page_spec.rb
 - spec/pagination/parameters_spec.rb
@@ -578,7 +580,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements:
 - Ruby >= 2.7.2
-rubygems_version: 3.2.22
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: 'REST services accelerator: Rails gem providing an easy, active-record-like
@@ -703,6 +705,7 @@ test_files:
 - spec/pagination/link/pages_left_spec.rb
 - spec/pagination/link/parallel_spec.rb
 - spec/pagination/link/total_spec.rb
+- spec/pagination/next_offset_spec.rb
 - spec/pagination/offset/pages_left_spec.rb
 - spec/pagination/offset_page_spec.rb
 - spec/pagination/parameters_spec.rb