dhs 1.3.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1ed15754a20492dd70e8d58fba832f6db2c747d4f13d04ad23457ab72fd81eda
-  data.tar.gz: 2626f410568a801f540f592e32c8f2f6ca5f9c21fa9e1a74229d9f0866139f21
+  metadata.gz: 690186fffb1221bbc432b5a502bdfecb895b1f117e42602e6bef6b2c55cd7bd1
+  data.tar.gz: 7f544c63db8a4db092a6c3980b3cb2f437879876db145a6ac7d8c6cfc7647a05
 SHA512:
-  metadata.gz: 86b1ecd19fe674a0464ad2032337bd5801c9a4a6ef5209515e11b4f38b16d14a896f1b577e0cd37d2399159a7bb7c7a1cb2b0c2cd0d4dac1925616940d22bcb1
-  data.tar.gz: a663b02f76a6571a9f50ee00af2268be77c16d8c7606d01391e2d36c20d5e81fd0ba66b38f71226c9f643a403d8bd84b5162fa581447c68713c2a1748be6b324
+  metadata.gz: '087dacf286f24b6860419759c5ea7cd443d69e2cd8f4900b2ee17fdf93d2d9f55918534ec0aff8294e6a4906c2dad51d3c4eb2b6ad8b3f19111a4133ed15f6c5'
+  data.tar.gz: f14b0dad8dffb72c65cd6b785469a59c590a99187639e554865e99bcf8e696434f262c4e95d64b1241bfd997180ef278ce2f0f0f5b91c6e5f11a673466f7eff8

data/README.md

@@ -164,7 +164,7 @@ You can use DHS also to fetch records from GraphQL Endpoints:
 
 class Record < DHS::Record
 
-  configuration items_keys: [:data, :ethereum, :address, 0, :balances]
+  configuration items_key: [:data, :ethereum, :address, 0, :balances]
 
   endpoint 'https://graphql.bitquery.io/',
     graphql: {
@@ -1386,6 +1386,51 @@ Sequentially:
   GET https://service.example.com/records?from_record_id=xcaoXBmuMyFFEcFDSgNgDQ&limit=100
 ```
 
+##### Pagination strategy: next_offset
+
+The `next_offset` strategy continuously follows in-response offset information to following pages until the last page is reached (indicated by next offset being either empty or 0).
+
+*WARNING*
+
+Loading all pages from a resource paginated with next_offset only can result in very poor performance, as pages can only be loaded sequentially!
+
+```ruby
+# app/models/record.rb
+
+class Search < DHS::Record
+  configuration pagination_strategy: 'next_offset'
+
+  endpoint '{+service}/assets'
+end
+```
+
+```ruby
+# app/controllers/some_controller.rb
+
+Record.all
+
+```
+```
+GET https://service.example.com/assets?limit=100
+{
+  items: [{...}, ...],
+  limit: 10,
+  next_offset: 29
+}
+GET https://service.example.com/assets?offset=29
+{
+  items: [{...}, ...],
+  limit: 10,
+  next_offset: 39
+}
+GET https://service.example.com/assets?offset=39
+{
+  items: [{...}, ...],
+  limit: 10,
+  next_offset: 0
+}
+```
+
 #### Pagination keys
 
 ##### limit_key

data/lib/dhs/concerns/record/pagination.rb

@@ -30,6 +30,8 @@ class DHS::Record
           DHS::Pagination::Start
         when :link
           DHS::Pagination::Link
+        when :next_offset
+          DHS::Pagination::NextOffset
         else
           DHS::Pagination::Offset
         end
@@ -43,7 +45,8 @@ class DHS::Record
       def paginated?(raw)
         raw.is_a?(Hash) && (
           raw.dig(*total_key).present? ||
-          raw.dig(*limit_key(:body)).present?
+          raw.dig(*limit_key(:body)).present? ||
+          raw.dig(*pagination_key(:body)).present?
         )
       end
     end

data/lib/dhs/concerns/record/request.rb

@@ -204,7 +204,7 @@ class DHS::Record
       end
 
       def load_and_merge_not_paginated_collection!(data, options)
-        return if data.length.zero?
+        return if data.empty?
         options = options.is_a?(Hash) ? options : {}
         limit = options.dig(:params, limit_key(:parameter)) || pagination_class::DEFAULT_LIMIT
         offset = options.dig(:params, pagination_key(:parameter)) || pagination_class::DEFAULT_OFFSET
@@ -230,14 +230,14 @@ class DHS::Record
       end
 
       def load_and_merge_paginated_collection!(data, options)
-        set_nested_data(data._raw, limit_key(:body), data.length) if data._raw.dig(*limit_key(:body)).blank? && !data.length.zero?
+        set_nested_data(data._raw, limit_key(:body), data.length) if data._raw.dig(*limit_key(:body)).blank? && !data.empty?
         pagination = data._record.pagination(data)
         return data unless pagination.pages_left?
         record = data._record
         if pagination.parallel?
           load_and_merge_parallel_requests!(record, data, pagination, options)
         else
-          load_and_merge_sequential_requests!(record, data, options, data._raw.dig(:next, :href), pagination)
+          load_and_merge_sequential_requests!(record, data, options, pagination)
         end
       end
 
@@ -249,13 +249,16 @@ class DHS::Record
         end
       end
 
-      def load_and_merge_sequential_requests!(record, data, options, next_link, pagination)
-        warn '[WARNING] You are loading all pages from a resource paginated with links only. As this is performed sequentially, it can result in very poor performance! (https://github.com/DePayFi/dhs#pagination-strategy-link).'
-        while next_link.present?
-          page_data = record.request(
-            options.except(:all).merge(url: next_link)
-          )
-          next_link = page_data._raw.dig(:next, :href)
+      def load_and_merge_sequential_requests!(record, data, options, pagination)
+        warn '[WARNING] You are loading all pages from a resource paginated with sequential pagination.'
+        next_value = pagination.next(data._raw)
+        while next_value.present?
+          page_data = if next_value.is_a?(String) && next_value.match(/^http/)
+             record.request(options.except(:all).merge(url: next_value))
+          else
+            record.request(options.except(:all).merge(params: (options.dig(:params) || {}).merge(next_value)))
+          end
+          next_value = pagination.next(page_data._raw)
           merge_batch_data_with_parent!(page_data, data, pagination)
         end
       end

data/lib/dhs/pagination/link.rb

@@ -7,6 +7,10 @@ class DHS::Pagination::Link < DHS::Pagination::Base
 
   alias count total
 
+  def next(current)
+    current.dig(:next, :href)
+  end
+
   def pages_left
     pages_left? ? 1 : 0
   end

data/lib/dhs/pagination/next_offset.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class DHS::Pagination::NextOffset < DHS::Pagination::Base
+
+  DEFAULT_OFFSET = 0
+
+  def total
+    data._raw.dig(*_record.items_key).count || 0
+  end
+  alias count total
+
+  def parallel?
+    false
+  end
+
+  def pages_left?
+    next_offset = data._raw.dig(*_record.pagination_key(:body))
+    next_offset.present? && !next_offset.zero?
+  end
+
+  def next(current)
+    next_value = current.dig(*_record.pagination_key(:body))
+    return if next_value.blank? || next_value.zero?
+    {
+      _record.pagination_key(:parameter) => current.dig(*_record.pagination_key(:body))
+    }
+  end
+end

data/lib/dhs/problems/errors.rb

@@ -54,7 +54,7 @@ module DHS::Problems
     end
 
     def messages_from_response(response = nil)
-      return {} if !response || !response.body.is_a?(String) || response.body.length.zero?
+      return {} if !response || !response.body.is_a?(String) || response.body.empty?
       json = JSON.parse(response.body)
       parse_messages(json)
     end

data/lib/dhs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DHS
-  VERSION = '1.3.0'
+  VERSION = '1.4.1'
 end

data/lib/dhs.rb

@@ -39,6 +39,7 @@ module DHS
     autoload :Page, 'dhs/pagination/page'
     autoload :TotalPages, 'dhs/pagination/total_pages'
     autoload :OffsetPage, 'dhs/pagination/offset_page'
+    autoload :NextOffset, 'dhs/pagination/next_offset'
     autoload :Start, 'dhs/pagination/start'
     autoload :Link, 'dhs/pagination/link'
   end

data/spec/concerns/record/request_spec.rb

@@ -19,15 +19,15 @@ describe DHS::Record::Request do
 
     it 'calls correct prepare method for a Hash' do
       expect(subject).to receive(:prepare_option_for_include_all_request!)
-        .with(abc: 'def').and_return('ignore')
-      expect(subject.send(:prepare_options_for_include_all_request!, abc: 'def')).to eq(abc: 'def')
+        .with({ abc: 'def' }).and_return('ignore')
+      expect(subject.send(:prepare_options_for_include_all_request!, { abc: 'def' })).to eq(abc: 'def')
     end
 
     it 'calls correct prepare method for a Hash (collection)' do
       expect(subject).to receive(:prepare_option_for_include_all_request!)
-        .with(abc: 'def').and_return('ignore')
+        .with({ abc: 'def' }).and_return('ignore')
       expect(subject).to receive(:prepare_option_for_include_all_request!)
-        .with(hij: 'kel').and_return('ignore')
+        .with({ hij: 'kel' }).and_return('ignore')
       expect(subject.send(:prepare_options_for_include_all_request!, [{ abc: 'def' }, { hij: 'kel' }]))
         .to eq([{ abc: 'def' }, { hij: 'kel' }])
     end

data/spec/graphql/main_spec.rb

@@ -88,11 +88,13 @@ describe 'main graphql support' do
   end
 
   before do
+    DHC.config.placeholder('bitquery', 'https://graphql.bitquery.io/')
+
     class Record < DHS::Record
 
       configuration items_key: [:data, :ethereum, :address, 0, :balances]
 
-      endpoint 'https://graphql.bitquery.io/',
+      endpoint '{+bitquery}',
         graphql: {
           query: %{
             query ($network: EthereumNetwork!, $address: String!) {

data/spec/pagination/next_offset_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe DHS::Record do
+  context 'pagination' do
+    def stub_api_request(next_offset:, items: [], offset: nil)
+      stub_request(:get, ['http://depay.fi/v2/transactions?limit=100', offset ? "offset=#{offset}" : nil].compact.join('&'))
+        .to_return(body: { items: items, next_offset: next_offset }.to_json)
+    end
+
+    let!(:requests) do
+      stub_api_request(items: (0...100).to_a, next_offset: 99)
+      stub_api_request(items: (100...200).to_a, offset: 99, next_offset: 199)
+      stub_api_request(items: (200...300).to_a, offset: 199, next_offset: 0)
+    end
+
+    before do
+      class Transaction < DHS::Record
+        configuration pagination_strategy: :next_offset, pagination_key: { body: :next_offset, parameter: :offset }
+
+        endpoint 'http://depay.fi/v2/transactions'
+      end
+    end
+
+    it 'fetches all the pages' do
+      transactions = Transaction.all.fetch
+      expect(transactions.to_a).to eq (0...300).to_a
+    end
+  end
+end

data/spec/record/endpoint_options_spec.rb

@@ -16,12 +16,12 @@ describe DHS::Record do
 
     it 'uses the options that are configured for an endpoint' do
       expect(DHC).to receive(:request)
-        .with(
-          cache_expires_in: 1.day,
-          retry: 2,
-          cache: true,
-          url: 'backend/v2/feedbacks/1'
-        ).and_call_original
+        .with({
+                cache_expires_in: 1.day,
+                retry: 2,
+                cache: true,
+                url: 'backend/v2/feedbacks/1'
+              }).and_call_original
 
       stub_request(:get, 'http://backend/v2/feedbacks/1').to_return(status: 200)
       Record.find(1)

data/spec/record/includes_spec.rb

@@ -167,7 +167,7 @@ describe DHS::Record do
           .includes(:users, contracts: :products)
           .find(1)
       end).to output(
-        %r{\[WARNING\] You are loading all pages from a resource paginated with links only. As this is performed sequentially, it can result in very poor performance! \(https://github.com/DePayFi/dhs#pagination-strategy-link\).}
+        %r{\[WARNING\] You are loading all pages from a resource paginated with sequential pagination.}
       ).to_stderr
 
       expect(customer.users.length).to eq amount_of_users
@@ -214,12 +214,12 @@ describe DHS::Record do
 
       it 'overwrites existing pagination paramters if they are already contained in a string' do
         expect(DHC).to receive(:request)
-          .with(url: 'http://datastore/customers/1').and_call_original
+          .with({ url: 'http://datastore/customers/1' }).and_call_original
 
         expect(DHC).to receive(:request)
-          .with(url: 'http://datastore/customers/1/contracts',
-                all: true,
-                params: { limit: 100 }).and_call_original
+          .with({ url: 'http://datastore/customers/1/contracts',
+                  all: true,
+                  params: { limit: 100 } }).and_call_original
 
         expect(DHC).to receive(:request)
           .with([{ url: 'http://datastore/customers/1/contracts',

data/spec/record/paginatable_collection_spec.rb

@@ -348,7 +348,7 @@ describe DHS::Record do
       expect(lambda do
         all = Record.all.fetch
       end).to output(
-        %r{\[WARNING\] You are loading all pages from a resource paginated with links only. As this is performed sequentially, it can result in very poor performance! \(https://github.com/DePayFi/dhs#pagination-strategy-link\).}
+        %r{\[WARNING\] You are loading all pages from a resource paginated with sequential pagination.}
       ).to_stderr
 
       expect(all).to be_kind_of Record

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dhs
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.1
 platform: ruby
 authors:
 - https://github.com/DePayFi/dhs/graphs/contributors
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-21 00:00:00.000000000 Z
+date: 2023-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -327,6 +327,7 @@ files:
 - lib/dhs/item.rb
 - lib/dhs/pagination/base.rb
 - lib/dhs/pagination/link.rb
+- lib/dhs/pagination/next_offset.rb
 - lib/dhs/pagination/offset.rb
 - lib/dhs/pagination/offset_page.rb
 - lib/dhs/pagination/page.rb
@@ -466,6 +467,7 @@ files:
 - spec/pagination/link/pages_left_spec.rb
 - spec/pagination/link/parallel_spec.rb
 - spec/pagination/link/total_spec.rb
+- spec/pagination/next_offset_spec.rb
 - spec/pagination/offset/pages_left_spec.rb
 - spec/pagination/offset_page_spec.rb
 - spec/pagination/parameters_spec.rb
@@ -578,7 +580,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements:
 - Ruby >= 2.7.2
-rubygems_version: 3.2.22
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: 'REST services accelerator: Rails gem providing an easy, active-record-like
@@ -703,6 +705,7 @@ test_files:
 - spec/pagination/link/pages_left_spec.rb
 - spec/pagination/link/parallel_spec.rb
 - spec/pagination/link/total_spec.rb
+- spec/pagination/next_offset_spec.rb
 - spec/pagination/offset/pages_left_spec.rb
 - spec/pagination/offset_page_spec.rb
 - spec/pagination/parameters_spec.rb