dhs 1.4.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 690186fffb1221bbc432b5a502bdfecb895b1f117e42602e6bef6b2c55cd7bd1
-  data.tar.gz: 7f544c63db8a4db092a6c3980b3cb2f437879876db145a6ac7d8c6cfc7647a05
+  metadata.gz: c02b8006074cd3482fac996fcf904ed9e904d6ab8f8b504afcd9c27318dd5724
+  data.tar.gz: 2ce157d1fd6f0333d9638516e650f2ef040b6f08145664e4f3e8b041f908ca66
 SHA512:
-  metadata.gz: '087dacf286f24b6860419759c5ea7cd443d69e2cd8f4900b2ee17fdf93d2d9f55918534ec0aff8294e6a4906c2dad51d3c4eb2b6ad8b3f19111a4133ed15f6c5'
-  data.tar.gz: f14b0dad8dffb72c65cd6b785469a59c590a99187639e554865e99bcf8e696434f262c4e95d64b1241bfd997180ef278ce2f0f0f5b91c6e5f11a673466f7eff8
+  metadata.gz: 8975d2399f0a7a63066fdf8e23b96dbf2b9809bd9e75efc73a85779b1a32e97eae997dce4573222ed5d7b1556ca9dc2276d12eee5dddfa911d71698ae26b10d0
+  data.tar.gz: 9d00278274df289db27965f24a6bc2a106d46e64ea09814b8129d5aeae15f6e0e2d7e7cde37a7f7b49447142ecc52a960ea458bef1cb9f46396ec9457c2acd97

data/README.md

@@ -1431,6 +1431,50 @@ GET https://service.example.com/assets?offset=39
 }
 ```
 
+##### Pagination strategy: next_parameter
+
+The `next_parameter` strategy continuously follows in-response offset parameter information to following pages until the last page is reached (indicated by next parameter being empty or unset).
+
+*WARNING*
+
+Loading all pages from a resource paginated with next_parameter only can result in very poor performance, as pages can only be loaded sequentially!
+
+```ruby
+# app/models/record.rb
+
+class Search < DHS::Record
+  configuration pagination_strategy: 'next_parameter'
+
+  endpoint '{+service}/assets'
+end
+```
+
+```ruby
+# app/controllers/some_controller.rb
+
+Record.all
+
+```
+```
+GET https://service.example.com/assets?limit=100
+{
+  items: [{...}, ...],
+  limit: 10,
+  next: "LXBrPTIzMzE4NDE0"
+}
+GET https://service.example.com/assets?next=LXBrPTIzMzE4NDE0
+{
+  items: [{...}, ...],
+  limit: 10,
+  next: "ASDASdads9123"
+}
+GET https://service.example.com/assets?next=ASDASdads9123
+{
+  items: [{...}, ...],
+  limit: 10
+}
+```
+
 #### Pagination keys
 
 ##### limit_key

data/lib/dhs/concerns/record/pagination.rb

@@ -32,6 +32,8 @@ class DHS::Record
           DHS::Pagination::Link
         when :next_offset
           DHS::Pagination::NextOffset
+        when :next_parameter
+          DHS::Pagination::NextParameter
         else
           DHS::Pagination::Offset
         end

data/lib/dhs/concerns/record/request.rb

@@ -387,6 +387,7 @@ class DHS::Record
       end
 
       def merge_batch_data_with_parent!(batch_data, parent_data, pagination = nil)
+        return if batch_data.raw_items.blank?
         parent_data.concat(input: parent_data._raw, items: batch_data.raw_items, record: self)
         return if pagination.present? && pagination.is_a?(DHS::Pagination::Link)
         [limit_key(:body), total_key, pagination_key(:body)].each do |pagination_attribute|

data/lib/dhs/pagination/next_parameter.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class DHS::Pagination::NextParameter < DHS::Pagination::Base
+
+  DEFAULT_OFFSET = nil
+
+  def total
+    data._raw.dig(*_record.items_key).count || 0
+  end
+  alias count total
+
+  def self.next_offset(offset, limit)
+    nil
+  end
+
+  def parallel?
+    false
+  end
+
+  def pages_left?
+    next_offset = data._raw.dig(*_record.pagination_key(:body))
+    next_offset.present? && !next_offset.blank?
+  end
+
+  def next(current)
+    next_value = current.dig(*_record.pagination_key(:body))
+    return if next_value.blank? || next_value.blank?
+    {
+      _record.pagination_key(:parameter) => current.dig(*_record.pagination_key(:body))
+    }
+  end
+end

data/lib/dhs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DHS
-  VERSION = '1.4.1'
+  VERSION = '1.5.0'
 end

data/lib/dhs.rb

@@ -40,6 +40,7 @@ module DHS
     autoload :TotalPages, 'dhs/pagination/total_pages'
     autoload :OffsetPage, 'dhs/pagination/offset_page'
     autoload :NextOffset, 'dhs/pagination/next_offset'
+    autoload :NextParameter, 'dhs/pagination/next_parameter'
     autoload :Start, 'dhs/pagination/start'
     autoload :Link, 'dhs/pagination/link'
   end

data/spec/pagination/next_offset_spec.rb

@@ -3,7 +3,7 @@
 require 'rails_helper'
 
 describe DHS::Record do
-  context 'pagination' do
+  context 'next offset pagination' do
     def stub_api_request(next_offset:, items: [], offset: nil)
       stub_request(:get, ['http://depay.fi/v2/transactions?limit=100', offset ? "offset=#{offset}" : nil].compact.join('&'))
         .to_return(body: { items: items, next_offset: next_offset }.to_json)
@@ -28,4 +28,56 @@ describe DHS::Record do
       expect(transactions.to_a).to eq (0...300).to_a
     end
   end
+
+  context 'next offset pagination with items configuration' do
+    def stub_api_request(next_offset:, items: [], offset: nil)
+      stub_request(:get, ['http://depay.fi/v2/transactions?limit=100', offset ? "offset=#{offset}" : nil].compact.join('&'))
+        .to_return(body: { assets: items, next_offset: next_offset }.to_json)
+    end
+
+    let!(:requests) do
+      stub_api_request(items: (0...100).to_a, next_offset: 99)
+      stub_api_request(items: (100...200).to_a, offset: 99, next_offset: 199)
+      stub_api_request(items: (200...300).to_a, offset: 199, next_offset: 0)
+    end
+
+    before do
+      class Transaction < DHS::Record
+        configuration items_key: :assets, pagination_strategy: :next_offset, pagination_key: { body: :next_offset, parameter: :offset }
+
+        endpoint 'http://depay.fi/v2/transactions'
+      end
+    end
+
+    it 'fetches and merges all the assets/items' do
+      transactions = Transaction.all.fetch
+      expect(transactions.to_a).to eq (0...300).to_a
+    end
+  end
+
+  context 'next offset pagination with items configuration and nil response' do
+    def stub_api_request(next_offset:, items: [], offset: nil)
+      stub_request(:get, ['http://depay.fi/v2/transactions?limit=100', offset ? "offset=#{offset}" : nil].compact.join('&'))
+        .to_return(body: { assets: items, next_offset: next_offset }.to_json)
+    end
+
+    let!(:requests) do
+      stub_api_request(items: (0...100).to_a, next_offset: 99)
+      stub_api_request(items: (100...200).to_a, offset: 99, next_offset: 199)
+      stub_api_request(items: nil, offset: 199, next_offset: 0)
+    end
+
+    before do
+      class Transaction < DHS::Record
+        configuration items_key: :assets, pagination_strategy: :next_offset, pagination_key: { body: :next_offset, parameter: :offset }
+
+        endpoint 'http://depay.fi/v2/transactions'
+      end
+    end
+
+    it 'fetches and merges all the assets/items' do
+      transactions = Transaction.all.fetch
+      expect(transactions.to_a).to eq (0...200).to_a
+    end
+  end
 end

data/spec/pagination/next_parameter_spec.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe DHS::Record do
+  context 'next parameter pagination' do
+    
+    def stub_api_request(this_parameter:, next_parameter:, items: [])
+      stub_request(:get, ["http://depay.fi/v2/transactions?limit=100", this_parameter ? "next=#{this_parameter}" : nil].compact.join('&'))
+        .to_return(body: { items: items, next: next_parameter }.to_json)
+    end
+
+    let!(:requests) do
+      stub_api_request(items: (0...100).to_a, this_parameter: nil, next_parameter: 'NEXT1')
+      stub_api_request(items: (100...200).to_a, this_parameter: 'NEXT1', next_parameter: 'NEXT2')
+      stub_api_request(items: nil, this_parameter: 'NEXT2', next_parameter: nil)
+    end
+
+    before do
+      class Transaction < DHS::Record
+        configuration pagination_strategy: :next_parameter, pagination_key: { body: :next, parameter: :next }
+
+        endpoint 'http://depay.fi/v2/transactions'
+      end
+    end
+
+    it 'fetches all the pages' do
+      transactions = Transaction.all.fetch
+      expect(transactions.to_a).to eq (0...200).to_a
+    end
+  end
+
+  context 'next parameter pagination with items configuration and nil response' do
+
+    def stub_api_request(this_parameter:, next_parameter:, items: [])
+      stub_request(:get, ["http://depay.fi/v2/transactions?limit=100", this_parameter ? "next=#{this_parameter}" : nil].compact.join('&'))
+        .to_return(body: { assets: items, next: next_parameter }.to_json)
+    end
+
+    let!(:requests) do
+      stub_api_request(items: (0...100).to_a, this_parameter: nil, next_parameter: 'NEXT1')
+      stub_api_request(items: (100...200).to_a, this_parameter: 'NEXT1', next_parameter: 'NEXT2')
+      stub_api_request(items: nil, this_parameter: 'NEXT2', next_parameter: nil)
+    end
+
+    before do
+      class Transaction < DHS::Record
+        configuration items_key: :assets, pagination_strategy: :next_parameter, pagination_key: { body: :next, parameter: :next }
+
+        endpoint 'http://depay.fi/v2/transactions'
+      end
+    end
+
+    it 'fetches and merges all the assets/items' do
+      transactions = Transaction.all.fetch
+      expect(transactions.to_a).to eq (0...200).to_a
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dhs
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.5.0
 platform: ruby
 authors:
 - https://github.com/DePayFi/dhs/graphs/contributors
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-25 00:00:00.000000000 Z
+date: 2023-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -328,6 +328,7 @@ files:
 - lib/dhs/pagination/base.rb
 - lib/dhs/pagination/link.rb
 - lib/dhs/pagination/next_offset.rb
+- lib/dhs/pagination/next_parameter.rb
 - lib/dhs/pagination/offset.rb
 - lib/dhs/pagination/offset_page.rb
 - lib/dhs/pagination/page.rb
@@ -468,6 +469,7 @@ files:
 - spec/pagination/link/parallel_spec.rb
 - spec/pagination/link/total_spec.rb
 - spec/pagination/next_offset_spec.rb
+- spec/pagination/next_parameter_spec.rb
 - spec/pagination/offset/pages_left_spec.rb
 - spec/pagination/offset_page_spec.rb
 - spec/pagination/parameters_spec.rb
@@ -706,6 +708,7 @@ test_files:
 - spec/pagination/link/parallel_spec.rb
 - spec/pagination/link/total_spec.rb
 - spec/pagination/next_offset_spec.rb
+- spec/pagination/next_parameter_spec.rb
 - spec/pagination/offset/pages_left_spec.rb
 - spec/pagination/offset_page_spec.rb
 - spec/pagination/parameters_spec.rb