lhs 19.5.0 → 19.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f710860a3f8422148310faddde72364d0fbf9916c79109b3a00df7c48e2de12d
-  data.tar.gz: 19fbb0d10e8a240c8312ed97776d0c15f6b03ef2b2bac9b54f0b0ccdb7236a47
+  metadata.gz: 27484e8b67d86a110c1e880e53d42f605646754cd0fe7b52369a5dcc381f3d97
+  data.tar.gz: b359ed2464317011e78586fbc1804812b269fa6da8f053028d8bbca946acac18
 SHA512:
-  metadata.gz: 05ce1a8bab26b8a39cacdb47a0625d5e00d00a69d4ecd0cfc9950951690ccddf726e10e3de6fdea3dfe9aa61d98195a931b57dc3a26d1a1e7c1143c193f28c52
-  data.tar.gz: fe24c182642d2c5c61e842176a99c5fc1b53656471082c4cabf8edbe8d8d1d3219cd7e10e9db40e1d87e4fad67fdaefb4436c84e7c3ab1b6ba59d7c144ec160a
+  metadata.gz: 779e482a39d72b249c30ed2ab82ab9c26d5fcd922c5d8149bfc1d0bd2838daec87ad1ff170503f532962f6bb07e9806c932c00c630ecae77d4af600aa45f2e6c
+  data.tar.gz: 85ba8a79e1e6b0c8f66fd7c4592636140f61d8a12b1b2d86333bb5ef86f2a2f48a44aeeae51732e1fd54752bcd90456f6951539a59cc5c4f94f7d2a7a875bb95

data/README.md

@@ -1230,6 +1230,44 @@ In parallel:
   GET https://service.example.com/records?limit=100&startAt=201
 ```
 
+##### Pagination strategy: link
+
+The `link` strategy continuously follows in-response embedded links to following pages until the last page is reached (indicated by no more `next` link).
+
+*WARNING*
+
+Loading all pages from a resource paginated with links only can result in very poor performance, as pages can only be loaded sequentially!
+
+```ruby
+# app/models/record.rb
+
+class Search < LHS::Record
+  configuration pagination_strategy: 'link'
+
+  endpoint '{+service}/search'
+end
+```
+
+```ruby
+# app/controllers/some_controller.rb
+
+Record.all
+
+```
+```
+GET https://service.example.com/records?limit=100
+{
+  items: [{...}, ...],
+  limit: 100,
+  next: {
+    href: 'https://service.example.com/records?from_record_id=p62qM5p0NK_qryO52Ze-eg&limit=100'
+  }
+}
+Sequentially:
+  GET https://service.example.com/records?from_record_id=p62qM5p0NK_qryO52Ze-eg&limit=100
+  GET https://service.example.com/records?from_record_id=xcaoXBmuMyFFEcFDSgNgDQ&limit=100
+```
+
 #### Pagination keys
 
 ##### limit_key

data/friday.yml

@@ -0,0 +1,2 @@
+---
+  request_reviews: true

data/lib/lhs.rb

@@ -32,6 +32,8 @@ module LHS
       'lhs/pagination/page'
     autoload :Start,
       'lhs/pagination/start'
+    autoload :Link,
+      'lhs/pagination/link'
   end
   autoload :Problems,
     'lhs/problems/base'

data/lib/lhs/concerns/record/pagination.rb

@@ -24,6 +24,8 @@ class LHS::Record
           LHS::Pagination::Page
         when :start
           LHS::Pagination::Start
+        when :link
+          LHS::Pagination::Link
         else
           LHS::Pagination::Offset
         end
@@ -35,7 +37,10 @@ class LHS::Record
 
       # Checks if given raw is paginated or not
       def paginated?(raw)
-        !!(raw.is_a?(Hash) && raw.dig(*total_key))
+        raw.is_a?(Hash) && (
+          raw.dig(*total_key).present? ||
+          raw.dig(*limit_key(:body)).present?
+        )
       end
     end
   end

data/lib/lhs/concerns/record/request.rb

@@ -277,8 +277,16 @@ class LHS::Record
       def load_and_merge_paginated_collection!(data, options)
         set_nested_data(data._raw, limit_key(:body), data.length) if data._raw.dig(*limit_key(:body)).blank? && !data.length.zero?
         pagination = data._record.pagination(data)
-        return data if pagination.pages_left.zero?
+        return data unless pagination.pages_left?
         record = data._record
+        if pagination.parallel?
+          load_and_merge_parallel_requests!(record, data, pagination, options)
+        else
+          load_and_merge_sequential_requests!(record, data, options, data._raw.dig(:next, :href), pagination)
+        end
+      end
+
+      def load_and_merge_parallel_requests!(record, data, pagination, options)
         record.request(
           options_for_next_batch(record, pagination, options)
         ).each do |batch_data|
@@ -286,13 +294,24 @@ class LHS::Record
         end
       end
 
+      def load_and_merge_sequential_requests!(record, data, options, next_link, pagination)
+        warn "[WARNING] You are loading all pages from a resource paginated with links only. As this is performed sequentially, it can result in very poor performance! (https://github.com/local-ch/lhs#pagination-strategy-link)."
+        while next_link.present?
+          page_data = record.request(
+            options.except(:all).merge(url: next_link)
+          )
+          next_link = page_data._raw.dig(:next, :href)
+          merge_batch_data_with_parent!(page_data, data, pagination)
+        end
+      end
+
       def load_and_merge_set_of_paginated_collections!(data, options)
         options_for_next_batch = []
         options.each_with_index do |element, index|
           next if element.nil?
           record = data[index]._record
           pagination = record.pagination(data[index])
-          next if pagination.pages_left.zero?
+          next unless pagination.pages_left?
           options_for_next_batch.push(
             options_for_next_batch(record, pagination, options[index]).tap do |options|
               options.each do |option|
@@ -345,7 +364,8 @@ class LHS::Record
       # paginates itself to ensure all records are fetched
       def load_all_included!(record, options)
         data = record.request(options)
-        load_and_merge_remaining_objects!(data: data, options: options)
+        pagination = data._record.pagination(data)
+        load_and_merge_remaining_objects!(data: data, options: options) if pagination.parallel?
         data
       end
 
@@ -386,8 +406,9 @@ class LHS::Record
         options || {}
       end
 
-      def merge_batch_data_with_parent!(batch_data, parent_data)
+      def merge_batch_data_with_parent!(batch_data, parent_data, pagination = nil)
         parent_data.concat(input: parent_data._raw, items: batch_data.raw_items, record: self)
+        return if pagination.present? && pagination.is_a?(LHS::Pagination::Link)
         [limit_key(:body), total_key, pagination_key(:body)].each do |pagination_attribute|
           set_nested_data(
             parent_data._raw,

data/lib/lhs/pagination/base.rb

@@ -35,12 +35,16 @@ module LHS::Pagination
       total_pages - current_page
     end
 
+    def pages_left?
+      pages_left > 0
+    end
+
     def next_offset(_step = 1)
       raise 'to be implemented in subclass'
     end
 
     def current_page
-      raise 'to be implemented in subclass'
+      # should be implemented in subclass (optional)
     end
 
     def first_page
@@ -51,6 +55,10 @@ module LHS::Pagination
       total_pages
     end
 
+    def parallel?
+      true
+    end
+
     def next?
       data._raw[:next].present?
     end

data/lib/lhs/pagination/link.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class LHS::Pagination::Link < LHS::Pagination::Base
+  def total
+    data._raw.dig(*_record.items_key).count || 0
+  end
+
+  alias count total
+
+  def pages_left
+    pages_left? ? 1 : 0
+  end
+
+  def pages_left?
+    data._raw[:next].present?
+  end
+
+  def parallel?
+    false
+  end
+end

data/lib/lhs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LHS
-  VERSION = '19.5.0'
+  VERSION = '19.6.0'
 end

data/spec/pagination/link/current_page_spec.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe LHS::Record do
+  let(:data_hash) do
+    { items: 98.times.map { { foo: 'bar' } }, limit: 10, next: { href: 'http://example.com/users?from_user_id=100&limit=100' } }
+  end
+
+  let(:data) do
+    LHS::Data.new(data_hash, nil, Record)
+  end
+
+  let(:pagination) { LHS::Pagination::Link.new(data) }
+
+  it 'responds to parallel?' do
+    expect(pagination.current_page).to be nil
+  end
+end

data/spec/pagination/link/pages_left_spec.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe LHS::Record do
+  let(:next_parameter) do
+    { next: { href: 'http://example.com/users?from_user_id=100&limit=100' } }
+  end
+  let(:data_hash) { { items: 98.times.map { { foo: 'bar' } }, limit: 10 }.merge(next_parameter) }
+
+  let(:data) do
+    LHS::Data.new(data_hash, nil, Record)
+  end
+
+  let(:pagination) { LHS::Pagination::Link.new(data) }
+
+  it 'responds to pages_left' do
+    expect(pagination.pages_left).to eq(1)
+  end
+
+  it 'responds to pages_left?' do
+    expect(pagination.pages_left?).to be true
+  end
+
+  context 'when there is no next' do
+    let(:next_parameter) { {} }
+
+    it 'responds to pages_left' do
+      expect(pagination.pages_left).to eq(0)
+    end
+
+    it 'responds to pages_left?' do
+      expect(pagination.pages_left?).to be false
+    end
+  end
+end

data/spec/pagination/link/parallel_spec.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe LHS::Record do
+  let(:data_hash) do
+    { items: 98.times.map { { foo: 'bar' } }, limit: 10, next: { href: 'http://example.com/users?from_user_id=100&limit=100' } }
+  end
+
+  let(:data) do
+    LHS::Data.new(data_hash, nil, Record)
+  end
+
+  let(:pagination) { LHS::Pagination::Link.new(data) }
+
+  it 'responds to parallel?' do
+    expect(pagination.parallel?).to be false
+  end
+end

data/spec/pagination/link/total_spec.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require 'webrick'
+
+describe LHS::Record do
+
+  before do
+    class User < LHS::Record
+      endpoint 'http://example.com/users'
+      configuration(
+        limit_key: 'limit',
+        items_key: 'items',
+        pagination_strategy: 'link'
+      )
+    end
+  end
+
+  context 'explicit pagination parameters for retrieving pages' do
+    it 'uses explicit parameters when retrieving pages' do
+      stub_request(:get, "http://example.com/users?limit=100")
+        .to_return(body: {
+          items: 100.times.map { |_| { name: WEBrick::Utils.random_string(10) } },
+          limit: 100,
+          next: { href: 'http://example.com/users?from_user_id=100&limit=100' }
+        }.to_json)
+
+      stub_request(:get, 'http://example.com/users?from_user_id=100&limit=100')
+        .to_return(body: {
+          items: 3.times.map { |_| { name: WEBrick::Utils.random_string(10) } },
+          limit: 100,
+          next: { href: 'http://example.com/users?from_user_id=200&limit=100' }
+        }.to_json)
+
+      stub_request(:get, 'http://example.com/users?from_user_id=200&limit=100')
+        .to_return(body: {
+          items: [],
+          limit: 100
+        }.to_json)
+
+      users = User.all.fetch
+      expect(users.total).to eq 103
+      expect(users.count).to eq 103
+    end
+  end
+end

data/spec/pagination/{pages_left_spec.rb → offset/pages_left_spec.rb}

@@ -12,12 +12,6 @@ describe LHS::Record do
 
   let(:pagination) { LHS::Pagination::Offset.new(data) }
 
-  before do
-    class Record < LHS::Record
-      endpoint '{+datastore}/v2/data'
-    end
-  end
-
   it 'responds to pages_left' do
     expect(pagination.pages_left).to eq(9)
   end
@@ -26,7 +20,6 @@ describe LHS::Record do
     let(:offset) { nil }
 
     it 'responds to pages_left' do
-      # TODO i now set the nil offset to zero. Is this ok or wrong?
       expect(pagination.pages_left).to eq(9)
     end
   end

data/spec/record/includes_all_spec.rb

@@ -8,20 +8,33 @@ describe LHS::Record do
       class Customer < LHS::Record
         endpoint 'http://datastore/customers/{id}'
       end
+
+      class User < LHS::Record
+        configuration pagination_strategy: 'link'
+        endpoint 'http://datastore/users'
+      end
     end
 
     let(:amount_of_contracts) { 33 }
     let(:amount_of_products) { 22 }
+    let(:amount_of_users_1st_page) { 10 }
+    let(:amount_of_users_2nd_page) { 3 }
+    let(:amount_of_users) { amount_of_users_1st_page + amount_of_users_2nd_page }
 
     let!(:customer_request) do
       stub_request(:get, 'http://datastore/customers/1')
         .to_return(
           body: {
-            contracts: { href: 'http://datastore/customers/1/contracts' }
+            contracts: { href: 'http://datastore/customers/1/contracts' },
+            users: { href: 'http://datastore/users?limit=10' }
           }.to_json
         )
     end
 
+    #
+    # Contracts
+    #
+
     let!(:contracts_request) do
       stub_request(:get, "http://datastore/customers/1/contracts?limit=100")
         .to_return(
@@ -66,6 +79,10 @@ describe LHS::Record do
       additional_contracts_request(30, 3)
     end
 
+    #
+    # Products
+    #
+
     let!(:products_request) do
       stub_request(:get, "http://datastore/products?limit=100")
         .to_return(
@@ -102,10 +119,58 @@ describe LHS::Record do
       additional_products_request(20, 2)
     end
 
+    #
+    # Users
+    #
+
+    let!(:users_request) do
+      stub_request(:get, 'http://datastore/users?limit=10')
+        .to_return(
+          body: {
+            items: amount_of_users_1st_page.times.map do
+              { name: 'Hans Muster' }
+            end,
+            limit: 10,
+            next: { href: 'http://datastore/users?for_user_id=10&limit=10' }
+          }.to_json
+        )
+    end
+
+    let!(:users_request_page_2) do
+      stub_request(:get, 'http://datastore/users?for_user_id=10&limit=10')
+        .to_return(
+          body: {
+            items: amount_of_users_2nd_page.times.map do
+              { name: 'Lisa Müller' }
+            end,
+            limit: 10,
+            next: { href: 'http://datastore/users?for_user_id=13&limit=10' }
+          }.to_json
+        )
+    end
+
+    let!(:users_request_page_3) do
+      stub_request(:get, 'http://datastore/users?for_user_id=13&limit=10')
+        .to_return(
+          body: {
+            items: [],
+            limit: 10
+          }.to_json
+        )
+    end
+
     it 'includes all linked business objects no matter pagination' do
-      customer = Customer
-        .includes_all(contracts: :products)
-        .find(1)
+      customer = nil
+
+      expect(lambda do
+        customer = Customer
+          .includes_all(:users, contracts: :products)
+          .find(1)
+      end).to output(
+        %r{\[WARNING\] You are loading all pages from a resource paginated with links only. As this is performed sequentially, it can result in very poor performance! \(https://github.com/local-ch/lhs#pagination-strategy-link\).}
+      ).to_stderr
+
+      expect(customer.users.length).to eq amount_of_users
       expect(customer.contracts.length).to eq amount_of_contracts
       expect(customer.contracts.first.products.length).to eq amount_of_products
       expect(customer_request).to have_been_requested.at_least_once
@@ -116,6 +181,9 @@ describe LHS::Record do
       expect(products_request).to have_been_requested.at_least_once
       expect(products_request_page_2).to have_been_requested.at_least_once
       expect(products_request_page_3).to have_been_requested.at_least_once
+      expect(users_request).to have_been_requested.at_least_once
+      expect(users_request_page_2).to have_been_requested.at_least_once
+      expect(users_request_page_3).to have_been_requested.at_least_once
     end
 
     context 'links already contain pagination parameters' do

data/spec/record/paginatable_collection_spec.rb

@@ -284,4 +284,77 @@ describe LHS::Record do
       expect(all.last).to eq 223
     end
   end
+
+  context 'pagination using links' do
+    before do
+      class Record < LHS::Record
+        configuration pagination_strategy: 'link'
+        endpoint '{+datastore}/feedbacks'
+      end
+    end
+
+    it 'fetches all records from the backend' do
+      stub_request(:get, "#{datastore}/feedbacks?limit=100")
+        .to_return(
+          status: 200,
+          body: { items: (1..100).to_a, limit: 100, next: { href: "#{datastore}/feedbacks?limit=100&cursor=x" } }.to_json
+        )
+      stub_request(:get, "#{datastore}/feedbacks?limit=100&cursor=x")
+        .to_return(
+          status: 200,
+          body: { items: (101..200).to_a, limit: 100, next: { href: "#{datastore}/feedbacks?limit=100&cursor=y" } }.to_json
+        )
+      stub_request(:get, "#{datastore}/feedbacks?limit=100&cursor=y")
+        .to_return(
+          status: 200,
+          body: { items: (201..300).to_a, limit: 100, next: { href: "#{datastore}/feedbacks?limit=100&cursor=z" } }.to_json
+        )
+      stub_request(:get, "#{datastore}/feedbacks?limit=100&cursor=z")
+        .to_return(
+          status: 200,
+          body: { items: [], limit: 100 }.to_json
+        )
+      all = Record.all
+
+      expect(all).to be_kind_of Record
+      expect(all._data._proxy).to be_kind_of LHS::Collection
+      expect(all.count).to eq 300
+      expect(all.last).to eq 300
+    end
+
+    it 'fetches all, also if there is a rest and the total is not divideable trough the limit' do
+      stub_request(:get, "#{datastore}/feedbacks?limit=100")
+        .to_return(
+          status: 200,
+          body: { items: (1..100).to_a, limit: 100, next: { href: "#{datastore}/feedbacks?limit=100&cursor=x" } }.to_json
+        )
+      stub_request(:get, "#{datastore}/feedbacks?limit=100&cursor=x")
+        .to_return(
+          status: 200,
+          body: { items: (101..200).to_a, limit: 100, next: { href: "#{datastore}/feedbacks?limit=100&cursor=y" } }.to_json
+        )
+      stub_request(:get, "#{datastore}/feedbacks?limit=100&cursor=y")
+        .to_return(
+          status: 200,
+          body: { items: (201..300).to_a, limit: 100, next: { href: "#{datastore}/feedbacks?limit=100&cursor=z" } }.to_json
+        )
+      stub_request(:get, "#{datastore}/feedbacks?limit=100&cursor=z")
+        .to_return(
+          status: 200,
+          body: { items: [301], limit: 100 }.to_json
+        )
+
+      all = nil
+      expect(lambda do
+        all = Record.all.fetch
+      end).to output(
+        %r{\[WARNING\] You are loading all pages from a resource paginated with links only. As this is performed sequentially, it can result in very poor performance! \(https://github.com/local-ch/lhs#pagination-strategy-link\).}
+      ).to_stderr
+
+      expect(all).to be_kind_of Record
+      expect(all._data._proxy).to be_kind_of LHS::Collection
+      expect(all.count).to eq 301
+      expect(all.last).to eq 301
+    end
+  end
 end

data/spec/record/tracing_spec.rb

@@ -77,7 +77,6 @@ describe LHS::Record do
         end
 
         it 'forwards tracing options to lhc' do
-          # binding.pry
           Place.last
         end
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lhs
 version: !ruby/object:Gem::Version
-  version: 19.5.0
+  version: 19.6.0
 platform: ruby
 authors:
 - https://github.com/local-ch/lhs/graphs/contributors
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-08 00:00:00.000000000 Z
+date: 2019-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -215,6 +215,7 @@ files:
 - cider-ci/task_components/rspec.yml
 - cider-ci/task_components/rubocop.yml
 - cider-ci/task_components/ruby.yml
+- friday.yml
 - lhs.gemspec
 - lib/lhs.rb
 - lib/lhs/collection.rb
@@ -266,6 +267,7 @@ files:
 - lib/lhs/endpoint.rb
 - lib/lhs/item.rb
 - lib/lhs/pagination/base.rb
+- lib/lhs/pagination/link.rb
 - lib/lhs/pagination/offset.rb
 - lib/lhs/pagination/page.rb
 - lib/lhs/pagination/start.rb
@@ -380,7 +382,11 @@ files:
 - spec/item/validation_spec.rb
 - spec/item/warning_codes_spec.rb
 - spec/item/warnings_spec.rb
-- spec/pagination/pages_left_spec.rb
+- spec/pagination/link/current_page_spec.rb
+- spec/pagination/link/pages_left_spec.rb
+- spec/pagination/link/parallel_spec.rb
+- spec/pagination/link/total_spec.rb
+- spec/pagination/offset/pages_left_spec.rb
 - spec/pagination/parameters_spec.rb
 - spec/proxy/create_sub_resource_spec.rb
 - spec/proxy/load_spec.rb
@@ -584,7 +590,11 @@ test_files:
 - spec/item/validation_spec.rb
 - spec/item/warning_codes_spec.rb
 - spec/item/warnings_spec.rb
-- spec/pagination/pages_left_spec.rb
+- spec/pagination/link/current_page_spec.rb
+- spec/pagination/link/pages_left_spec.rb
+- spec/pagination/link/parallel_spec.rb
+- spec/pagination/link/total_spec.rb
+- spec/pagination/offset/pages_left_spec.rb
 - spec/pagination/parameters_spec.rb
 - spec/proxy/create_sub_resource_spec.rb
 - spec/proxy/load_spec.rb