lhs 13.2.3 → 14.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2bda2ff4ae84ce367ee21ad8922b46e600b288c5
-  data.tar.gz: c8fc0746ffca39f2b7e5ad370ba71cc884535ec9
+  metadata.gz: 2844d363e539a18b08de95cd58f510dcaa101651
+  data.tar.gz: 0e4fc872e5730d30b8ead542f1b135cb28c96d81
 SHA512:
-  metadata.gz: e298a06e35a61cdbaddbceffc23e7e982e4387b9d21c7753165202e309bc0e463339b511039abf20c3038c389460e4676ef2ce933a285b83cb53b6543ffb180b
-  data.tar.gz: d2020691e22febd3f22256c2f689d093e65fd92fb0bb5519ff5413905f54f6a946fe5ce8cbb83ba9b2681322033f7f93ec26f50c4eba56b19e92cb580b267c1c
+  metadata.gz: 4ecde150a77ea8c199abc608331f35b854e546714f20e155003fa52b8cca63e6088128cec7d6ae0466bc9432124650d26d86fe16bc4bf840d8824bf1a44482ac
+  data.tar.gz: dd4f3cee3b961267307ca3a3f40cfa1bebffa53baa5b958255ac3594a6437095f00c1944e3f5dc4f68a97993ed0e90ea3c3e6c30a36e27d03063eb268b03aaab

data/README.md

@@ -336,6 +336,8 @@ Record.all(color: 'blue')
 # All three are doing the same thing: fetching all records with the color 'blue' from the endpoint while resolving pagingation if endpoint is paginated
 ```
 
+In case an API does not provide pagination information (limit, offset and total), LHS keeps on loading pages when requesting `all` until the first empty page responds.
+
 [Count vs. Length](#count-vs-length)
 
 `find_each` is a more fine grained way to process single records that are fetched in batches.

data/lib/lhs/concerns/record/request.rb

@@ -21,6 +21,15 @@ class LHS::Record
 
       private
 
+      def single_request_load_and_merge_remaining_objects!(data, options, endpoint)
+        return unless options[:all]
+        load_and_merge_remaining_objects!(
+          data: data,
+          options: process_options(options, endpoint),
+          load_not_paginated_collection: true
+        )
+      end
+
       def filter_empty_request_options(options)
         options.map do |option|
           option if !option || !option.key?(:url) || !option[:url].nil?
@@ -208,15 +217,36 @@ class LHS::Record
       # we can evaluate if there are further remote objects remaining
       # and after preparing all the requests that have to be made in order to fetch all
       # remote items during this batch, they are fetched in parallel
-      def load_and_merge_remaining_objects!(data, options)
+      def load_and_merge_remaining_objects!(data:, options:, load_not_paginated_collection: false)
         if paginated?(data._raw)
           load_and_merge_paginated_collection!(data, options)
         elsif data.collection? && paginated?(data.first.try(:_raw))
           load_and_merge_set_of_paginated_collections!(data, options)
+        elsif load_not_paginated_collection
+          load_and_merge_not_paginated_collection!(data, options)
+        end
+      end
+
+      def load_and_merge_not_paginated_collection!(data, options)
+        return if data.length.zero?
+        options = options.is_a?(Hash) ? options : {}
+        limit = options.dig(:params, limit_key) || pagination_class::DEFAULT_LIMIT
+        offset = options.dig(:params, pagination_key) || pagination_class::DEFAULT_OFFSET
+        options[:params] = options.fetch(:params, {}).merge(
+          limit_key => limit,
+          pagination_key => pagination_class.next_offset(
+            offset,
+            limit
+          )
+        )
+        additional_data = data._record.request(options)
+        additional_data.each do |item_data|
+          data.concat(input: data._raw, items: [item_data], record: self)
         end
       end
 
       def load_and_merge_paginated_collection!(data, options)
+        data._raw[limit_key] = data.length if data._raw[limit_key].blank? && !data.length.zero?
         pagination = data._record.pagination(data)
         return data if pagination.pages_left.zero?
         record = data._record
@@ -279,13 +309,13 @@ class LHS::Record
       # paginates itself to ensure all records are fetched
       def load_all_included!(record, options)
         data = record.request(options)
-        load_and_merge_remaining_objects!(data, options)
+        load_and_merge_remaining_objects!(data: data, options: options)
         data
       end
 
       # Checks if given raw is paginated or not
       def paginated?(raw)
-        !!(raw.is_a?(Hash) && raw[total_key] && raw[pagination_key])
+        !!(raw.is_a?(Hash) && raw[total_key])
       end
 
       def prepare_options_for_include_all_request!(options)
@@ -466,7 +496,7 @@ class LHS::Record
         apply_limit!(options) if options[:all]
         response = LHC.request(process_options(options, endpoint))
         data = LHS::Data.new(response.body, nil, self, response.request, endpoint)
-        load_and_merge_remaining_objects!(data, process_options(options, endpoint)) if paginated?(data._raw) && options[:all]
+        single_request_load_and_merge_remaining_objects!(data, options, endpoint)
         expand_items(data, options[:expanded]) if data.collection? && options[:expanded]
         handle_includes(including, data, referencing) if including.present? && data.present?
         data

data/lib/lhs/pagination/base.rb

@@ -23,7 +23,7 @@ module LHS::Pagination
     end
 
     def offset
-      data._raw.dig(*_record.pagination_key) || 0
+      data._raw.dig(*_record.pagination_key) || self.class::DEFAULT_OFFSET
     end
     alias current_page offset
     alias start offset

data/lib/lhs/pagination/offset.rb

@@ -1,14 +1,20 @@
 class LHS::Pagination::Offset < LHS::Pagination::Base
 
+  DEFAULT_OFFSET = 0
+
   def current_page
     (offset + limit) / limit
   end
 
   def next_offset(step = 1)
-    offset + limit * step
+    self.class.next_offset(offset, limit, step)
   end
 
   def self.page_to_offset(page, limit = DEFAULT_LIMIT)
     (page.to_i - 1) * limit.to_i
   end
+
+  def self.next_offset(offset, limit, step = 1)
+    offset.to_i + limit.to_i * step.to_i
+  end
 end

data/lib/lhs/pagination/page.rb

@@ -1,10 +1,16 @@
 class LHS::Pagination::Page < LHS::Pagination::Base
 
+  DEFAULT_OFFSET = 1
+
   def current_page
     offset
   end
 
   def next_offset(step = 1)
-    current_page + step
+    self.class.next_offset(current_page, limit, step)
+  end
+
+  def self.next_offset(current_page, _limit, step = 1)
+    current_page.to_i + step.to_i
   end
 end

data/lib/lhs/pagination/start.rb

@@ -1,14 +1,20 @@
 class LHS::Pagination::Start < LHS::Pagination::Base
 
+  DEFAULT_OFFSET = 1
+
   def current_page
     (offset + limit - 1) / limit
   end
 
   def next_offset(step = 1)
-    offset + limit * step
+    self.class.next_offset(offset, limit, step)
   end
 
   def self.page_to_offset(page, limit = DEFAULT_LIMIT)
     (page.to_i - 1) * limit.to_i + 1
   end
+
+  def self.next_offset(offset, limit, step = 1)
+    offset.to_i + limit.to_i * step.to_i
+  end
 end

data/lib/lhs/version.rb

@@ -1,3 +1,3 @@
 module LHS
-  VERSION = "13.2.3"
+  VERSION = "14.0.0"
 end

data/spec/record/all_spec.rb

@@ -1,13 +1,13 @@
 require 'rails_helper'
 
 describe LHS::Record do
-  before(:each) do
-    class Record < LHS::Record
-      endpoint 'http://datastore/feedbacks'
+  context 'all' do
+    before(:each) do
+      class Record < LHS::Record
+        endpoint 'http://datastore/feedbacks'
+      end
     end
-  end
 
-  context 'all' do
     it 'is querying endpoint without pagination when using all' do
       stub_request(:get, "http://datastore/feedbacks?limit=100").to_return(body: { items: 300.times.map { { foo: 'bar' } }, total: 300 }.to_json)
       records = Record.all
@@ -58,4 +58,48 @@ describe LHS::Record do
       end
     end
   end
+
+  context 'all without current page indicator' do
+    before(:each) do
+      class Category < LHS::Record
+        configuration(
+          items_key: %i(response results),
+          limit_key: :max,
+          pagination_key: :offset,
+          pagination_strategy: :offset
+        )
+
+        endpoint 'http://store/categories'
+      end
+    end
+
+    def stub_batch(url, items = 10)
+      stub_request(:get, url)
+        .to_return(
+          body: {
+            response: {
+              results: items.times.map { { name: 'category' } }
+            }
+          }.to_json
+        )
+    end
+
+    it 'is able to fetch all remote objects without any current page indicator by simply increasing the offset until response is empty' do
+      stub_batch('http://store/categories?language=en&max=10&offset=0')
+      stub_batch('http://store/categories?language=en&max=10&offset=10')
+      stub_batch('http://store/categories?language=en&max=10&offset=20')
+      stub_batch('http://store/categories?language=en&max=10&offset=30', 0)
+      records = Category.limit(10).all(language: 'en').fetch
+      expect(records.length).to eq 30
+    end
+
+    it 'is able to fetch all remote objects without any current page indicator by simply increasing the offset until response is empty' do
+      stub_batch('http://store/categories?language=en&max=100', 100)
+      stub_batch('http://store/categories?language=en&max=100&offset=100', 100)
+      stub_batch('http://store/categories?language=en&max=100&offset=200', 100)
+      stub_batch('http://store/categories?language=en&max=100&offset=300', 0)
+      records = Category.all(language: 'en').fetch
+      expect(records.length).to eq 300
+    end
+  end
 end

data/spec/record/endpoints_spec.rb

@@ -81,6 +81,8 @@ describe LHS::Record do
       it 'uses urls instead of trying to find base endpoint of parent class' do
         stub_request(:get, "#{datastore}/entry/123/contracts?limit=100")
           .to_return(body: [{ product: { href: "#{datastore}/products/LBC" } }].to_json)
+        stub_request(:get, "#{datastore}/entry/123/contracts?limit=100&offset=100")
+          .to_return(body: [].to_json)
         stub_request(:get, "#{datastore}/products/LBC")
           .to_return(body: { name: 'Local Business Card' }.to_json)
         expect(lambda {

data/spec/record/includes_all_spec.rb

@@ -346,7 +346,10 @@ describe LHS::Record do
         .to_return(
           body: {
             href:  "http://datastore/v2/places/1/contracts?offset=0&limit=10",
-            items: [{ href: "http://datastore/v2/contracts/1" }]
+            items: [{ href: "http://datastore/v2/contracts/1" }],
+            offset: 0,
+            limit: 10,
+            total: 10
           }.to_json
         )
 

data/spec/record/paginatable_collection_spec.rb

@@ -30,7 +30,8 @@ describe LHS::Record do
     end
 
     it 'also works when there is no total in the stubbing' do
-      stub_request(:get, %r{/feedbacks}).to_return(body: { items: (1..100).to_a }.to_json)
+      stub_request(:get, "http://local.ch/v2/feedbacks?limit=100").to_return(body: { items: (1..100).to_a }.to_json)
+      stub_request(:get, "http://local.ch/v2/feedbacks?limit=100&offset=100").to_return(body: { items: [] }.to_json)
       all = Record.all
       expect(all).to be_kind_of Record
       expect(all._proxy).to be_kind_of LHS::Collection
@@ -38,7 +39,8 @@ describe LHS::Record do
     end
 
     it 'also works when there is no key "items" in the stubbing' do
-      stub_request(:get, %r{/feedbacks}).to_return(body: (1..100).to_a.to_json)
+      stub_request(:get, "http://local.ch/v2/feedbacks?limit=100").to_return(body: (1..100).to_a.to_json)
+      stub_request(:get, "http://local.ch/v2/feedbacks?limit=100&offset=100").to_return(body: [].to_json)
       all = Record.all
       expect(all).to be_kind_of Record
       expect(all._proxy).to be_kind_of LHS::Collection

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lhs
 version: !ruby/object:Gem::Version
-  version: 13.2.3
+  version: 14.0.0
 platform: ruby
 authors:
 - https://github.com/local-ch/lhs/graphs/contributors