fmrest 0.7.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 94745e25176074d96540a4a7659dce7ee15d6ea88867a58c35c3f38af8cd466f
-  data.tar.gz: 512669f188c9abe0c403318d50da19ac5376b6cc6bed74880eff9aebb5d642af
+  metadata.gz: 5873c0192a2b166cfef71b33c521665dd6953660bd9d23d3ef728b89c55e886f
+  data.tar.gz: 7615faa115cc7506d3a472d2fac70ea293bd3dbec1e03bb3c19f0faca51b7c9d
 SHA512:
-  metadata.gz: 620ee792b4b585d80358857d5464870c3384211d4311385a034ff6b941c40672d48bfa9f42b48935885131c2fe579d8a51b7e84544dbe08483d467d27966ba0c
-  data.tar.gz: e422690817ba000c9bbecd828a0c58118e223238ba44b76e86a6160cd87cb854cad0a584efd7932237cb58f125a5e567f292c0849022b897dc3f4c89ab66d96d
+  metadata.gz: b627994fa66842be8200692895077649837a7b343c337aaf5762926c633de02baf14ba6428cc33b8b3cc0d0314d7d62a4f6cf16173edb09d32ba8218c533d7d6
+  data.tar.gz: 2cd9e7f7003f9ec05120587f50ee892076d8e02045a6b18e9311276792bf831dee6c4434969990b17f4f5898f95040da0614aabbdb5940fa8906f3c507f7f0e8

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## Changelog
 
+### 0.8.0
+
+* Improved metadata when using `FmRest::Spyke::Model`. Metadata now uses
+  Struct/OpenStruct, so properties are accessible through `.property`, as well
+  as `[:property]`
+* Implemented `.find_in_batches` and `.find_each` for `FmRest::Spyke::Model`
+
 ### 0.7.1
 
 * Made sure `Model.find_one` and `Model.find_some` work without needing to call

data/README.md

@@ -195,7 +195,7 @@ FmRest.token_store = FmRest::TokenStore::Redis.new(redis: Redis.new, prefix: "my
 FmRest.token_store = FmRest::TokenStore::Redis.new(prefix: "my-fancy-prefix:", host: "10.0.1.1", port: 6380, db: 15)
 ```
 
-**NOTE:** redis-rb is not included as a gem dependency of fmrest-ruby, so you'll
+NOTE: redis-rb is not included as a gem dependency of fmrest-ruby, so you'll
 have to add it to your Gemfile.
 
 ### Moneta
@@ -232,7 +232,7 @@ FmRest.token_store = FmRest::TokenStore::Moneta.new(
 )
 ```
 
-**NOTE:** the moneta gem is not included as a dependency of fmrest-ruby, so
+NOTE: the moneta gem is not included as a dependency of fmrest-ruby, so
 you'll have to add it to your Gemfile.
 
 
@@ -555,7 +555,7 @@ Honeybee.limit(10)
 ```
 
 NOTE: You can also set a default limit value for a model class, see
-[Other notes on querying](#other-notes-on-querying).
+[other notes on querying](#other-notes-on-querying).
 
 You can also use `.limit` to set limits on portals:
 
@@ -727,15 +727,15 @@ the scope object:
 Honeybee.limit(10).sort(:name).find_some # => [<Honeybee...>, ...]
 ```
 
-If you want just a single result you can use `.find_one` instead (this will
+If you want just a single result you can use `.first` instead (this will
 force `.limit(1)`):
 
 ```ruby
-Honeybee.query(name: "Hutch").find_one # => <Honeybee...>
+Honeybee.query(name: "Hutch").first # => <Honeybee...>
 ```
 
 If you know the id of the record you should use `.find(id)` instead of
-`.query(id: id).find_one` (so that the sent request is
+`.query(id: id).first` (so that the sent request is
 `GET ../:layout/records/:id` instead of `POST ../:layout/_find`).
 
 ```ruby
@@ -746,6 +746,52 @@ Note also that if you use `.find(id)` your `.query()` parameters (as well as
 limit, offset and sort parameters) will be discarded as they're not supported
 by the single record endpoint.
 
+
+### Finding records in batches
+
+Sometimes you want to iterate over a very large number of records to do some
+processing, but requesting them all at once would result in one huge request to
+the Data API, and loading too many records in memory all at once.
+
+To mitigate this problem you can use `.find_in_batches` and `.find_each`. If
+you've used ActiveRecord you're probably familiar with how they operate:
+
+```ruby
+# Find records in batches of 100 each
+Honeybee.query(hive: "Queensville").find_in_batches(batch_size: 100) do |batch|
+  dispatch_bees(batch)
+end
+
+# Iterate over all records using batches
+Honeybee.query(hive: "Queensville").find_each(batch_size: 100) do |bee|
+  bee.dispatch
+end
+```
+
+`.find_in_batches` yields collections of records (batches), while `.find_each`
+yields individual records, but using batches behind the scenes.
+
+Both methods accept a block-less form in which case they return an
+`Enumerator`:
+
+```ruby
+batch_enum = Honeybee.find_in_batches
+
+batch = batch_enum.next # => Spyke::Collection
+
+batch_enum.each do |batch|
+  process_batch(batch)
+end
+
+record_enum = Honeybee.find_each
+
+record_enum.next # => Honeybee
+```
+
+NOTE: By its nature, batch processing is subject to race conditions if other
+processes are modifying the database.
+
+
 ### Container fields
 
 You can define container fields on your model class with `container`:
@@ -783,6 +829,7 @@ bee.photo.upload(filename_or_io) # Upload a file to the container
 * `:content_type` - The MIME content type to use (defaults to
   `application/octet-stream`)
 
+
 ### Script execution
 
 The Data API allows running scripts as part of many types of requests.
@@ -870,7 +917,7 @@ separately, under their matching key.
 ```ruby
 bee.save(script: { presort: "My Presort Script", after: "My Script" })
 
-Honeybee.last_request_metadata[:script]
+Honeybee.last_request_metadata.script
 # => { after: { result: "oh hi", error: "0" }, presort: { result: "lo", error: "0" } }
 ```
 
@@ -884,7 +931,7 @@ is performed on that scope.
 
 ```ruby
 # Find one Honeybee record executing a presort and after script
-Honeybee.script(presort: ["My Presort Script", "parameter"], after: "My Script").find_one
+Honeybee.script(presort: ["My Presort Script", "parameter"], after: "My Script").first
 ```
 
 The model class' `.last_request_metadata` will be set in case you need to get the result.

data/lib/fmrest/spyke/model/orm.rb

@@ -24,7 +24,8 @@ module FmRest
           # Methods delegated to FmRest::Spyke::Relation
           delegate :limit, :offset, :sort, :order, :query, :omit, :portal,
                    :portals, :includes, :with_all_portals, :without_portals,
-                   :script, :find_one, :find_some, to: :all
+                   :script, :find_one, :first, :any, :find_some,
+                   :find_in_batches, :find_each, to: :all
 
           def all
             # Use FmRest's Relation instead of Spyke's vanilla one

data/lib/fmrest/spyke/relation.rb

@@ -190,6 +190,79 @@ module FmRest
       rescue ::Spyke::ConnectionError => error
         fallback_or_reraise(error, default: nil)
       end
+      alias_method :first, :find_one
+      alias_method :any, :find_one
+
+      # Yields each batch of records that was found by the find options.
+      #
+      # NOTE: By its nature, batch processing is subject to race conditions if
+      # other processes are modifying the database
+      #
+      # @param batch_size [Integer] Specifies the size of the batch.
+      # @return [Enumerator] if called without a block.
+      def find_in_batches(batch_size: 1000)
+        unless block_given?
+          return to_enum(:find_in_batches, batch_size: batch_size) do
+            total = limit(1).find_some.metadata.data_info.found_count
+            (total - 1).div(batch_size) + 1
+          end
+        end
+
+        offset = 1 # DAPI offset is 1-based
+
+        loop do
+          relation = offset(offset).limit(batch_size)
+
+          records = relation.find_some
+
+          yield records if records.length > 0
+
+          break if records.length < batch_size
+
+          # Save one iteration if the total is a multiple of batch_size
+          if found_count = records.metadata.data_info && records.metadata.data_info.found_count
+            break if found_count == (offset - 1) + batch_size
+          end
+
+          offset += batch_size
+        end
+      end
+
+      # Looping through a collection of records from the database (using the
+      # #all method, for example) is very inefficient since it will fetch and
+      # instantiate all the objects at once.
+      #
+      # In that case, batch processing methods allow you to work with the
+      # records in batches, thereby greatly reducing memory consumption and be
+      # lighter on the Data API server.
+      #
+      # The find_each method uses #find_in_batches with a batch size of 1000
+      # (or as specified by the :batch_size option).
+      #
+      # NOTE: By its nature, batch processing is subject to race conditions if
+      # other processes are modifying the database
+      #
+      # @param (see #find_in_batches)
+      # @example
+      #   Person.find_each do |person|
+      #     person.greet
+      #   end
+      #
+      #   Person.query(name: "==Mitch").find_each do |person|
+      #     person.say_hi
+      #   end
+      # @return (see #find_in_batches)
+      def find_each(batch_size: 1000)
+        unless block_given?
+          return to_enum(:find_each, batch_size: batch_size) do
+            limit(1).find_some.metadata.data_info.found_count
+          end
+        end
+
+        find_in_batches(batch_size: batch_size) do |records|
+          records.each { |r| yield r }
+        end
+      end
 
       protected
 

data/lib/fmrest/spyke/spyke_formatter.rb

@@ -1,9 +1,21 @@
 # frozen_string_literal: true
 
 require "json"
+require "ostruct"
 
 module FmRest
   module Spyke
+    # Metadata class to be passed to Spyke::Collection#metadata
+    class Metadata < Struct.new(:messages, :script, :data_info)
+      alias_method :scripts, :script
+    end
+
+    class DataInfo < OpenStruct
+      def total_record_count; totalRecordCount; end
+      def found_count; foundCount; end
+      def returned_count; returnedCount; end
+    end
+
     # Response Faraday middleware for converting FM API's response JSON into
     # Spyke's expected format
     class SpykeFormatter < ::Faraday::Response::Middleware
@@ -77,36 +89,61 @@ module FmRest
 
       # @param json [Hash]
       # @param include_errors [Boolean]
-      # @return [Hash] the skeleton structure for a Spyke-formatted response
+      # @return [FmRest::Spyke::Metadata] the skeleton structure for a
+      #   Spyke-formatted response
       def build_base_hash(json, include_errors = false)
         {
-          metadata: { messages: json[:messages] }.merge(script: prepare_script_results(json).presence),
-          errors:   include_errors ? prepare_errors(json) : {}
+          metadata: Metadata.new(
+            prepare_messages(json),
+            prepare_script_results(json),
+            prepare_data_info(json)
+          ).freeze,
+          errors: include_errors ? prepare_errors(json) : {}
         }
       end
 
       # @param json [Hash]
-      # @return [Hash] the script(s) execution results for Spyke metadata format
+      # @return [Array<OpenStruct>] the skeleton structure for a
+      #   Spyke-formatted response
+      def prepare_messages(json)
+        return [] unless json[:messages]
+        json[:messages].map { |m| OpenStruct.new(m).freeze }.freeze
+      end
+
+      # @param json [Hash]
+      # @return [OpenStruct] the script(s) execution results for Spyke metadata
+      #   format
       def prepare_script_results(json)
         results = {}
 
         [:prerequest, :presort].each do |s|
           if json[:response][:"scriptError.#{s}"]
-            results[s] = {
+            results[s] = OpenStruct.new(
               result: json[:response][:"scriptResult.#{s}"],
               error:  json[:response][:"scriptError.#{s}"]
-            }
+            ).freeze
           end
         end
 
         if json[:response][:scriptError]
-          results[:after] = {
+          results[:after] = OpenStruct.new(
             result: json[:response][:scriptResult],
             error:  json[:response][:scriptError]
-          }
+          ).freeze
         end
 
-        results
+        results.present? ? OpenStruct.new(results).freeze : nil
+      end
+
+      # @param json [Hash]
+      # @return [OpenStruct] the script(s) execution results for
+      #   Spyke metadata format
+      def prepare_data_info(json)
+        data_info = json[:response] && json[:response][:dataInfo]
+
+        return nil unless data_info.present?
+
+        DataInfo.new(data_info).freeze
       end
 
       # @param json [Hash]

data/lib/fmrest/v1/connection.rb

@@ -31,10 +31,6 @@ module FmRest
           conn.request :multipart
           conn.request :json
 
-          if options[:log]
-            conn.response :logger, nil, bodies: true, headers: true
-          end
-
           # Allow overriding the default response middleware
           if block_given?
             yield conn, options
@@ -43,6 +39,10 @@ module FmRest
             conn.response :json
           end
 
+          if options[:log]
+            conn.response :logger, nil, bodies: true, headers: true
+          end
+
           conn.adapter Faraday.default_adapter
         end
       end

data/lib/fmrest/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FmRest
-  VERSION = "0.7.1"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fmrest
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.8.0
 platform: ruby
 authors:
 - Pedro Carbajal
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-13 00:00:00.000000000 Z
+date: 2020-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday