synced 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b945df81754cad42074724a1571ee354c1933224
-  data.tar.gz: 751cd8bbe6f45e57291d803b84a732ce85f8b94f
+  metadata.gz: d0c560c40b86f1e58dc3d88f4c8784124f018999
+  data.tar.gz: b9c289d5ca8cda22e6d9dc7551f7bbae965d2cde
 SHA512:
-  metadata.gz: 8ddb414c76f87761d53d9cef876709c216da97bff6ad3deed9e972919559d41ec71d1deef1e6c6bb3e7a584f3c8e2dae4d9ad590d5ac9e873daa1667b737e4d5
-  data.tar.gz: ac416d727ee33daf592af9b7d4d0717d151212d0d3c39070abcb65e3492ffc28b6e5e04acded3bc97dcf783f3e9a1d9789250cd704536653d6d8bfe6249a5b59
+  metadata.gz: 3aa68754bac3e77f985740aa5e65b16649075d58ded301f3d660dd9627094dd346a93a10b46c2dfec2cfb28e43786b06419f921426fb2b350005f9b96268be0e
+  data.tar.gz: 465449c7d9c65f0355fc783957149fc621cb4a0f6f99c20737fc866251da48c4f60654a97ad635cf301d35eaa887009c7c83447c5ddcb3c159cab0c918835e41

data/README.md

@@ -76,6 +76,43 @@ rental = account.rentals.first
 rental.synced_data.bedrooms # => 4
 rental.synced_data.rental_type # => "villa"
 ```
+## Fetching data
+
+You can choose between two ways of [fetching remote data](https://github.com/BookingSync/bookingsync-api#pagination):
+1. `auto_paginate` - default strategy, which fetches and persists all data at once. This strategy ensures that resources are fetched as quickly as possible, therefore minimizing risk of data changes during request. However, this way may become cumbersome when working with large data-sets (high memory usage).
+2. `pagination with block` -  fetches and persists records in batches. Especially helpful, when dealing with large data-sets. Increases overall syncing time, but significantly reduces memory usage. Number of entries per page can be customized by `:per_page` attribute inside `:query_params`
+
+In order to switch to `pagination with block`, you just need to use `auto_paginate: false`:
+```ruby
+class Rental
+  synced auto_paginate: false, query_params: { per_page: 50 }
+end
+```
+### Persisted objects
+
+There is another major difference between `auto_paginate` and `pagination with block` - when using `auto_paginate`, `.synchronize` returns collection of all **persisted** records. On the other hand, `pagination with block` returns last batch of **fetched** resources.
+If you need to process persisted data we encourage you to use `handle_processed_objects_proc`. This proc  takes one argument (persisted records) and is called after persisting each batch of remote objects. So when using `auto_paginate`, this:
+
+```ruby
+class Rental
+  synced handle_processed_objects_proc: Proc.new { |persisted_rentals|
+      persisted_rentals.each { |rental| rental.do_stuff }
+    }
+end
+```
+would be an equivalent of overriding `.synchronize`:
+```ruby
+class Rental
+  synced
+
+  def self.synchronize(options)
+    super.tap do |persisted_rentals|
+      persisted_rentals.each { |rental| rental.do_stuff }
+    end
+  end
+end
+```
+**When using `pagination with block` only the former will work.**
 
 ## Synced strategies
 
@@ -346,6 +383,7 @@ Location.synchronize(remote: remote_locations)
 ```
 
 NOTE: Partial updates are disabled when providing remote objects.
+**WARNING:** When using `remove: true` with `remote`, remember that synced will remove **ALL** records that are not passed to `remote`.
 
 ## Removing local objects
 
@@ -418,6 +456,7 @@ If you want to access synced attribute with different name, you can pass a Hash:
 class Photo < ActiveRecord::Base
   synced delegate_attributes: {title: :name}
 end
+```
 
 keys are delegated attributes' names and values are keys on synced data Hash. This is a simpler
 version of `delegate :name, to: :synced_data` which works with Hash reserved attributes names, like
@@ -436,7 +475,9 @@ Option name          | Default value    | Description
 `:include`           | `[]`             | [An array of associations to be fetched](#including-associations-in-synced_data)  | YES    | YES         |
 `:fields`            | `[]`             | [An array of fields to be fetched](#selecting-fields-to-be-synchronized)          | YES    | YES         |
 `:remote`            | `nil`            | [Remote objects to be synchronized with local ones](#synchronization-of-given-remote-objects) | NO | YES |
-`:delegate_attributes`| `[]`            | [Define delegators to synced data Hash attributes](#delegate-attributes) | YES | NO |
+`:delegate_attributes`| `[]`            | [Define delegators to synced data Hash attributes](#delegate-attributes)          | YES    | NO |
+`:auto_paginate`     | `true`           | [Whether data should be fetched in batches or as one response](#fetching-methods)                 | YES    | YES |
+`:handle_processed_objects_proc` | `nil` | [Custom proc taking persisted remote objects, called after persisting batch of data](#persisted-objects)   | YES    | NO |
 
 ## Documentation
 

data/lib/synced/model.rb

@@ -35,16 +35,21 @@ module Synced
     #   on synchronized object and delegated to synced_data Hash
     # @option options [Hash] query_params: Given attributes and their values
     #   which will be passed to api client to perform search
+    # @option options [Boolean] auto_paginate: If true (default) will fetch and save all
+    #   records at once. If false will fetch and save records in batches.
+    # @option options [Proc] handle_processed_objects_proc: Proc taking one argument (persisted remote objects).
+    #   Called after persisting remote objects (once in case of auto_paginate, after each batch
+    #   when paginating with block).
     def synced(strategy: :updated_since, **options)
       options.assert_valid_keys(:associations, :data_key, :fields,
         :globalized_attributes, :id_key, :include, :initial_sync_since,
-        :local_attributes, :mapper, :only_updated, :remove,
-        :delegate_attributes, :query_params, :timestamp_strategy)
+        :local_attributes, :mapper, :only_updated, :remove, :auto_paginate,
+        :delegate_attributes, :query_params, :timestamp_strategy, :handle_processed_objects_proc)
       class_attribute :synced_id_key, :synced_data_key,
         :synced_local_attributes, :synced_associations, :synced_only_updated,
-        :synced_mapper, :synced_remove, :synced_include, :synced_fields,
+        :synced_mapper, :synced_remove, :synced_include, :synced_fields, :synced_auto_paginate,
         :synced_globalized_attributes, :synced_initial_sync_since, :synced_delegate_attributes,
-        :synced_query_params, :synced_timestamp_strategy, :synced_strategy
+        :synced_query_params, :synced_timestamp_strategy, :synced_strategy, :synced_handle_processed_objects_proc
       self.synced_strategy              = strategy
       self.synced_id_key                = options.fetch(:id_key, :synced_id)
       self.synced_data_key              = options.fetch(:data_key,
@@ -63,6 +68,8 @@ module Synced
       self.synced_delegate_attributes   = options.fetch(:delegate_attributes, [])
       self.synced_query_params          = options.fetch(:query_params, {})
       self.synced_timestamp_strategy    = options.fetch(:timestamp_strategy, nil)
+      self.synced_auto_paginate         = options.fetch(:auto_paginate, true)
+      self.synced_handle_processed_objects_proc  = options.fetch(:handle_processed_objects_proc, nil)
       include Synced::DelegateAttributes
       include Synced::HasSyncedData
     end
@@ -84,6 +91,8 @@ module Synced
     #   You can also force method to remove local objects by passing it
     #   to remove: :mark_as_missing. This option can be defined in the model
     #   and then overwritten in the synchronize method.
+    # @param auto_paginate [Boolean] - If true (default) will fetch and save all
+    #   records at once. If false will fetch and save records in batches.
     # @param api [BookingSync::API::Client] - API client to be used for fetching
     #   remote objects
     # @example Synchronizing amenities
@@ -97,11 +106,12 @@ module Synced
     #  website.rentals.synchronize(remote: remote_rentals)
     #
     def synchronize(scope: scope_from_relation, strategy: synced_strategy, **options)
-      options.assert_valid_keys(:api, :fields, :include, :remote, :remove, :query_params, :association_sync)
+      options.assert_valid_keys(:api, :fields, :include, :remote, :remove, :query_params, :association_sync, :auto_paginate)
       options[:remove]  = synced_remove unless options.has_key?(:remove)
       options[:include] = Array.wrap(synced_include) unless options.has_key?(:include)
       options[:fields]  = Array.wrap(synced_fields) unless options.has_key?(:fields)
       options[:query_params] = synced_query_params unless options.has_key?(:query_params)
+      options[:auto_paginate] = synced_auto_paginate unless options.has_key?(:auto_paginate)
       options.merge!({
         scope:                 scope,
         strategy:              strategy,
@@ -114,7 +124,8 @@ module Synced
         mapper:                synced_mapper,
         globalized_attributes: synced_globalized_attributes,
         initial_sync_since:    synced_initial_sync_since,
-        timestamp_strategy:    synced_timestamp_strategy
+        timestamp_strategy:    synced_timestamp_strategy,
+        handle_processed_objects_proc:  synced_handle_processed_objects_proc
       })
       Synced::Synchronizer.new(self, options).perform
     end

data/lib/synced/strategies/check.rb

@@ -19,24 +19,32 @@ module Synced
       #      ActiveRecord::Model #changes hash is returned - changed objects
       # @return [Synced::Strategies::Check::Result] Integrity check result
       def perform
+        process_remote_objects(remote_objects_tester)
         result.additional = remove_relation.to_a
-        remote_objects.map do |remote|
-          if local_object = local_object_by_remote_id(remote.id)
-            remote.extend(@mapper) if @mapper
-            local_object.attributes = default_attributes_mapping(remote)
-            local_object.attributes = local_attributes_mapping(remote)
-            if @globalized_attributes.present?
-              local_object.attributes = globalized_attributes_mapping(remote,
-                local_object.translations.translated_locales)
-            end
-            if local_object.changed?
-              result.changed << [{ id: local_object.id }, local_object.changes]
+        result
+      end
+
+      def remote_objects_tester
+        lambda do |remote_objects|
+          @remote_objects_ids.concat(remote_objects.map(&:id))
+
+          remote_objects.map do |remote|
+            if local_object = local_object_by_remote_id(remote.id)
+              remote.extend(@mapper) if @mapper
+              local_object.attributes = default_attributes_mapping(remote)
+              local_object.attributes = local_attributes_mapping(remote)
+              if @globalized_attributes.present?
+                local_object.attributes = globalized_attributes_mapping(remote,
+                  local_object.translations.translated_locales)
+              end
+              if local_object.changed?
+                result.changed << [{ id: local_object.id }, local_object.changes]
+              end
+            else
+              result.missing << remote
             end
-          else
-            result.missing << remote
           end
         end
-        result
       end
 
       # If we check model which uses cancel instead of destroy, we skip canceled

data/lib/synced/strategies/full.rb

@@ -42,6 +42,8 @@ module Synced
       #   mapping remote objects attributes into local object attributes
       # @option options [Array|Hash] globalized_attributes: A list of attributes
       #   which will be mapped with their translations.
+      # @option options [Boolean] auto_paginate: If true (default) will fetch and save all
+      #   records at once. If false will fetch and save records in batches.
       def initialize(model_class, options = {})
         @model_class           = model_class
         @scope                 = options[:scope]
@@ -61,30 +63,21 @@ module Synced
         @remote_objects        = Array.wrap(options[:remote]) unless @perform_request
         @globalized_attributes = synced_attributes_as_hash(options[:globalized_attributes])
         @query_params         = options[:query_params]
+        @auto_paginate         = options[:auto_paginate]
+        @handle_processed_objects_proc = options[:handle_processed_objects_proc]
+        @remote_objects_ids = []
       end
 
       def perform
         instrument("perform.synced", model: @model_class) do
           relation_scope.transaction do
+            processed_objects = instrument("sync_perform.synced", model: @model_class) do
+              process_remote_objects(remote_objects_persistor)
+            end
             instrument("remove_perform.synced", model: @model_class) do
               remove_relation.send(remove_strategy) if @remove
             end
-            instrument("sync_perform.synced", model: @model_class) do
-              remote_objects.map do |remote|
-                remote.extend(@mapper) if @mapper
-                local_object = local_object_by_remote_id(remote.id) || relation_scope.new
-                local_object.attributes = default_attributes_mapping(remote)
-                local_object.attributes = local_attributes_mapping(remote)
-                if @globalized_attributes.present?
-                  local_object.attributes = globalized_attributes_mapping(remote,
-                    local_object.translations.translated_locales)
-                end
-                local_object.save! if local_object.changed?
-                local_object.tap do |local_object|
-                  synchronize_associations(remote, local_object)
-                end
-              end
-            end
+            processed_objects
           end
         end
       end
@@ -95,6 +88,32 @@ module Synced
 
       private
 
+      def remote_objects_persistor
+        lambda do |remote_objects|
+          additional_errors_check
+          @remote_objects_ids.concat(remote_objects.map(&:id))
+
+          processed_objects =
+            remote_objects.map do |remote|
+              remote.extend(@mapper) if @mapper
+              local_object = local_object_by_remote_id(remote.id) || relation_scope.new
+              local_object.attributes = default_attributes_mapping(remote)
+              local_object.attributes = local_attributes_mapping(remote)
+              if @globalized_attributes.present?
+                local_object.attributes = globalized_attributes_mapping(remote,
+                  local_object.translations.translated_locales)
+              end
+              local_object.save! if local_object.changed?
+              local_object.tap do |local_object|
+                synchronize_associations(remote, local_object)
+              end
+            end
+
+          @handle_processed_objects_proc.call(processed_objects) if @handle_processed_objects_proc.respond_to?(:call)
+          processed_objects
+        end
+      end
+
       def synchronize_associations(remote, local_object)
         @associations.each do |association|
           klass = association.to_s.classify.constantize
@@ -158,20 +177,32 @@ module Synced
       end
 
       def local_objects
-        @local_objects ||= relation_scope.where(@id_key => remote_objects_ids).to_a
+        relation_scope.where(@id_key => remote_objects_ids).to_a
       end
 
       def remote_objects_ids
-        @remote_objects_ids ||= remote_objects.map(&:id)
+        @remote_objects_ids
       end
 
-      def remote_objects
-        @remote_objects ||= @perform_request ? fetch_remote_objects : nil
+      def process_remote_objects(processor)
+        if @remote_objects
+          processor.call(@remote_objects)
+        elsif @perform_request
+          fetch_and_save_remote_objects(processor)
+        else
+          nil
+        end
       end
 
-      def fetch_remote_objects
+      def fetch_and_save_remote_objects(processor)
         instrument("fetch_remote_objects.synced", model: @model_class) do
-          api.paginate(resource_name, api_request_options)
+          if @auto_paginate
+            processor.call(api.paginate(resource_name, api_request_options))
+          else
+            api.paginate(resource_name, api_request_options) do |batch|
+              processor.call(batch)
+            end
+          end
         end
       end
 
@@ -183,7 +214,7 @@ module Synced
             options[:include] += @include
           end
           options[:fields] = @fields if @fields.present?
-          options[:auto_paginate] = true
+          options[:auto_paginate] = @auto_paginate
         end.merge(query_params)
       end
 
@@ -223,6 +254,9 @@ module Synced
         Synced.instrumenter.instrument(*args, &block)
       end
 
+      def additional_errors_check
+      end
+
       class MissingAPIClient < StandardError
         def initialize(scope, model_class)
           @scope = scope

data/lib/synced/strategies/updated_since.rb

@@ -16,11 +16,8 @@ module Synced
       end
 
       def perform
-        raise MissingTimestampError.new unless first_request_timestamp
         super.tap do |local_objects|
           instrument("update_synced_timestamp_perform.synced", model: @model_class) do
-            # TODO: it can't be Time.now. this value has to be fetched from the API as well
-            # https://github.com/BookingSync/synced/issues/29
             @timestamp_strategy.update(first_request_timestamp)
           end
         end
@@ -62,13 +59,13 @@ module Synced
       end
 
       def meta
-        remote_objects
-        @meta ||= api.last_response.meta
+        @meta ||=
+          (api.last_response && api.last_response.meta) || {}
       end
 
       def first_response_headers
-        remote_objects
-        @first_response_headers ||= api.pagination_first_response.headers
+        @first_response_headers ||=
+          (api.pagination_first_response && api.pagination_first_response.headers) || {}
       end
 
       # Remove all objects with ids from deleted_ids field in the meta key
@@ -76,6 +73,10 @@ module Synced
         relation_scope.where(@id_key => deleted_remote_objects_ids)
       end
 
+      def additional_errors_check
+        raise MissingTimestampError.new unless first_request_timestamp
+      end
+
       class CannotDeleteDueToNoDeletedIdsError < StandardError
         def initialize(model_class)
           @model_class = model_class

data/lib/synced/version.rb

@@ -1,3 +1,3 @@
 module Synced
-  VERSION = "1.4.0"
+  VERSION = "1.5.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: synced
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Sebastien Grosjean
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-01 00:00:00.000000000 Z
+date: 2017-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -31,14 +31,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.3
+        version: 0.1.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.3
+        version: 0.1.4
 - !ruby/object:Gem::Dependency
   name: hashie
   requirement: !ruby/object:Gem::Requirement
@@ -53,6 +53,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -155,14 +169,14 @@ dependencies:
   name: globalize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 4.0.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 4.0.2
 - !ruby/object:Gem::Dependency
@@ -224,7 +238,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
 summary: Keep your BookingSync Application synced with BookingSync.