synced 1.1.3 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1ab9aa2facc1afc7131e87f5a5d0172b6f1128bd
-  data.tar.gz: c527f9a1459dace9a26aa4969a3fc21ed5e598a4
+  metadata.gz: f3293144a530b0b41b214912ec147152b207e4b2
+  data.tar.gz: a016b6c1cb3628884c5c465a2c84e9f167c83a36
 SHA512:
-  metadata.gz: a1dcb83e4d53ac213cfa512b6682ceba9a8adef709aa4e2225c1bb83853bf7cc1d863f0e72cab10571b762a3d56f8fbdef15b8183a52427a01fedfff89568873
-  data.tar.gz: eb8b8fab78c71c261662fbf005bdf978ccf23f787e467df154d360a4bd470c5eca8053a900fa32791ce1bd0b2055af2a07a39c48642b31b278fb5df7bd3c8516
+  metadata.gz: 9b42ca3a24e7d58cfd666a01de3c8c22e28bd67db53f74649a77c678e6212b5e689977d3158a96d202bc3167e0dd41d180ea858e01c9ea29639fd5e95ef2e9e4
+  data.tar.gz: e9da5291a6c4fbbb7a2e52078b6a4817e6b6cd631eb6acbaeefa519ace7db4ebc295af66235065561ac48c0766c8d5cb8a10b6144a1b81c565196b89d0338eba

data/README.md

@@ -77,6 +77,13 @@ rental.synced_data.bedrooms # => 4
 rental.synced_data.rental_type # => "villa"
 ```
 
+## Synced strategies
+
+There are currently 3 synced strategies: `:full`, `:updated_since` and `:check`.
+- `:full` strategy fetches all available data each time, being simple but very inefficient in most cases.
+- `:updated_since` is default strategy and syncs only changes since last sync. It's more efficient, but also more complex.
+- `:check` strategy fetches everything like full one, but only compares the datas without updating anything.
+
 ## Synced database fields
 
 Option name          | Default value    | Description                             | Required |
@@ -89,8 +96,7 @@ Custom fields name can be configured in the `synced` statement of your model:
 
 ```ruby
 class Rental < ActiveRecord::Base
-  synced id_key: :remote_id, data_key: :remote_data,
-    synced_all_at_key: :remote_all_synced_at
+  synced id_key: :remote_id, data_key: :remote_data
 end
 ```
 
@@ -201,20 +207,30 @@ end
 
 This will map remote `:description` to local `:headline` attribute.
 
-## Partial updates (using updated since parameter)
+## Partial updates (using `:updated_since` strategy)
 
 Partial updates mean that first synchronization will copy all of the remote
 objects into local database and next synchronizations will sync only
 added/changed and removed objects. This significantly improves synchronization
 time and saves network traffic.
 
-In order to enable it add timestamp column named `synced_all_at` to your
-database. Synced will automatically detect it.
-
 NOTE: In order it to work, given endpoint needs to support updated_since
 parameter. Check [API documentation](http://docs.api.bookingsync.com/reference/)
 for given endpoint.
 
+### Storing last sync timestamps
+
+When using `:updated_since` sync strategy you need to store the timestamp of the last sync somewhere.
+By default `Synced::Strategies::SyncedAllAtTimestampStrategy` strategy is used, which requires
+`synced_all_at` column to be present in the synced model. This is simple solution but on large syncs it causes serious
+overhead on updating the timestamps on all the records.
+
+There is also a `Synced::Strategies::SyncedPerScopeTimestampStrategy`, that uses another model,
+`Synced::Timestamp`, to store the synchronization timestamps. Migration can be copied from the synced dummy app
+`spec/dummy/db/migrate/20160126082137_create_synced_timestamps.rb`.
+This strategy is added to fix the problems with massive updates on `synced_all_at`. Proper cleanup of timestamp records
+is needed once in a while with `Synced::Timestamp.cleanup` (cleans records older than 1 week).
+
 ### Forcing local objects to be re-synced with the API
 
 When you add a new column or change something in the synced attributes and you
@@ -392,7 +408,6 @@ Option name          | Default value    | Description
 ---------------------|------------------|-----------------------------------------------------------------------------------|--------|-------------|
 `:id_key`            | `:synced_id`     | ID of the object fetched from the API                                             | YES    | NO          |
 `:data_key`          | `:synced_data`   | Object fetched from the API                                                       | YES    | NO          |
-`:synced_all_at_key` | `:synced_all_at` | Time of the last synchronization                                                  | YES    | NO          |
 `:associations`      | `[]`             | [Sync remote associations to local ones](#associations)                           | YES    | NO          |
 `:local_attributes`  | `[]`             | [Sync remote attributes to local ones](#local-attributes)                         | YES    | NO          |
 `:mapper`            | `nil`            | [Module used for mapping remote objects](#local-attributes-with-mapping-modules)  | YES    | NO          |

data/lib/synced/model.rb

@@ -6,11 +6,10 @@ module Synced
     #
     # @param options [Hash] Configuration options for synced. They are inherited
     #   by subclasses, but can be overwritten in the subclass.
+    # @option options [Symbol] strategy: synchronization strategy, one of :full, :updated_since, :check.
+    #   Defaults to :updated_since
     # @option options [Symbol] id_key: attribute name under which
     #   remote object's ID is stored, default is :synced_id.
-    # @option options [Symbol] synced_all_at_key: attribute name under which
-    #   last synchronization time is stored, default is :synced_all_at. It's only
-    #   used when only_updated option is enabled.
     # @option options [Boolean] only_updated: If true requests to API will take
     #   advantage of updated_since param and fetch only created/changed/deleted
     #   remote objects
@@ -34,28 +33,25 @@ module Synced
     #   Works only for partial (updated_since param) synchronizations.
     # @option options [Array|Hash] delegate_attributes: Given attributes will be defined
     #   on synchronized object and delegated to synced_data Hash
-    # @option options [Hash] search_params: Given attributes and their values
+    # @option options [Hash] query_params: Given attributes and their values
     #   which will be passed to api client to perform search
-    def synced(options = {})
-      options.symbolize_keys!
+    def synced(strategy: :updated_since, **options)
       options.assert_valid_keys(:associations, :data_key, :fields,
         :globalized_attributes, :id_key, :include, :initial_sync_since,
-        :local_attributes, :mapper, :only_updated, :remove, :synced_all_at_key,
-        :delegate_attributes, :search_params)
-      class_attribute :synced_id_key, :synced_all_at_key, :synced_data_key,
+        :local_attributes, :mapper, :only_updated, :remove,
+        :delegate_attributes, :query_params, :timestamp_strategy)
+      class_attribute :synced_id_key, :synced_data_key,
         :synced_local_attributes, :synced_associations, :synced_only_updated,
         :synced_mapper, :synced_remove, :synced_include, :synced_fields,
         :synced_globalized_attributes, :synced_initial_sync_since, :synced_delegate_attributes,
-        :synced_search_params
+        :synced_query_params, :synced_timestamp_strategy, :synced_strategy
+      self.synced_strategy              = strategy
       self.synced_id_key                = options.fetch(:id_key, :synced_id)
-      self.synced_all_at_key            = options.fetch(:synced_all_at_key,
-        synced_column_presence(:synced_all_at))
       self.synced_data_key              = options.fetch(:data_key,
         synced_column_presence(:synced_data))
       self.synced_local_attributes      = options.fetch(:local_attributes, [])
       self.synced_associations          = options.fetch(:associations, [])
-      self.synced_only_updated          = options.fetch(:only_updated,
-        column_names.include?(synced_all_at_key.to_s))
+      self.synced_only_updated          = options.fetch(:only_updated, synced_strategy == :updated_since)
       self.synced_mapper                = options.fetch(:mapper, nil)
       self.synced_remove                = options.fetch(:remove, false)
       self.synced_include               = options.fetch(:include, [])
@@ -65,7 +61,8 @@ module Synced
       self.synced_initial_sync_since    = options.fetch(:initial_sync_since,
         nil)
       self.synced_delegate_attributes   = options.fetch(:delegate_attributes, [])
-      self.synced_search_params         = options.fetch(:search_params, {})
+      self.synced_query_params          = options.fetch(:query_params, {})
+      self.synced_timestamp_strategy    = options.fetch(:timestamp_strategy, nil)
       include Synced::DelegateAttributes
       include Synced::HasSyncedData
     end
@@ -77,7 +74,7 @@ module Synced
     # @param model_class [Class] - ActiveRecord model class to which remote objects
     #   will be synchronized.
     # @param scope [ActiveRecord::Base] - Within this object scope local objects
-    #   will be synchronized. By default it's model_class.
+    #   will be synchronized. By default it's model_class. Can be infered from active record association scope.
     # @param remove [Boolean] - If it's true all local objects within
     #   current scope which are not present in the remote array will be destroyed.
     #   If only_updated is enabled, ids of objects to be deleted will be taken
@@ -97,42 +94,55 @@ module Synced
     #   create/remove/update rentals only within website.
     #   It requires relation website.rentals to exist.
     #
-    #  Rental.synchronize(remote: remote_rentals, scope: website)
+    #  website.rentals.synchronize(remote: remote_rentals)
     #
-    def synchronize(options = {})
-      options.symbolize_keys!
-      options.assert_valid_keys(:api, :fields, :include, :remote, :remove,
-        :scope, :strategy, :search_params, :association_sync)
+    def synchronize(scope: scope_from_relation, strategy: synced_strategy, **options)
+      options.assert_valid_keys(:api, :fields, :include, :remote, :remove, :query_params, :association_sync)
       options[:remove]  = synced_remove unless options.has_key?(:remove)
       options[:include] = Array.wrap(synced_include) unless options.has_key?(:include)
       options[:fields]  = Array.wrap(synced_fields) unless options.has_key?(:fields)
-      options[:search_params] = synced_search_params unless options.has_key?(:search_params)
+      options[:query_params] = synced_query_params unless options.has_key?(:query_params)
       options.merge!({
+        scope:                 scope,
+        strategy:              strategy,
         id_key:                synced_id_key,
         synced_data_key:       synced_data_key,
-        synced_all_at_key:     synced_all_at_key,
         data_key:              synced_data_key,
         local_attributes:      synced_local_attributes,
         associations:          synced_associations,
         only_updated:          synced_only_updated,
         mapper:                synced_mapper,
         globalized_attributes: synced_globalized_attributes,
-        initial_sync_since:    synced_initial_sync_since
+        initial_sync_since:    synced_initial_sync_since,
+        timestamp_strategy:    synced_timestamp_strategy
       })
       Synced::Synchronizer.new(self, options).perform
     end
 
-    # Reset synced_all_at for given scope, this forces synced to sync
+    # Reset last sync timestamp for given scope, this forces synced to sync
     # all the records on the next sync. Useful for cases when you add
     # a new column to be synced and you use updated since strategy for faster
     # synchronization.
-    def reset_synced
-      return unless synced_only_updated
-      update_all(synced_all_at_key => nil)
+    def reset_synced(scope: scope_from_relation)
+      options = {
+        scope:                 scope,
+        strategy:              synced_strategy,
+        only_updated:          synced_only_updated,
+        initial_sync_since:    synced_initial_sync_since,
+        timestamp_strategy:    synced_timestamp_strategy
+      }
+      Synced::Synchronizer.new(self, options).reset_synced
     end
 
     private
 
+    # attempt to get scope from association reflection, so you could do:
+    # account.bookings.synchronize
+    # and the scope would be account
+    def scope_from_relation
+      all.proxy_association.owner if all.respond_to?(:proxy_association)
+    end
+
     def synced_column_presence(name)
       name if column_names.include?(name.to_s)
     end

data/lib/synced/strategies/full.rb

@@ -21,8 +21,6 @@ module Synced
       #   will be synchronized. By default it's model_class.
       # @option options [Symbol] id_key: attribute name under which
       #   remote object's ID is stored, default is :synced_id.
-      # @option options [Symbol] synced_all_at_key: attribute name under which
-      #   remote object's sync time is stored, default is :synced_all_at
       # @option options [Symbol] data_key: attribute name under which remote
       #   object's data is stored.
       # @option options [Array] local_attributes: Array of attributes in the remote
@@ -48,7 +46,6 @@ module Synced
         @model_class           = model_class
         @scope                 = options[:scope]
         @id_key                = options[:id_key]
-        @synced_all_at_key     = options[:synced_all_at_key]
         @data_key              = options[:data_key]
         @remove                = options[:remove]
         @only_updated          = options[:only_updated]
@@ -64,7 +61,7 @@ module Synced
         @perform_request       = options[:remote].nil? && !@association_sync
         @remote_objects        = Array.wrap(options[:remote]) unless @perform_request
         @globalized_attributes = synced_attributes_as_hash(options[:globalized_attributes])
-        @search_params         = options[:search_params]
+        @query_params         = options[:query_params]
       end
 
       def perform
@@ -93,6 +90,10 @@ module Synced
         end
       end
 
+      def reset_synced
+        RuntimeError.new("Full strategy does not support reset_synced functionality")
+      end
+
       private
 
       def synchronize_associations(remote, local_object)
@@ -184,11 +185,11 @@ module Synced
           end
           options[:fields] = @fields if @fields.present?
           options[:auto_paginate] = true
-        end.merge(search_params)
+        end.merge(query_params)
       end
 
-      def search_params
-        Hash[@search_params.map do |param, value|
+      def query_params
+        Hash[@query_params.map do |param, value|
           final_value = value.respond_to?(:call) ? search_param_value_for_lambda(value) : value
           [param, final_value]
         end]

data/lib/synced/strategies/synced_all_at_timestamp_strategy.rb

@@ -0,0 +1,31 @@
+module Synced
+  module Strategies
+    # This is a strategy for UpdatedSince defining how to store and update synced timestamps.
+    # It uses synced_all_at column on model to store update time.
+    class SyncedAllAtTimestampStrategy
+      attr_reader :relation_scope
+
+      def initialize(relation_scope:, **_options)
+        @relation_scope = relation_scope
+      end
+
+      def last_synced_at
+        relation_scope.minimum(synced_all_at_key)
+      end
+
+      def update(timestamp)
+        relation_scope.update_all(synced_all_at_key => timestamp)
+      end
+
+      def reset
+        relation_scope.update_all(synced_all_at_key => nil)
+      end
+
+      private
+
+      def synced_all_at_key
+        :synced_all_at
+      end
+    end
+  end
+end

data/lib/synced/strategies/synced_per_scope_timestamp_strategy.rb

@@ -0,0 +1,26 @@
+require 'synced/timestamp'
+
+module Synced
+  module Strategies
+    # This is a strategy for UpdatedSince defining how to store and update synced timestamps.
+    # It uses a separate timestamps table to track when different models were synced in specific scopes.
+    class SyncedPerScopeTimestampStrategy
+      def initialize(scope:, model_class:, **_options)
+        @scope = scope
+        @model_class = model_class
+      end
+
+      def last_synced_at
+        Synced::Timestamp.with_scope_and_model(@scope, @model_class).last_synced_at
+      end
+
+      def update(timestamp)
+        Synced::Timestamp.with_scope_and_model(@scope, @model_class).create!(synced_at: timestamp)
+      end
+
+      def reset
+        Synced::Timestamp.with_scope_and_model(@scope, @model_class).delete_all
+      end
+    end
+  end
+end

data/lib/synced/strategies/updated_since.rb

@@ -1,3 +1,6 @@
+require "synced/strategies/synced_all_at_timestamp_strategy"
+require "synced/strategies/synced_per_scope_timestamp_strategy"
+
 module Synced
   module Strategies
     # This strategy performs partial synchronization.
@@ -8,18 +11,24 @@ module Synced
       def initialize(model_class, options = {})
         super
         @initial_sync_since = options[:initial_sync_since]
+        timestampt_strategy_class = options[:timestamp_strategy] || Synced::Strategies::SyncedAllAtTimestampStrategy
+        @timestamp_strategy = timestampt_strategy_class.new(relation_scope: relation_scope, scope: @scope, model_class: model_class)
       end
 
       def perform
         super.tap do |local_objects|
-          instrument("update_synced_all_at_perform.synced", model: @model_class) do
+          instrument("update_synced_timestamp_perform.synced", model: @model_class) do
             # TODO: it can't be Time.now. this value has to be fetched from the API as well
             # https://github.com/BookingSync/synced/issues/29
-            relation_scope.update_all(@synced_all_at_key => Time.now)
+            @timestamp_strategy.update(Time.now)
           end
         end
       end
 
+      def reset_synced
+        @timestamp_strategy.reset
+      end
+
       private
 
       def api_request_options
@@ -37,7 +46,7 @@ module Synced
 
       def updated_since
         instrument("updated_since.synced") do
-          [relation_scope.minimum(@synced_all_at_key), initial_sync_since].compact.max
+          [@timestamp_strategy.last_synced_at, initial_sync_since].compact.max
         end
       end
 

data/lib/synced/synchronizer.rb

@@ -22,8 +22,6 @@ module Synced
     #   will be synchronized. By default it's model_class.
     # @option options [Symbol] id_key: attribute name under which
     #   remote object's ID is stored, default is :synced_id.
-    # @option options [Symbol] synced_all_at_key: attribute name under which
-    #   remote object's sync time is stored, default is :synced_all_at
     # @option options [Symbol] data_key: attribute name under which remote
     #   object's data is stored.
     # @option options [Array] local_attributes: Array of attributes in the remote
@@ -46,29 +44,31 @@ module Synced
     # @option options [Array|Hash] globalized_attributes: A list of attributes
     #   which will be mapped with their translations.
     # @option options [Symbol] strategy: Strategy to be used for synchronization
-    #   process, possible values are :full, :updated_since, :check and nil. Default
-    #   is nil, so strategy will be chosen automatically.
-    def initialize(model_class, options = {})
+    #   process, possible values are :full, :updated_since, :check.
+    def initialize(model_class, strategy:, **options)
       @model_class       = model_class
-      @synced_all_at_key = options[:synced_all_at_key]
       @only_updated      = options[:only_updated]
-      @perform_request   = options[:remote].nil?
-      @strategy          = strategy_class(options[:strategy]).new(model_class, options)
+      @remote            = options[:remote]
+      @strategy          = strategy_class(strategy).new(model_class, options)
     end
 
     def perform
       @strategy.perform
     end
 
+    def reset_synced
+      @strategy.reset_synced
+    end
+
     private
 
     def strategy_class(name)
-      name ||= updated_since? ? :updated_since : :full
+      name = :full if force_full_strategy?
       "Synced::Strategies::#{name.to_s.classify}".constantize
     end
 
-    def updated_since?
-      @only_updated && @synced_all_at_key && @perform_request
+    def force_full_strategy?
+      @remote
     end
   end
 end

data/lib/synced/timestamp.rb

@@ -0,0 +1,19 @@
+class Synced::Timestamp < ActiveRecord::Base
+  self.table_name = 'synced_timestamps'
+  belongs_to :parent_scope, polymorphic: true
+  scope :with_scope_and_model, ->(parent_scope, model_class) { where(parent_scope: parent_scope, model_class: model_class.to_s) }
+  validates :parent_scope, :model_class, :synced_at, presence: true
+  scope :old, -> { where('synced_at < ?', 1.week.ago) }
+
+  def model_class=(value)
+    write_attribute(:model_class, value.to_s)
+  end
+
+  def self.last_synced_at
+    maximum(:synced_at)
+  end
+
+  def self.cleanup
+    old.delete_all
+  end
+end

data/lib/synced/version.rb

@@ -1,3 +1,3 @@
 module Synced
-  VERSION = "1.1.3"
+  VERSION = "1.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: synced
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.2.0
 platform: ruby
 authors:
 - Sebastien Grosjean
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-07 00:00:00.000000000 Z
+date: 2016-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -184,8 +184,11 @@ files:
 - lib/synced/result_presenter.rb
 - lib/synced/strategies/check.rb
 - lib/synced/strategies/full.rb
+- lib/synced/strategies/synced_all_at_timestamp_strategy.rb
+- lib/synced/strategies/synced_per_scope_timestamp_strategy.rb
 - lib/synced/strategies/updated_since.rb
 - lib/synced/synchronizer.rb
+- lib/synced/timestamp.rb
 - lib/synced/version.rb
 homepage: https://github.com/BookingSync/synced
 licenses:
@@ -207,7 +210,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: Keep your BookingSync Application synced with BookingSync.