consyncful 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 258e4fe9fbea3099f0d78615c79b1e49658841c7b706f296a1d5402115c5ed7d
-  data.tar.gz: 8d47ad368c228b87a1f04fc5d1c4c236c16d6d3106bae1dda6a49de0a74a9c9d
+  metadata.gz: 14eda0769dbcdc516d59a9a4252fafb23eb6380ce0224a276e101f81d649c950
+  data.tar.gz: aa4aaf421824043dbea600c1cfabca2633a8d3470ed2a14aba5685b8bf8acd90
 SHA512:
-  metadata.gz: 18b498a8a55c814edf02fabec13c54d93a6653aae0e1e87143ea9e2d637d3eb61e6beaf045201660ad065bb745f4df97b411fa6f451bbd02858d45b1e041aa24
-  data.tar.gz: 841216be1a7c0149aea9e6136369cdd2bd3a1e5d9edb9854b677f98d2ef1596b7df3273370054713915fd798016578263398f24d8e23460360caa2bb5751871d
+  metadata.gz: 9ac1a66dda08d88396d83c736308516a798503afa519cd673e76f668d771dc933b6068a2e03c1f75926ec8980ad273c0936af6e2058d84d5919a250a498ebbd7
+  data.tar.gz: 260c353fe5eb6c646473830ecdec0a094016bf0f0e518ef357ceb7cc03862f38e76e4cccbcf656a4c368fad72641ef60788d13a5c76e768a5b8fc9652b375c96

data/.rubocop.yml

@@ -1,9 +1 @@
-
-
-
-Metrics/LineLength:
-  Max: 120
-
-Metrics/BlockLength:
-  Exclude:
-    - spec/**/*
+inherit_from: .rubocop_todo.yml

data/.rubocop_todo.yml

@@ -0,0 +1,48 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2021-03-01 10:54:42 +1300 using RuboCop version 0.79.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: AllowInHeredoc.
+Layout/TrailingWhitespace:
+  Exclude:
+    - 'lib/consyncful/item_mapper.rb'
+
+# Offense count: 1
+Metrics/AbcSize:
+  Max: 19
+
+# Offense count: 10
+# Configuration parameters: CountComments, ExcludedMethods.
+# ExcludedMethods: refine
+Metrics/BlockLength:
+  Max: 175
+
+# Offense count: 3
+# Cop supports --auto-correct.
+# Configuration parameters: AutoCorrect, EnforcedStyle.
+# SupportedStyles: nested, compact
+Style/ClassAndModuleChildren:
+  Exclude:
+    - 'lib/consyncful/railtie.rb'
+    - 'spec/consyncful/base_spec.rb'
+
+# Offense count: 3
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/consyncful.rb'
+    - 'lib/consyncful/railtie.rb'
+
+# Offense count: 56
+# Cop supports --auto-correct.
+# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Layout/LineLength:
+  Max: 226

data/.ruby-version

@@ -1 +1 @@
-2.5.5
+2.7.1

data/.travis.yml

@@ -1,8 +1,14 @@
+---
 sudo: false
 language: ruby
+cache: bundler
 rvm:
   - 2.5.1
-before_install: gem install bundler -v 1.16.1
-
+  - 2.6.5
+before_install: gem install bundler -v 2.1.0
 services:
   - mongodb
+
+script: 
+  - bundle exec rake spec
+  - bundle exec rubocop -P

data/Gemfile.lock

@@ -1,80 +1,108 @@
 PATH
   remote: .
   specs:
-    consyncful (0.2.0)
+    consyncful (0.5.0)
       contentful (>= 2.11.1, < 3.0.0)
+      hooks (>= 0.4.1)
       mongoid (>= 7.0.2, < 8.0.0)
-      term-ansicolor
+      rainbow
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (5.2.2)
-      activesupport (= 5.2.2)
-    activesupport (5.2.2)
+    activemodel (6.0.3.4)
+      activesupport (= 6.0.3.4)
+    activesupport (6.0.3.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    addressable (2.6.0)
-      public_suffix (>= 2.0.2, < 4.0)
-    bson (4.4.2)
-    concurrent-ruby (1.1.4)
-    contentful (2.11.1)
-      http (> 0.8, < 4.0)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.0)
+    bson (4.11.1)
+    concurrent-ruby (1.1.8)
+    contentful (2.15.4)
+      http (> 0.8, < 5.0)
       multi_json (~> 1)
+    database_cleaner (1.8.3)
     diff-lcs (1.3)
-    domain_name (0.5.20180417)
+    domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    http (3.3.0)
+    ffi (1.14.2)
+    ffi-compiler (1.0.1)
+      ffi (>= 1.0.0)
+      rake
+    hooks (0.4.1)
+      uber (~> 0.0.14)
+    http (4.4.1)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
-      http-form_data (~> 2.0)
-      http_parser.rb (~> 0.6.0)
+      http-form_data (~> 2.2)
+      http-parser (~> 1.2.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    http-form_data (2.1.1)
-    http_parser.rb (0.6.0)
-    i18n (1.5.3)
+    http-form_data (2.3.0)
+    http-parser (1.2.3)
+      ffi-compiler (>= 1.0, < 2.0)
+    i18n (1.8.7)
       concurrent-ruby (~> 1.0)
-    minitest (5.11.3)
-    mongo (2.7.0)
-      bson (>= 4.4.2, < 5.0.0)
-    mongoid (7.0.2)
-      activemodel (>= 5.1, < 6.0.0)
-      mongo (>= 2.5.1, < 3.0.0)
-    multi_json (1.13.1)
-    public_suffix (3.0.3)
-    rake (10.5.0)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.2)
+    jaro_winkler (1.5.4)
+    minitest (5.14.3)
+    mongo (2.14.0)
+      bson (>= 4.8.2, < 5.0.0)
+    mongoid (7.2.0)
+      activemodel (>= 5.1, < 6.1)
+      mongo (>= 2.10.5, < 3.0.0)
+    multi_json (1.15.0)
+    parallel (1.19.1)
+    parser (2.7.0.2)
+      ast (~> 2.4.0)
+    public_suffix (4.0.6)
+    rainbow (3.0.0)
+    rake (13.0.1)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
-    term-ansicolor (1.7.1)
-      tins (~> 1.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.2)
+    rubocop (0.79.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.7)
+    ruby-progressbar (1.10.1)
     thread_safe (0.3.6)
-    tins (1.20.2)
-    tzinfo (1.2.5)
+    tzinfo (1.2.9)
       thread_safe (~> 0.1)
+    uber (0.0.15)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.5)
+    unf_ext (0.0.7.7)
+    unicode-display_width (1.6.1)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.16)
+  bundler (~> 2)
   consyncful!
-  rake (~> 10.0)
+  database_cleaner
+  rake (~> 13.0)
   rspec (~> 3.0)
+  rubocop (= 0.79.0)
 
+BUNDLED WITH
+   2.1.0

data/README.md

@@ -1,20 +1,16 @@
-# Consyncful
-[![Build Status](https://travis-ci.com/boost/consyncful.svg?branch=master)](https://travis-ci.com/boost/consyncful)
+# Consyncful [![Build Status](https://travis-ci.org/boost/consyncful.svg?branch=master)](https://travis-ci.org/boost/consyncful)
 
 Contentful -> local database synchronisation for Rails
 
-Requesting complicated models from the Contentful Delivery API in Rails applications is often
-too slow, and makes testing applications painful. Consyncful uses Contentful's syncronisation API 
-to keep a local copy of the entire content in a Mongo database up to date.
+Requesting complicated models from the Contentful Delivery API in Rails applications is often too slow, and makes testing applications painful. Consyncful uses Contentful's synchronisation API to keep a local, up-to-date copy of the entire content in a Mongo database.
 
-Once the content is availble locally, finding and interact with contentful data is as easy as 
-using [Mongoid](https://docs.mongodb.com/mongoid/current/tutorials/mongoid-documents/) ODM. 
+Once the content is available locally, finding and interact with contentful data is as easy as using [Mongoid](https://docs.mongodb.com/mongoid/current/tutorials/mongoid-documents/) ODM.
 
-This gem doesn't provide any intergration with the management api or any way to update contentful models from the local store. It is strictly read only.
+This gem doesn't provide any integration with the management API, or any way to update Contentful models from the local store. It is strictly read only.
 
 ## Why do I have to use MongoDB?
 
-Consyncful currently only supports Mongoid ODM because models have dynamic schemas. And that's all we've had a chance to work out so far. :) 
+Consyncful currently only supports Mongoid ODM because models have dynamic schemas. And that's all we've had a chance to work out so far. :)
 The same pattern might be able to be extended to work with ActiveRecord, but having to migrate the local database as well as your contentful content type's seems tedious.
 
 ## Installation
@@ -29,12 +25,13 @@ And then execute:
 
     $ bundle
 
-If you don't already use mongoid, generate a mongoid.yml by running:
+If you don't already use Mongoid, generate a mongoid.yml by running:
 
     $ rake g mongoid:config
 
 Add an initializer:
-Consyncful uses [contentful.rb](https://github.com/contentful/contentful.rb) so client options are as documented there.
+
+Consyncful uses [contentful.rb](https://github.com/contentful/contentful.rb); client options are as documented there.
 ```ruby
   Consyncful.configure do |config|
     config.locale = 'en-NZ'
@@ -50,7 +47,7 @@ Consyncful uses [contentful.rb](https://github.com/contentful/contentful.rb) so
 
 ## Usage
 
-### Creating contentful models in your rails app
+### Creating contentful models in your Rails app
 
 Create models by inheriting from `Consyncful::Base`
 
@@ -60,9 +57,9 @@ class ModelName < Consyncful::Base
 end
 ```
 
-Model fields will be dynamicly assigned, but mongoid dynamic fields are not accessible if the entry has an empty field. If you want the accessor methods to be reliably available for fields it is recommended to define the fields in the model:
+Model fields will be dynamically assigned, but Mongoid dynamic fields are not accessible if the entry has an empty field. If you want the accessor methods to be reliably available for fields it is recommended to define the fields in the model:
 
-```ruby 
+```ruby
 class ModelName < Consyncful::Base
   contentful_model_name 'contentfulTypeName'
 
@@ -71,9 +68,9 @@ class ModelName < Consyncful::Base
 end
 ```
 
-Contentful reference fields are a bit special compared with standard mongoid associations, Consyncful provides the following helpers to set up the correct relationships:
+Contentful reference fields are a bit special compared with standard Mongoid associations. Consyncful provides the following helpers to set up the correct relationships:
 
-```ruby 
+```ruby
 class ModelWithReferences < Consyncful::Base
   contentful_model_name 'contentfulTypeName'
 
@@ -82,26 +79,26 @@ class ModelWithReferences < Consyncful::Base
 end
 ```
 
-### Syncronizing contentful data
+### Synchronizing contentful data
 
-To run a syncronization process run:
+To run a synchronization process run:
 
     $ rake consyncful:sync
 
-The first time you run this it will download all the contentful content, it will then check every 15 seconds for changes to the content and update/delete records in the database when changes are made in contentful.
+The first time you run this it will download all the Contentful content. It will then check every 15 seconds for changes to the content and update/delete records in the database when changes are made in Contentful.
 
-If you want to syncronise from scratch run:
+If you want to synchronise from scratch, run:
 
     $ rake consyncful:refresh
 
 It is recommended to refresh your data if you change model names.
 
-Now you've synced your data, it is all available via your rails models
+Now you've synced your data, it is all available via your Rails models.
 
 ### Finding and interacting with models
 
 #### Querying
-Models are available using standard mongoid [queries](https://docs.mongodb.com/mongoid/current/tutorials/mongoid-queries/).
+Models are available using standard Mongoid [queries](https://docs.mongodb.com/mongoid/current/tutorials/mongoid-queries/).
 
 ```ruby
 instance = ModelName.find_by(instance: 'foo')
@@ -110,7 +107,7 @@ instance.is_awesome # true
 ```
 
 #### References
-References work like you woule expect:
+References work like you would expect:
 
 ```ruby
 
@@ -121,25 +118,41 @@ instance.other_things # all the referenced things, polymorphic, so might be diff
 ```
 
 **Except**:
-`references_many` associations return objects in a different order from how they are ordered in contentful. If you want them in the order they appare in contentful, use the `.in_order` helper:
+`references_many` associations return objects in a different order from how they are ordered in Contentful. If you want them in the order they appear in Contentful, use the `.in_order` helper:
 
 ```ruby
-instance.other_things.in_order # ordered the same as in contentful
+instance.other_things.in_order # ordered the same as in Contentful
 ```
 
-#### Finding entrys from different content types
+#### Finding entries from different content types
 
-Because all contentful models are stored as polymorphic subtypes of Consyncful::Base, you can query all entries without knowing what type you are looking for:
+Because all Contentful models are stored as polymorphic subtypes of `Consyncful::Base`, you can query all entries without knowing what type you are looking for:
 
 ```ruby
   Consyncful::Base.where(title: 'a title') # [ #<ModelName>, #<OtherModelName> ]
 ```
 
-## Limitations
+### Sync callbacks
+
+You may want to attach some application logic to happen before or after a sync run, for example to update caches.
+
+Callbacks can be registered using:
+
+```ruby
+Consyncful::Sync.before_run do
+  # do something before the run
+end
+```
+
+```ruby
+Consyncful::Sync.after_run do |updated_ids|
+  # invalidate cache for updated_ids, or something
+end
+```
 
-### Locales
+### Using Locales for specific fields
 
-Current Consyncful only uses one globally configured locale to map the data to the database.
+If fields have multiple locales then the default locale will be mapped to the field name. Additional locales will have a suffix (lower snake case) on the field name. e.g title (default), title_mi_nz (New Zealand Maori mi-NZ)
 
 ## Development
 

data/consyncful.gemspec

@@ -7,8 +7,8 @@ require 'consyncful/version'
 Gem::Specification.new do |spec|
   spec.name          = 'consyncful'
   spec.version       = Consyncful::VERSION
-  spec.authors       = ['Andy Anastasiadis-Gray', 'Montgomery Anderson']
-  spec.email         = ['andy@boost.co.nz', 'montgomery@boost.co.nz']
+  spec.authors       = ['Andy Anastasiadis-Gray', 'Montgomery Anderson', 'Greg Rogan']
+  spec.email         = ['andy@boost.co.nz', 'montgomery@boost.co.nz', 'greg@boost.co.nz']
 
   spec.summary       = 'Contentful to local database synchronisation for Rails'
   spec.homepage      = 'https://github.com/boost/consyncful'
@@ -30,11 +30,14 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.16'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'bundler', '~> 2'
+  spec.add_development_dependency 'database_cleaner'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '0.79.0'
 
   spec.add_dependency 'contentful', ['>=2.11.1', '<3.0.0']
+  spec.add_dependency 'hooks', '>=0.4.1'
   spec.add_dependency 'mongoid', ['>=7.0.2', '<8.0.0']
-  spec.add_dependency 'term-ansicolor'
+  spec.add_dependency 'rainbow'
 end

data/lib/consyncful.rb

@@ -27,7 +27,7 @@ module Consyncful
       @contentful_client_options = {
         api_url: 'cdn.contentful.com'
       }
-      @locale = 'en-US'
+      @locale = 'en-NZ'
     end
   end
 

data/lib/consyncful/base.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 module Consyncful
+  ##
+  # Provides common functionality of Mongoid models created from contentful
+  # entries
   class Base
     include Mongoid::Document
     include Mongoid::Attributes::Dynamic

data/lib/consyncful/item_mapper.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 module Consyncful
+  ##
+  # Responsible for mapping an update received from Contentful's syncronisation API
+  # into useful fields for Consyncful::PersistedItem to store in the database.
   class ItemMapper
     def initialize(item)
       @item = item
@@ -22,17 +25,11 @@ module Consyncful
       @item.id
     end
 
-    def mapped_fields(locale)
+    def mapped_fields(default_locale)
       fields = generic_fields
 
-      @item.fields_with_locales.each do |field, value_with_locales|
-        value = value_with_locales[locale.to_sym]
-        next if value.is_a? Contentful::File # it is special
-
-        assign_field(fields, field, value)
-      end
-
-      fields[:file] = raw_file(locale) if type == 'asset'
+      fields.merge!(localized_fields(default_locale))
+      fields.merge!(localized_asset_fields(default_locale)) if type == 'asset'
 
       fields
     end
@@ -49,9 +46,31 @@ module Consyncful
       fields
     end
 
-    def raw_file(locale)
-      file_json = @item.raw.fetch('fields', {}).fetch('file', nil)
-      file_json[locale] unless file_json.nil?
+    def localized_fields(default_locale)
+      fields = {}
+
+      @item.fields_with_locales.each do |field, value_with_locales|
+        value_with_locales.each do |locale_code, value|
+          next if value.is_a? Contentful::File # assets are handeled below
+    
+          fieldname = locale_code == default_locale.to_sym ? field : "#{field}_#{locale_code.to_s.underscore}".to_sym
+          assign_field(fields, fieldname, value)
+        end
+      end
+
+      fields
+    end
+
+    def localized_asset_fields(default_locale)
+      fields = {}
+      files_by_locale = @item.raw.dig('fields', 'file') || {}
+
+      files_by_locale.each do |locale_code, details|
+        fieldname = locale_code == default_locale ? 'file' : "file_#{locale_code.to_s.underscore}"
+        fields[fieldname.to_sym] = details
+      end
+
+      fields
     end
 
     def reference_value?(value)

data/lib/consyncful/persisted_item.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Consyncful
+  ##
+  # Takes a mapped item from Contentful and creates/updates/deletes
+  # the relevant model in the local database.
+  class PersistedItem
+    DEFAULT_LOCALE = 'en-NZ'
+
+    def initialize(item, sync_id, stats)
+      @item = item
+      @sync_id = sync_id
+      @stats = stats
+    end
+
+    def persist
+      puts Rainbow("syncing: #{@item.id}").yellow
+      if @item.deletion?
+        delete_model(@item.id, @stats)
+      else
+        create_or_update_model(@item, @sync_id, @stats)
+      end
+    end
+
+    private
+
+    def delete_model(id, stats)
+      Base.find_by(id: id).destroy
+      stats.record_deleted
+    rescue Mongoid::Errors::DocumentNotFound
+      puts Rainbow("Deleted record not found: #{id}").yellow
+      nil
+    end
+
+    def create_or_update_model(item, sync_id, stats)
+      return if item.type.nil?
+
+      instance = find_or_initialize_item(item)
+      update_stats(instance, stats)
+
+      reset_fields(instance)
+
+      item.mapped_fields(DEFAULT_LOCALE).each do |field, value|
+        instance[field] = value
+      end
+
+      instance[:sync_id] = sync_id
+
+      instance.save
+    end
+
+    def find_or_initialize_item(item)
+      model_class(item.type).find_or_initialize_by(id: item.id)
+    end
+
+    def update_stats(instance, stats)
+      if instance.persisted?
+        stats.record_updated
+      else
+        stats.record_added
+      end
+    end
+
+    def model_class(type)
+      Base.model_map[type] || Base
+    end
+
+    def reset_fields(instance)
+      instance.attributes.each do |field_name, _value|
+        next if field_name.in? %w[_id _type]
+
+        instance[field_name] = nil
+      end
+    end
+  end
+end

data/lib/consyncful/stats.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
+require 'rainbow'
+
 module Consyncful
+  ##
+  # Responsible for recording changes during a sync for outputting in logs
   class Stats
     def initialize
       @stats = {
@@ -23,9 +27,9 @@ module Consyncful
     end
 
     def print_stats
-      puts "Added: #{@stats[:records_added]}, \
+      puts Rainbow("Added: #{@stats[:records_added]}, \
         updated:  #{@stats[:records_updated]}, \
-        deleted: #{@stats[:records_deleted]}".blue
+        deleted: #{@stats[:records_deleted]}").blue
     end
   end
 end

data/lib/consyncful/sync.rb

@@ -1,18 +1,26 @@
 # frozen_string_literal: true
 
-require 'term/ansicolor'
+require 'rainbow'
 require 'consyncful/item_mapper'
+require 'consyncful/persisted_item'
 require 'consyncful/stats'
-
-class String
-  include Term::ANSIColor
-end
+require 'hooks'
 
 module Consyncful
+  ##
+  # A mongoid model that stores the state of a syncronisation feed. Stores the
+  # next URL provided by Contentfuls Sync API.
+  #
+  # Sync's are affectivly singletons,
+  # there should only ever be one in the database
+  #
+  # Is also the entrypoint of a Syncronization run
   class Sync
     include Mongoid::Document
+    include Hooks
 
-    DEFAULT_LOCALE = 'en-NZ'
+    define_hook :before_run
+    define_hook :after_run
 
     field :next_url
     field :last_run_at, type: DateTime
@@ -21,31 +29,41 @@ module Consyncful
       last || new
     end
 
+    ##
+    # Delete the previous sync chains from database and create a fresh one.
+    # Used to completely resync all items from Contentful.
     def self.fresh
       destroy_all
       latest
     end
 
+    ##
+    # Makes sure that the database contains only records that have been provided
+    # during this chain of syncronisation.
     def drop_stale
       stale = Base.where(:sync_id.ne => id, :sync_id.exists => true)
-      puts "Dropping #{stale.count} records that haven't been touched in this sync".red
+      puts Rainbow("Dropping #{stale.count} records that haven't been touched in this sync").red
       stale.destroy
     end
 
+    ##
+    # Entry point to a syncronization run. Is responsible for updating Sync state
     def run
+      run_hook :before_run
+
       stats = Consyncful::Stats.new
       load_all_models
 
       sync = start_sync
 
-      sync_items(sync, stats)
+      changed_ids = sync_items(sync, stats)
 
       drop_stale
 
-      self.next_url = sync.next_sync_url
-      self.last_run_at = Time.current
-      save
+      update_run(sync.next_sync_url)
       stats.print_stats
+
+      run_hook :after_run, changed_ids
     end
 
     private
@@ -56,80 +74,36 @@ module Consyncful
       Rails.application.eager_load!
     end
 
+    def update_run(next_url)
+      self.next_url = next_url
+      self.last_run_at = Time.current
+      save
+    end
+
     def start_sync
       if next_url.present?
-        puts "Starting update, last update: #{last_run_at} (#{(Time.current - last_run_at).round(3)}s ago)".blue
+        puts Rainbow("Starting update, last update: #{last_run_at} (#{(Time.current - last_run_at).round(3)}s ago)").blue
         Consyncful.client.sync(next_url)
       else
-        puts 'Starting full refresh'.blue
+        puts Rainbow('Starting full refresh').blue
         Consyncful.client.sync(initial: true)
       end
     end
 
     def sync_items(sync, stats)
+      ids = []
       sync.each_page do |page|
         page.items.each do |item|
-          sync_item(ItemMapper.new(item), stats)
+          ids << sync_item(ItemMapper.new(item), stats)
         end
       end
+      ids
     end
 
     def sync_item(item, stats)
-      puts "syncing: #{item.id}".yellow
-      if item.deletion?
-        delete_model(item.id, stats)
-      else
-        create_or_update_model(item, stats)
-      end
-    end
-
-    def delete_model(id, stats)
-      Base.find_by(id: id).destroy
-      stats.record_deleted
-    rescue Mongoid::Errors::DocumentNotFound
-      puts "Deleted record not found: #{id}".yellow
-      nil
-    end
-
-    def create_or_update_model(item, stats)
-      return if item.type.nil?
-
-      instance = find_or_initialize_item(item)
-      update_stats(instance, stats)
-
-      reset_fields(instance)
-
-      item.mapped_fields(DEFAULT_LOCALE).each do |field, value|
-        instance[field] = value
-      end
-
-      instance[:sync_id] = id
-
-      instance.save
-    end
-
-    def find_or_initialize_item(item)
-      model_class(item.type).find_or_initialize_by(id: item.id)
-    end
-
-    def update_stats(instance, stats)
-      if instance.persisted?
-        stats.record_updated
-      else
-        stats.record_added
-      end
-    end
-
-    def model_class(type)
-      Base.model_map[type] || Base
-    end
-
-    def reset_fields(instance)
-      instance.attributes.each do |field_name, _value|
-        next if field_name.in? %w[_id _type]
-
-        instance[field_name] = nil
-      end
+      puts Rainbow("syncing: #{item.id}").yellow
+      PersistedItem.new(item, id, stats).persist
+      item.id
     end
   end
 end

data/lib/consyncful/tasks/consyncful.rake

@@ -19,12 +19,19 @@ namespace :consyncful do
   end
 
   task update_model_names: [:environment] do
-    Rails.application.eager_load!
-    puts 'Updating model names:'.blue
+    if Object.const_defined?('Zeitwerk::Loader') && Rails.application.config.autoloader.to_s == 'zeitwerk'
+      Zeitwerk::Loader.eager_load_all
+    else
+      Rails.application.eager_load!
+    end
+
+    puts Rainbow('Updating model names:').blue
+
     Consyncful::Base.model_map.each do |contentful_name, constant|
-      puts "#{contentful_name}: #{constant}".yellow
+      puts Rainbow("#{contentful_name}: #{constant}").yellow
       Consyncful::Base.where(contentful_type: contentful_name).update_all(_type: constant.to_s)
     end
+
     Consyncful::Base.where(:contentful_type.nin => Consyncful::Base.model_map.keys).update_all(_type: 'Consyncful::Base')
   end
 end

data/lib/consyncful/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Consyncful
-  VERSION = '0.3.0'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,15 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: consyncful
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Andy Anastasiadis-Gray
 - Montgomery Anderson
+- Greg Rogan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-07-02 00:00:00.000000000 Z
+date: 2021-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -17,28 +18,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -53,6 +68,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.79.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.79.0
 - !ruby/object:Gem::Dependency
   name: contentful
   requirement: !ruby/object:Gem::Requirement
@@ -73,6 +102,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: hooks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.1
 - !ruby/object:Gem::Dependency
   name: mongoid
   requirement: !ruby/object:Gem::Requirement
@@ -94,7 +137,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 8.0.0
 - !ruby/object:Gem::Dependency
-  name: term-ansicolor
+  name: rainbow
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -111,6 +154,7 @@ description:
 email:
 - andy@boost.co.nz
 - montgomery@boost.co.nz
+- greg@boost.co.nz
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -118,6 +162,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
+- ".rubocop_todo.yml"
 - ".ruby-version"
 - ".travis.yml"
 - Gemfile
@@ -133,6 +178,7 @@ files:
 - lib/consyncful.rb
 - lib/consyncful/base.rb
 - lib/consyncful/item_mapper.rb
+- lib/consyncful/persisted_item.rb
 - lib/consyncful/railtie.rb
 - lib/consyncful/stats.rb
 - lib/consyncful/sync.rb
@@ -157,8 +203,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Contentful to local database synchronisation for Rails