consyncful 0.2.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -0
- data/.rubocop_todo.yml +46 -0
- data/.ruby-version +1 -0
- data/.travis.yml +8 -2
- data/Gemfile +4 -2
- data/Gemfile.lock +72 -46
- data/README.md +20 -3
- data/Rakefile +5 -3
- data/bin/console +4 -3
- data/consyncful.gemspec +20 -16
- data/docker-compose.yml +1 -1
- data/lib/consyncful.rb +7 -5
- data/lib/consyncful/persisted_item.rb +74 -0
- data/lib/consyncful/stats.rb +4 -2
- data/lib/consyncful/sync.rb +36 -70
- data/lib/consyncful/tasks/consyncful.rake +11 -4
- data/lib/consyncful/version.rb +3 -1
- metadata +70 -24
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5798e871965ab5ca699b12f71a397ef37025d75547786ea804e1c38a9b0091ac
|
4
|
+
data.tar.gz: abfc466d422b44adc07abdc2d1b6bc36ff6e3dde8b57006d856bd809aaae86d5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8dbb9580e3aaa1aee124608323d877ab21f89da591976394bedd69c400682f88714b83598a14bacf32c18987089bfd84225273d2c868a4c17a033bb366c0ff3d
|
7
|
+
data.tar.gz: e3dc1aaf34278f60014f30c7f6487ba4558a7a0c9810ece3b44ee61e387a2036e52bb4eeb8955a94297a800f87ee2f6a5dedc85e26fc24d3e1bc91e256a6fbe7
|
data/.rubocop.yml
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
inherit_from: .rubocop_todo.yml
|
data/.rubocop_todo.yml
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
# This configuration was generated by
|
2
|
+
# `rubocop --auto-gen-config`
|
3
|
+
# on 2020-01-24 14:47:44 +1300 using RuboCop version 0.79.0.
|
4
|
+
# The point is for the user to remove these configuration records
|
5
|
+
# one by one as the offenses are removed from the code base.
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
8
|
+
|
9
|
+
# Offense count: 1
|
10
|
+
Metrics/AbcSize:
|
11
|
+
Max: 19
|
12
|
+
|
13
|
+
# Offense count: 7
|
14
|
+
# Configuration parameters: CountComments, ExcludedMethods.
|
15
|
+
# ExcludedMethods: refine
|
16
|
+
Metrics/BlockLength:
|
17
|
+
Max: 153
|
18
|
+
|
19
|
+
# Offense count: 4
|
20
|
+
# Cop supports --auto-correct.
|
21
|
+
# Configuration parameters: AutoCorrect, EnforcedStyle.
|
22
|
+
# SupportedStyles: nested, compact
|
23
|
+
Style/ClassAndModuleChildren:
|
24
|
+
Exclude:
|
25
|
+
- 'lib/consyncful/railtie.rb'
|
26
|
+
- 'spec/consyncful/base_spec.rb'
|
27
|
+
- 'spec/consyncful/sync_spec.rb'
|
28
|
+
|
29
|
+
# Offense count: 7
|
30
|
+
Style/Documentation:
|
31
|
+
Exclude:
|
32
|
+
- 'spec/**/*'
|
33
|
+
- 'test/**/*'
|
34
|
+
- 'lib/consyncful.rb'
|
35
|
+
- 'lib/consyncful/base.rb'
|
36
|
+
- 'lib/consyncful/item_mapper.rb'
|
37
|
+
- 'lib/consyncful/railtie.rb'
|
38
|
+
- 'lib/consyncful/stats.rb'
|
39
|
+
- 'lib/consyncful/sync.rb'
|
40
|
+
|
41
|
+
# Offense count: 42
|
42
|
+
# Cop supports --auto-correct.
|
43
|
+
# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
44
|
+
# URISchemes: http, https
|
45
|
+
Layout/LineLength:
|
46
|
+
Max: 226
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.5.5
|
data/.travis.yml
CHANGED
@@ -1,8 +1,14 @@
|
|
1
|
+
---
|
1
2
|
sudo: false
|
2
3
|
language: ruby
|
4
|
+
cache: bundler
|
3
5
|
rvm:
|
4
6
|
- 2.5.1
|
5
|
-
|
6
|
-
|
7
|
+
- 2.6.5
|
8
|
+
before_install: gem install bundler -v 2.1.0
|
7
9
|
services:
|
8
10
|
- mongodb
|
11
|
+
|
12
|
+
script:
|
13
|
+
- bundle exec rake spec
|
14
|
+
- bundle exec rubocop -P
|
data/Gemfile
CHANGED
@@ -1,6 +1,8 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
source 'https://rubygems.org'
|
4
|
+
|
5
|
+
git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
|
4
6
|
|
5
7
|
# Specify your gem's dependencies in consyncful.gemspec
|
6
8
|
gemspec
|
data/Gemfile.lock
CHANGED
@@ -1,82 +1,108 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
consyncful (0.
|
4
|
+
consyncful (0.4.0)
|
5
5
|
contentful (>= 2.11.1, < 3.0.0)
|
6
|
+
hooks (>= 0.4.1)
|
6
7
|
mongoid (>= 7.0.2, < 8.0.0)
|
7
|
-
|
8
|
+
rainbow
|
8
9
|
|
9
10
|
GEM
|
10
11
|
remote: https://rubygems.org/
|
11
12
|
specs:
|
12
|
-
activemodel (
|
13
|
-
activesupport (=
|
14
|
-
activesupport (
|
13
|
+
activemodel (6.0.3.4)
|
14
|
+
activesupport (= 6.0.3.4)
|
15
|
+
activesupport (6.0.3.4)
|
15
16
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
16
17
|
i18n (>= 0.7, < 2)
|
17
18
|
minitest (~> 5.1)
|
18
19
|
tzinfo (~> 1.1)
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
20
|
+
zeitwerk (~> 2.2, >= 2.2.2)
|
21
|
+
addressable (2.7.0)
|
22
|
+
public_suffix (>= 2.0.2, < 5.0)
|
23
|
+
ast (2.4.0)
|
24
|
+
bson (4.11.1)
|
25
|
+
concurrent-ruby (1.1.8)
|
26
|
+
contentful (2.15.4)
|
27
|
+
http (> 0.8, < 5.0)
|
25
28
|
multi_json (~> 1)
|
29
|
+
database_cleaner (1.8.3)
|
26
30
|
diff-lcs (1.3)
|
27
|
-
domain_name (0.5.
|
31
|
+
domain_name (0.5.20190701)
|
28
32
|
unf (>= 0.0.5, < 1.0.0)
|
29
|
-
|
33
|
+
ffi (1.14.2)
|
34
|
+
ffi-compiler (1.0.1)
|
35
|
+
ffi (>= 1.0.0)
|
36
|
+
rake
|
37
|
+
hooks (0.4.1)
|
38
|
+
uber (~> 0.0.14)
|
39
|
+
http (4.4.1)
|
30
40
|
addressable (~> 2.3)
|
31
41
|
http-cookie (~> 1.0)
|
32
|
-
http-form_data (~> 2.
|
33
|
-
|
42
|
+
http-form_data (~> 2.2)
|
43
|
+
http-parser (~> 1.2.0)
|
34
44
|
http-cookie (1.0.3)
|
35
45
|
domain_name (~> 0.5)
|
36
|
-
http-form_data (2.
|
37
|
-
|
38
|
-
|
46
|
+
http-form_data (2.3.0)
|
47
|
+
http-parser (1.2.3)
|
48
|
+
ffi-compiler (>= 1.0, < 2.0)
|
49
|
+
i18n (1.8.7)
|
39
50
|
concurrent-ruby (~> 1.0)
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
51
|
+
jaro_winkler (1.5.4)
|
52
|
+
minitest (5.14.3)
|
53
|
+
mongo (2.14.0)
|
54
|
+
bson (>= 4.8.2, < 5.0.0)
|
55
|
+
mongoid (7.2.0)
|
56
|
+
activemodel (>= 5.1, < 6.1)
|
57
|
+
mongo (>= 2.10.5, < 3.0.0)
|
58
|
+
multi_json (1.15.0)
|
59
|
+
parallel (1.19.1)
|
60
|
+
parser (2.7.0.2)
|
61
|
+
ast (~> 2.4.0)
|
62
|
+
public_suffix (4.0.6)
|
63
|
+
rainbow (3.0.0)
|
64
|
+
rake (13.0.1)
|
65
|
+
rspec (3.9.0)
|
66
|
+
rspec-core (~> 3.9.0)
|
67
|
+
rspec-expectations (~> 3.9.0)
|
68
|
+
rspec-mocks (~> 3.9.0)
|
69
|
+
rspec-core (3.9.1)
|
70
|
+
rspec-support (~> 3.9.1)
|
71
|
+
rspec-expectations (3.9.0)
|
56
72
|
diff-lcs (>= 1.2.0, < 2.0)
|
57
|
-
rspec-support (~> 3.
|
58
|
-
rspec-mocks (3.
|
73
|
+
rspec-support (~> 3.9.0)
|
74
|
+
rspec-mocks (3.9.1)
|
59
75
|
diff-lcs (>= 1.2.0, < 2.0)
|
60
|
-
rspec-support (~> 3.
|
61
|
-
rspec-support (3.
|
62
|
-
|
63
|
-
|
76
|
+
rspec-support (~> 3.9.0)
|
77
|
+
rspec-support (3.9.2)
|
78
|
+
rubocop (0.79.0)
|
79
|
+
jaro_winkler (~> 1.5.1)
|
80
|
+
parallel (~> 1.10)
|
81
|
+
parser (>= 2.7.0.1)
|
82
|
+
rainbow (>= 2.2.2, < 4.0)
|
83
|
+
ruby-progressbar (~> 1.7)
|
84
|
+
unicode-display_width (>= 1.4.0, < 1.7)
|
85
|
+
ruby-progressbar (1.10.1)
|
64
86
|
thread_safe (0.3.6)
|
65
|
-
|
66
|
-
tzinfo (1.2.5)
|
87
|
+
tzinfo (1.2.9)
|
67
88
|
thread_safe (~> 0.1)
|
89
|
+
uber (0.0.15)
|
68
90
|
unf (0.1.4)
|
69
91
|
unf_ext
|
70
|
-
unf_ext (0.0.7.
|
92
|
+
unf_ext (0.0.7.7)
|
93
|
+
unicode-display_width (1.6.1)
|
94
|
+
zeitwerk (2.4.2)
|
71
95
|
|
72
96
|
PLATFORMS
|
73
97
|
ruby
|
74
98
|
|
75
99
|
DEPENDENCIES
|
76
|
-
bundler (~>
|
100
|
+
bundler (~> 2)
|
77
101
|
consyncful!
|
78
|
-
|
102
|
+
database_cleaner
|
103
|
+
rake (~> 13.0)
|
79
104
|
rspec (~> 3.0)
|
105
|
+
rubocop (= 0.79.0)
|
80
106
|
|
81
107
|
BUNDLED WITH
|
82
|
-
1.
|
108
|
+
2.1.0
|
data/README.md
CHANGED
@@ -1,5 +1,4 @@
|
|
1
|
-
# Consyncful
|
2
|
-
[![Build Status](https://travis-ci.com/boost/consyncful.svg?branch=master)](https://travis-ci.com/boost/consyncful)
|
1
|
+
# Consyncful [![Build Status](https://travis-ci.org/boost/consyncful.svg?branch=master)](https://travis-ci.org/boost/consyncful)
|
3
2
|
|
4
3
|
Contentful -> local database synchronisation for Rails
|
5
4
|
|
@@ -90,7 +89,7 @@ To run a syncronization process run:
|
|
90
89
|
|
91
90
|
The first time you run this it will download all the contentful content, it will then check every 15 seconds for changes to the content and update/delete records in the database when changes are made in contentful.
|
92
91
|
|
93
|
-
If you want to
|
92
|
+
If you want to syncronise from scratch run:
|
94
93
|
|
95
94
|
$ rake consyncful:refresh
|
96
95
|
|
@@ -135,6 +134,24 @@ Because all contentful models are stored as polymorphic subtypes of Consyncful::
|
|
135
134
|
Consyncful::Base.where(title: 'a title') # [ #<ModelName>, #<OtherModelName> ]
|
136
135
|
```
|
137
136
|
|
137
|
+
### Sync callbacks
|
138
|
+
|
139
|
+
You may want to attach some application logic to happen before or after a sync run, for example to update caches or something.
|
140
|
+
|
141
|
+
Callbacks can be registered using:
|
142
|
+
|
143
|
+
```ruby
|
144
|
+
Consyncful::Sync.before_run do
|
145
|
+
# do something before the run
|
146
|
+
end
|
147
|
+
```
|
148
|
+
|
149
|
+
```ruby
|
150
|
+
Consyncful::Sync.after_run do |updated_ids|
|
151
|
+
# invalidate cache for updated_ids, or something
|
152
|
+
end
|
153
|
+
```
|
154
|
+
|
138
155
|
## Limitations
|
139
156
|
|
140
157
|
### Locales
|
data/Rakefile
CHANGED
data/bin/console
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
2
3
|
|
3
|
-
require
|
4
|
-
require
|
4
|
+
require 'bundler/setup'
|
5
|
+
require 'consyncful'
|
5
6
|
|
6
7
|
# You can add fixtures and/or initialization code here to make experimenting
|
7
8
|
# with your gem easier. You can also use a different console, if you like.
|
@@ -10,5 +11,5 @@ require "consyncful"
|
|
10
11
|
# require "pry"
|
11
12
|
# Pry.start
|
12
13
|
|
13
|
-
require
|
14
|
+
require 'irb'
|
14
15
|
IRB.start(__FILE__)
|
data/consyncful.gemspec
CHANGED
@@ -1,17 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
|
2
|
-
lib = File.expand_path(
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require
|
5
|
+
require 'consyncful/version'
|
5
6
|
|
6
7
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name =
|
8
|
+
spec.name = 'consyncful'
|
8
9
|
spec.version = Consyncful::VERSION
|
9
|
-
spec.authors = [
|
10
|
-
spec.email = [
|
10
|
+
spec.authors = ['Andy Anastasiadis-Gray', 'Montgomery Anderson']
|
11
|
+
spec.email = ['andy@boost.co.nz', 'montgomery@boost.co.nz']
|
11
12
|
|
12
|
-
spec.summary =
|
13
|
-
|
14
|
-
spec.license =
|
13
|
+
spec.summary = 'Contentful to local database synchronisation for Rails'
|
14
|
+
spec.homepage = 'https://github.com/boost/consyncful'
|
15
|
+
spec.license = 'MIT'
|
15
16
|
|
16
17
|
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
17
18
|
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
@@ -25,15 +26,18 @@ Gem::Specification.new do |spec|
|
|
25
26
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
26
27
|
f.match(%r{^(test|spec|features)/})
|
27
28
|
end
|
28
|
-
spec.bindir =
|
29
|
+
spec.bindir = 'exe'
|
29
30
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
30
|
-
spec.require_paths = [
|
31
|
+
spec.require_paths = ['lib']
|
31
32
|
|
32
|
-
spec.add_development_dependency
|
33
|
-
spec.add_development_dependency
|
34
|
-
spec.add_development_dependency
|
33
|
+
spec.add_development_dependency 'bundler', '~> 2'
|
34
|
+
spec.add_development_dependency 'database_cleaner'
|
35
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
36
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
37
|
+
spec.add_development_dependency 'rubocop', '0.79.0'
|
35
38
|
|
36
|
-
spec.add_dependency
|
37
|
-
spec.add_dependency
|
38
|
-
spec.add_dependency '
|
39
|
+
spec.add_dependency 'contentful', ['>=2.11.1', '<3.0.0']
|
40
|
+
spec.add_dependency 'hooks', '>=0.4.1'
|
41
|
+
spec.add_dependency 'mongoid', ['>=7.0.2', '<8.0.0']
|
42
|
+
spec.add_dependency 'rainbow'
|
39
43
|
end
|
data/docker-compose.yml
CHANGED
data/lib/consyncful.rb
CHANGED
@@ -1,12 +1,14 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'consyncful/version'
|
2
4
|
|
3
5
|
require 'mongoid'
|
4
6
|
require 'contentful'
|
5
7
|
|
6
|
-
require
|
7
|
-
require
|
8
|
+
require 'consyncful/base'
|
9
|
+
require 'consyncful/sync'
|
8
10
|
|
9
|
-
require
|
11
|
+
require 'consyncful/railtie' if defined?(Rails)
|
10
12
|
|
11
13
|
module Consyncful
|
12
14
|
class << self
|
@@ -32,7 +34,7 @@ module Consyncful
|
|
32
34
|
DEFAULT_CLIENT_OPTIONS = {
|
33
35
|
reuse_entries: true,
|
34
36
|
api_url: 'cdn.contentful.com'
|
35
|
-
}
|
37
|
+
}.freeze
|
36
38
|
|
37
39
|
def self.client
|
38
40
|
@client ||= begin
|
@@ -0,0 +1,74 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Consyncful
|
4
|
+
# Takes a mapped item from contentful and applies it to the local storage.
|
5
|
+
class PersistedItem
|
6
|
+
DEFAULT_LOCALE = 'en-NZ'
|
7
|
+
|
8
|
+
def initialize(item, sync_id, stats)
|
9
|
+
@item = item
|
10
|
+
@sync_id = sync_id
|
11
|
+
@stats = stats
|
12
|
+
end
|
13
|
+
|
14
|
+
def persist
|
15
|
+
puts Rainbow("syncing: #{@item.id}").yellow
|
16
|
+
if @item.deletion?
|
17
|
+
delete_model(@item.id, @stats)
|
18
|
+
else
|
19
|
+
create_or_update_model(@item, @sync_id, @stats)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def delete_model(id, stats)
|
26
|
+
Base.find_by(id: id).destroy
|
27
|
+
stats.record_deleted
|
28
|
+
rescue Mongoid::Errors::DocumentNotFound
|
29
|
+
puts Rainbow("Deleted record not found: #{id}").yellow
|
30
|
+
nil
|
31
|
+
end
|
32
|
+
|
33
|
+
def create_or_update_model(item, sync_id, stats)
|
34
|
+
return if item.type.nil?
|
35
|
+
|
36
|
+
instance = find_or_initialize_item(item)
|
37
|
+
update_stats(instance, stats)
|
38
|
+
|
39
|
+
reset_fields(instance)
|
40
|
+
|
41
|
+
item.mapped_fields(DEFAULT_LOCALE).each do |field, value|
|
42
|
+
instance[field] = value
|
43
|
+
end
|
44
|
+
|
45
|
+
instance[:sync_id] = sync_id
|
46
|
+
|
47
|
+
instance.save
|
48
|
+
end
|
49
|
+
|
50
|
+
def find_or_initialize_item(item)
|
51
|
+
model_class(item.type).find_or_initialize_by(id: item.id)
|
52
|
+
end
|
53
|
+
|
54
|
+
def update_stats(instance, stats)
|
55
|
+
if instance.persisted?
|
56
|
+
stats.record_updated
|
57
|
+
else
|
58
|
+
stats.record_added
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def model_class(type)
|
63
|
+
Base.model_map[type] || Base
|
64
|
+
end
|
65
|
+
|
66
|
+
def reset_fields(instance)
|
67
|
+
instance.attributes.each do |field_name, _value|
|
68
|
+
next if field_name.in? %w[_id _type]
|
69
|
+
|
70
|
+
instance[field_name] = nil
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
data/lib/consyncful/stats.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'rainbow'
|
4
|
+
|
3
5
|
module Consyncful
|
4
6
|
class Stats
|
5
7
|
def initialize
|
@@ -23,9 +25,9 @@ module Consyncful
|
|
23
25
|
end
|
24
26
|
|
25
27
|
def print_stats
|
26
|
-
puts "Added: #{@stats[:records_added]}, \
|
28
|
+
puts Rainbow("Added: #{@stats[:records_added]}, \
|
27
29
|
updated: #{@stats[:records_updated]}, \
|
28
|
-
deleted: #{@stats[:records_deleted]}".blue
|
30
|
+
deleted: #{@stats[:records_deleted]}").blue
|
29
31
|
end
|
30
32
|
end
|
31
33
|
end
|
data/lib/consyncful/sync.rb
CHANGED
@@ -1,16 +1,18 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require '
|
3
|
+
require 'rainbow'
|
4
4
|
require 'consyncful/item_mapper'
|
5
|
+
require 'consyncful/persisted_item'
|
5
6
|
require 'consyncful/stats'
|
6
|
-
|
7
|
-
class String
|
8
|
-
include Term::ANSIColor
|
9
|
-
end
|
7
|
+
require 'hooks'
|
10
8
|
|
11
9
|
module Consyncful
|
12
10
|
class Sync
|
13
11
|
include Mongoid::Document
|
12
|
+
include Hooks
|
13
|
+
|
14
|
+
define_hook :before_run
|
15
|
+
define_hook :after_run
|
14
16
|
|
15
17
|
DEFAULT_LOCALE = 'en-NZ'
|
16
18
|
|
@@ -21,109 +23,73 @@ module Consyncful
|
|
21
23
|
last || new
|
22
24
|
end
|
23
25
|
|
24
|
-
def self.
|
26
|
+
def self.fresh
|
25
27
|
destroy_all
|
26
|
-
|
28
|
+
latest
|
27
29
|
end
|
28
30
|
|
29
|
-
def
|
30
|
-
|
31
|
-
|
31
|
+
def drop_stale
|
32
|
+
stale = Base.where(:sync_id.ne => id, :sync_id.exists => true)
|
33
|
+
puts Rainbow("Dropping #{stale.count} records that haven't been touched in this sync").red
|
34
|
+
stale.destroy
|
32
35
|
end
|
33
36
|
|
34
37
|
def run
|
38
|
+
run_hook :before_run
|
39
|
+
|
35
40
|
stats = Consyncful::Stats.new
|
36
41
|
load_all_models
|
37
42
|
|
38
43
|
sync = start_sync
|
39
44
|
|
40
|
-
sync_items(sync, stats)
|
45
|
+
changed_ids = sync_items(sync, stats)
|
41
46
|
|
42
|
-
|
43
|
-
|
44
|
-
|
47
|
+
drop_stale
|
48
|
+
|
49
|
+
update_run(sync.next_sync_url)
|
45
50
|
stats.print_stats
|
51
|
+
|
52
|
+
run_hook :after_run, changed_ids
|
46
53
|
end
|
47
54
|
|
48
55
|
private
|
49
56
|
|
50
57
|
def load_all_models
|
51
58
|
return unless defined? Rails
|
59
|
+
|
52
60
|
Rails.application.eager_load!
|
53
61
|
end
|
54
62
|
|
63
|
+
def update_run(next_url)
|
64
|
+
self.next_url = next_url
|
65
|
+
self.last_run_at = Time.current
|
66
|
+
save
|
67
|
+
end
|
68
|
+
|
55
69
|
def start_sync
|
56
70
|
if next_url.present?
|
57
|
-
puts "Starting update, last update: #{last_run_at} (#{(Time.current - last_run_at).round(3)}s ago)".blue
|
71
|
+
puts Rainbow("Starting update, last update: #{last_run_at} (#{(Time.current - last_run_at).round(3)}s ago)").blue
|
58
72
|
Consyncful.client.sync(next_url)
|
59
73
|
else
|
60
|
-
puts 'Starting full refresh'.blue
|
74
|
+
puts Rainbow('Starting full refresh').blue
|
61
75
|
Consyncful.client.sync(initial: true)
|
62
76
|
end
|
63
77
|
end
|
64
78
|
|
65
79
|
def sync_items(sync, stats)
|
80
|
+
ids = []
|
66
81
|
sync.each_page do |page|
|
67
82
|
page.items.each do |item|
|
68
|
-
sync_item(ItemMapper.new(item), stats)
|
83
|
+
ids << sync_item(ItemMapper.new(item), stats)
|
69
84
|
end
|
70
85
|
end
|
86
|
+
ids
|
71
87
|
end
|
72
88
|
|
73
89
|
def sync_item(item, stats)
|
74
|
-
puts "syncing: #{item.id}".yellow
|
75
|
-
|
76
|
-
|
77
|
-
else
|
78
|
-
create_or_update_model(item, stats)
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
def delete_model(id, stats)
|
83
|
-
Base.find_by(id: id).destroy
|
84
|
-
stats.record_deleted
|
85
|
-
rescue Mongoid::Errors::DocumentNotFound
|
86
|
-
puts "Deleted record not found: #{id}".yellow
|
87
|
-
nil
|
88
|
-
end
|
89
|
-
|
90
|
-
def create_or_update_model(item, stats)
|
91
|
-
return if item.type.nil?
|
92
|
-
|
93
|
-
instance = find_or_initialize_item(item)
|
94
|
-
update_stats(instance, stats)
|
95
|
-
|
96
|
-
reset_fields(instance)
|
97
|
-
|
98
|
-
item.mapped_fields(DEFAULT_LOCALE).each do |field, value|
|
99
|
-
instance[field] = value
|
100
|
-
end
|
101
|
-
|
102
|
-
instance.save
|
103
|
-
end
|
104
|
-
|
105
|
-
def find_or_initialize_item(item)
|
106
|
-
model_class(item.type).find_or_initialize_by(id: item.id)
|
107
|
-
end
|
108
|
-
|
109
|
-
def update_stats(instance, stats)
|
110
|
-
if instance.persisted?
|
111
|
-
stats.record_updated
|
112
|
-
else
|
113
|
-
stats.record_added
|
114
|
-
end
|
115
|
-
end
|
116
|
-
|
117
|
-
def model_class(type)
|
118
|
-
Base.model_map[type] || Base
|
119
|
-
end
|
120
|
-
|
121
|
-
def reset_fields(instance)
|
122
|
-
instance.attributes.each do |field_name, _value|
|
123
|
-
next if field_name.in? %w[_id _type]
|
124
|
-
|
125
|
-
instance[field_name] = nil
|
126
|
-
end
|
90
|
+
puts Rainbow("syncing: #{item.id}").yellow
|
91
|
+
PersistedItem.new(item, id, stats).persist
|
92
|
+
item.id
|
127
93
|
end
|
128
94
|
end
|
129
95
|
end
|
@@ -9,7 +9,7 @@ namespace :consyncful do
|
|
9
9
|
Consyncful::Sync.fresh.run
|
10
10
|
end
|
11
11
|
|
12
|
-
task :sync, [:seconds] => [
|
12
|
+
task :sync, [:seconds] => %i[environment update_model_names] do |_task, args|
|
13
13
|
seconds = args[:seconds].to_i
|
14
14
|
seconds = 15 if seconds.zero?
|
15
15
|
loop do
|
@@ -19,12 +19,19 @@ namespace :consyncful do
|
|
19
19
|
end
|
20
20
|
|
21
21
|
task update_model_names: [:environment] do
|
22
|
-
Rails.application.
|
23
|
-
|
22
|
+
if Object.const_defined?('Zeitwerk::Loader') && Rails.application.config.autoloader.to_s == 'zeitwerk'
|
23
|
+
Zeitwerk::Loader.eager_load_all
|
24
|
+
else
|
25
|
+
Rails.application.eager_load!
|
26
|
+
end
|
27
|
+
|
28
|
+
puts Rainbow('Updating model names:').blue
|
29
|
+
|
24
30
|
Consyncful::Base.model_map.each do |contentful_name, constant|
|
25
|
-
puts "#{contentful_name}: #{constant}".yellow
|
31
|
+
puts Rainbow("#{contentful_name}: #{constant}").yellow
|
26
32
|
Consyncful::Base.where(contentful_type: contentful_name).update_all(_type: constant.to_s)
|
27
33
|
end
|
34
|
+
|
28
35
|
Consyncful::Base.where(:contentful_type.nin => Consyncful::Base.model_map.keys).update_all(_type: 'Consyncful::Base')
|
29
36
|
end
|
30
37
|
end
|
data/lib/consyncful/version.rb
CHANGED
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: consyncful
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andy Anastasiadis-Gray
|
8
8
|
- Montgomery Anderson
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2021-02-03 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -17,28 +17,42 @@ dependencies:
|
|
17
17
|
requirements:
|
18
18
|
- - "~>"
|
19
19
|
- !ruby/object:Gem::Version
|
20
|
-
version: '
|
20
|
+
version: '2'
|
21
21
|
type: :development
|
22
22
|
prerelease: false
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
25
|
- - "~>"
|
26
26
|
- !ruby/object:Gem::Version
|
27
|
-
version: '
|
27
|
+
version: '2'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: database_cleaner
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '0'
|
35
|
+
type: :development
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0'
|
28
42
|
- !ruby/object:Gem::Dependency
|
29
43
|
name: rake
|
30
44
|
requirement: !ruby/object:Gem::Requirement
|
31
45
|
requirements:
|
32
46
|
- - "~>"
|
33
47
|
- !ruby/object:Gem::Version
|
34
|
-
version: '
|
48
|
+
version: '13.0'
|
35
49
|
type: :development
|
36
50
|
prerelease: false
|
37
51
|
version_requirements: !ruby/object:Gem::Requirement
|
38
52
|
requirements:
|
39
53
|
- - "~>"
|
40
54
|
- !ruby/object:Gem::Version
|
41
|
-
version: '
|
55
|
+
version: '13.0'
|
42
56
|
- !ruby/object:Gem::Dependency
|
43
57
|
name: rspec
|
44
58
|
requirement: !ruby/object:Gem::Requirement
|
@@ -54,47 +68,75 @@ dependencies:
|
|
54
68
|
- !ruby/object:Gem::Version
|
55
69
|
version: '3.0'
|
56
70
|
- !ruby/object:Gem::Dependency
|
57
|
-
name:
|
71
|
+
name: rubocop
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - '='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: 0.79.0
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - '='
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: 0.79.0
|
84
|
+
- !ruby/object:Gem::Dependency
|
85
|
+
name: contentful
|
58
86
|
requirement: !ruby/object:Gem::Requirement
|
59
87
|
requirements:
|
60
88
|
- - ">="
|
61
89
|
- !ruby/object:Gem::Version
|
62
|
-
version:
|
90
|
+
version: 2.11.1
|
63
91
|
- - "<"
|
64
92
|
- !ruby/object:Gem::Version
|
65
|
-
version:
|
93
|
+
version: 3.0.0
|
66
94
|
type: :runtime
|
67
95
|
prerelease: false
|
68
96
|
version_requirements: !ruby/object:Gem::Requirement
|
69
97
|
requirements:
|
70
98
|
- - ">="
|
71
99
|
- !ruby/object:Gem::Version
|
72
|
-
version:
|
100
|
+
version: 2.11.1
|
73
101
|
- - "<"
|
74
102
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
103
|
+
version: 3.0.0
|
76
104
|
- !ruby/object:Gem::Dependency
|
77
|
-
name:
|
105
|
+
name: hooks
|
78
106
|
requirement: !ruby/object:Gem::Requirement
|
79
107
|
requirements:
|
80
108
|
- - ">="
|
81
109
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
110
|
+
version: 0.4.1
|
111
|
+
type: :runtime
|
112
|
+
prerelease: false
|
113
|
+
version_requirements: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 0.4.1
|
118
|
+
- !ruby/object:Gem::Dependency
|
119
|
+
name: mongoid
|
120
|
+
requirement: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: 7.0.2
|
83
125
|
- - "<"
|
84
126
|
- !ruby/object:Gem::Version
|
85
|
-
version:
|
127
|
+
version: 8.0.0
|
86
128
|
type: :runtime
|
87
129
|
prerelease: false
|
88
130
|
version_requirements: !ruby/object:Gem::Requirement
|
89
131
|
requirements:
|
90
132
|
- - ">="
|
91
133
|
- !ruby/object:Gem::Version
|
92
|
-
version:
|
134
|
+
version: 7.0.2
|
93
135
|
- - "<"
|
94
136
|
- !ruby/object:Gem::Version
|
95
|
-
version:
|
137
|
+
version: 8.0.0
|
96
138
|
- !ruby/object:Gem::Dependency
|
97
|
-
name:
|
139
|
+
name: rainbow
|
98
140
|
requirement: !ruby/object:Gem::Requirement
|
99
141
|
requirements:
|
100
142
|
- - ">="
|
@@ -107,7 +149,7 @@ dependencies:
|
|
107
149
|
- - ">="
|
108
150
|
- !ruby/object:Gem::Version
|
109
151
|
version: '0'
|
110
|
-
description:
|
152
|
+
description:
|
111
153
|
email:
|
112
154
|
- andy@boost.co.nz
|
113
155
|
- montgomery@boost.co.nz
|
@@ -117,6 +159,9 @@ extra_rdoc_files: []
|
|
117
159
|
files:
|
118
160
|
- ".gitignore"
|
119
161
|
- ".rspec"
|
162
|
+
- ".rubocop.yml"
|
163
|
+
- ".rubocop_todo.yml"
|
164
|
+
- ".ruby-version"
|
120
165
|
- ".travis.yml"
|
121
166
|
- Gemfile
|
122
167
|
- Gemfile.lock
|
@@ -131,16 +176,17 @@ files:
|
|
131
176
|
- lib/consyncful.rb
|
132
177
|
- lib/consyncful/base.rb
|
133
178
|
- lib/consyncful/item_mapper.rb
|
179
|
+
- lib/consyncful/persisted_item.rb
|
134
180
|
- lib/consyncful/railtie.rb
|
135
181
|
- lib/consyncful/stats.rb
|
136
182
|
- lib/consyncful/sync.rb
|
137
183
|
- lib/consyncful/tasks/consyncful.rake
|
138
184
|
- lib/consyncful/version.rb
|
139
|
-
homepage:
|
185
|
+
homepage: https://github.com/boost/consyncful
|
140
186
|
licenses:
|
141
187
|
- MIT
|
142
188
|
metadata: {}
|
143
|
-
post_install_message:
|
189
|
+
post_install_message:
|
144
190
|
rdoc_options: []
|
145
191
|
require_paths:
|
146
192
|
- lib
|
@@ -155,9 +201,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
155
201
|
- !ruby/object:Gem::Version
|
156
202
|
version: '0'
|
157
203
|
requirements: []
|
158
|
-
rubyforge_project:
|
159
|
-
rubygems_version: 2.7.6
|
160
|
-
signing_key:
|
204
|
+
rubyforge_project:
|
205
|
+
rubygems_version: 2.7.6.2
|
206
|
+
signing_key:
|
161
207
|
specification_version: 4
|
162
208
|
summary: Contentful to local database synchronisation for Rails
|
163
209
|
test_files: []
|