consyncful 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -0
- data/.rubocop_todo.yml +46 -0
- data/.ruby-version +1 -0
- data/.travis.yml +8 -2
- data/Gemfile +4 -2
- data/Gemfile.lock +72 -46
- data/README.md +20 -3
- data/Rakefile +5 -3
- data/bin/console +4 -3
- data/consyncful.gemspec +20 -16
- data/docker-compose.yml +1 -1
- data/lib/consyncful.rb +7 -5
- data/lib/consyncful/persisted_item.rb +74 -0
- data/lib/consyncful/stats.rb +4 -2
- data/lib/consyncful/sync.rb +36 -70
- data/lib/consyncful/tasks/consyncful.rake +11 -4
- data/lib/consyncful/version.rb +3 -1
- metadata +70 -24
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5798e871965ab5ca699b12f71a397ef37025d75547786ea804e1c38a9b0091ac
|
|
4
|
+
data.tar.gz: abfc466d422b44adc07abdc2d1b6bc36ff6e3dde8b57006d856bd809aaae86d5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8dbb9580e3aaa1aee124608323d877ab21f89da591976394bedd69c400682f88714b83598a14bacf32c18987089bfd84225273d2c868a4c17a033bb366c0ff3d
|
|
7
|
+
data.tar.gz: e3dc1aaf34278f60014f30c7f6487ba4558a7a0c9810ece3b44ee61e387a2036e52bb4eeb8955a94297a800f87ee2f6a5dedc85e26fc24d3e1bc91e256a6fbe7
|
data/.rubocop.yml
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
inherit_from: .rubocop_todo.yml
|
data/.rubocop_todo.yml
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# This configuration was generated by
|
|
2
|
+
# `rubocop --auto-gen-config`
|
|
3
|
+
# on 2020-01-24 14:47:44 +1300 using RuboCop version 0.79.0.
|
|
4
|
+
# The point is for the user to remove these configuration records
|
|
5
|
+
# one by one as the offenses are removed from the code base.
|
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
|
8
|
+
|
|
9
|
+
# Offense count: 1
|
|
10
|
+
Metrics/AbcSize:
|
|
11
|
+
Max: 19
|
|
12
|
+
|
|
13
|
+
# Offense count: 7
|
|
14
|
+
# Configuration parameters: CountComments, ExcludedMethods.
|
|
15
|
+
# ExcludedMethods: refine
|
|
16
|
+
Metrics/BlockLength:
|
|
17
|
+
Max: 153
|
|
18
|
+
|
|
19
|
+
# Offense count: 4
|
|
20
|
+
# Cop supports --auto-correct.
|
|
21
|
+
# Configuration parameters: AutoCorrect, EnforcedStyle.
|
|
22
|
+
# SupportedStyles: nested, compact
|
|
23
|
+
Style/ClassAndModuleChildren:
|
|
24
|
+
Exclude:
|
|
25
|
+
- 'lib/consyncful/railtie.rb'
|
|
26
|
+
- 'spec/consyncful/base_spec.rb'
|
|
27
|
+
- 'spec/consyncful/sync_spec.rb'
|
|
28
|
+
|
|
29
|
+
# Offense count: 7
|
|
30
|
+
Style/Documentation:
|
|
31
|
+
Exclude:
|
|
32
|
+
- 'spec/**/*'
|
|
33
|
+
- 'test/**/*'
|
|
34
|
+
- 'lib/consyncful.rb'
|
|
35
|
+
- 'lib/consyncful/base.rb'
|
|
36
|
+
- 'lib/consyncful/item_mapper.rb'
|
|
37
|
+
- 'lib/consyncful/railtie.rb'
|
|
38
|
+
- 'lib/consyncful/stats.rb'
|
|
39
|
+
- 'lib/consyncful/sync.rb'
|
|
40
|
+
|
|
41
|
+
# Offense count: 42
|
|
42
|
+
# Cop supports --auto-correct.
|
|
43
|
+
# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
|
44
|
+
# URISchemes: http, https
|
|
45
|
+
Layout/LineLength:
|
|
46
|
+
Max: 226
|
data/.ruby-version
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
2.5.5
|
data/.travis.yml
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
|
+
---
|
|
1
2
|
sudo: false
|
|
2
3
|
language: ruby
|
|
4
|
+
cache: bundler
|
|
3
5
|
rvm:
|
|
4
6
|
- 2.5.1
|
|
5
|
-
|
|
6
|
-
|
|
7
|
+
- 2.6.5
|
|
8
|
+
before_install: gem install bundler -v 2.1.0
|
|
7
9
|
services:
|
|
8
10
|
- mongodb
|
|
11
|
+
|
|
12
|
+
script:
|
|
13
|
+
- bundle exec rake spec
|
|
14
|
+
- bundle exec rubocop -P
|
data/Gemfile
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
source 'https://rubygems.org'
|
|
4
|
+
|
|
5
|
+
git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
|
|
4
6
|
|
|
5
7
|
# Specify your gem's dependencies in consyncful.gemspec
|
|
6
8
|
gemspec
|
data/Gemfile.lock
CHANGED
|
@@ -1,82 +1,108 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
consyncful (0.
|
|
4
|
+
consyncful (0.4.0)
|
|
5
5
|
contentful (>= 2.11.1, < 3.0.0)
|
|
6
|
+
hooks (>= 0.4.1)
|
|
6
7
|
mongoid (>= 7.0.2, < 8.0.0)
|
|
7
|
-
|
|
8
|
+
rainbow
|
|
8
9
|
|
|
9
10
|
GEM
|
|
10
11
|
remote: https://rubygems.org/
|
|
11
12
|
specs:
|
|
12
|
-
activemodel (
|
|
13
|
-
activesupport (=
|
|
14
|
-
activesupport (
|
|
13
|
+
activemodel (6.0.3.4)
|
|
14
|
+
activesupport (= 6.0.3.4)
|
|
15
|
+
activesupport (6.0.3.4)
|
|
15
16
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
16
17
|
i18n (>= 0.7, < 2)
|
|
17
18
|
minitest (~> 5.1)
|
|
18
19
|
tzinfo (~> 1.1)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
20
|
+
zeitwerk (~> 2.2, >= 2.2.2)
|
|
21
|
+
addressable (2.7.0)
|
|
22
|
+
public_suffix (>= 2.0.2, < 5.0)
|
|
23
|
+
ast (2.4.0)
|
|
24
|
+
bson (4.11.1)
|
|
25
|
+
concurrent-ruby (1.1.8)
|
|
26
|
+
contentful (2.15.4)
|
|
27
|
+
http (> 0.8, < 5.0)
|
|
25
28
|
multi_json (~> 1)
|
|
29
|
+
database_cleaner (1.8.3)
|
|
26
30
|
diff-lcs (1.3)
|
|
27
|
-
domain_name (0.5.
|
|
31
|
+
domain_name (0.5.20190701)
|
|
28
32
|
unf (>= 0.0.5, < 1.0.0)
|
|
29
|
-
|
|
33
|
+
ffi (1.14.2)
|
|
34
|
+
ffi-compiler (1.0.1)
|
|
35
|
+
ffi (>= 1.0.0)
|
|
36
|
+
rake
|
|
37
|
+
hooks (0.4.1)
|
|
38
|
+
uber (~> 0.0.14)
|
|
39
|
+
http (4.4.1)
|
|
30
40
|
addressable (~> 2.3)
|
|
31
41
|
http-cookie (~> 1.0)
|
|
32
|
-
http-form_data (~> 2.
|
|
33
|
-
|
|
42
|
+
http-form_data (~> 2.2)
|
|
43
|
+
http-parser (~> 1.2.0)
|
|
34
44
|
http-cookie (1.0.3)
|
|
35
45
|
domain_name (~> 0.5)
|
|
36
|
-
http-form_data (2.
|
|
37
|
-
|
|
38
|
-
|
|
46
|
+
http-form_data (2.3.0)
|
|
47
|
+
http-parser (1.2.3)
|
|
48
|
+
ffi-compiler (>= 1.0, < 2.0)
|
|
49
|
+
i18n (1.8.7)
|
|
39
50
|
concurrent-ruby (~> 1.0)
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
51
|
+
jaro_winkler (1.5.4)
|
|
52
|
+
minitest (5.14.3)
|
|
53
|
+
mongo (2.14.0)
|
|
54
|
+
bson (>= 4.8.2, < 5.0.0)
|
|
55
|
+
mongoid (7.2.0)
|
|
56
|
+
activemodel (>= 5.1, < 6.1)
|
|
57
|
+
mongo (>= 2.10.5, < 3.0.0)
|
|
58
|
+
multi_json (1.15.0)
|
|
59
|
+
parallel (1.19.1)
|
|
60
|
+
parser (2.7.0.2)
|
|
61
|
+
ast (~> 2.4.0)
|
|
62
|
+
public_suffix (4.0.6)
|
|
63
|
+
rainbow (3.0.0)
|
|
64
|
+
rake (13.0.1)
|
|
65
|
+
rspec (3.9.0)
|
|
66
|
+
rspec-core (~> 3.9.0)
|
|
67
|
+
rspec-expectations (~> 3.9.0)
|
|
68
|
+
rspec-mocks (~> 3.9.0)
|
|
69
|
+
rspec-core (3.9.1)
|
|
70
|
+
rspec-support (~> 3.9.1)
|
|
71
|
+
rspec-expectations (3.9.0)
|
|
56
72
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
57
|
-
rspec-support (~> 3.
|
|
58
|
-
rspec-mocks (3.
|
|
73
|
+
rspec-support (~> 3.9.0)
|
|
74
|
+
rspec-mocks (3.9.1)
|
|
59
75
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
60
|
-
rspec-support (~> 3.
|
|
61
|
-
rspec-support (3.
|
|
62
|
-
|
|
63
|
-
|
|
76
|
+
rspec-support (~> 3.9.0)
|
|
77
|
+
rspec-support (3.9.2)
|
|
78
|
+
rubocop (0.79.0)
|
|
79
|
+
jaro_winkler (~> 1.5.1)
|
|
80
|
+
parallel (~> 1.10)
|
|
81
|
+
parser (>= 2.7.0.1)
|
|
82
|
+
rainbow (>= 2.2.2, < 4.0)
|
|
83
|
+
ruby-progressbar (~> 1.7)
|
|
84
|
+
unicode-display_width (>= 1.4.0, < 1.7)
|
|
85
|
+
ruby-progressbar (1.10.1)
|
|
64
86
|
thread_safe (0.3.6)
|
|
65
|
-
|
|
66
|
-
tzinfo (1.2.5)
|
|
87
|
+
tzinfo (1.2.9)
|
|
67
88
|
thread_safe (~> 0.1)
|
|
89
|
+
uber (0.0.15)
|
|
68
90
|
unf (0.1.4)
|
|
69
91
|
unf_ext
|
|
70
|
-
unf_ext (0.0.7.
|
|
92
|
+
unf_ext (0.0.7.7)
|
|
93
|
+
unicode-display_width (1.6.1)
|
|
94
|
+
zeitwerk (2.4.2)
|
|
71
95
|
|
|
72
96
|
PLATFORMS
|
|
73
97
|
ruby
|
|
74
98
|
|
|
75
99
|
DEPENDENCIES
|
|
76
|
-
bundler (~>
|
|
100
|
+
bundler (~> 2)
|
|
77
101
|
consyncful!
|
|
78
|
-
|
|
102
|
+
database_cleaner
|
|
103
|
+
rake (~> 13.0)
|
|
79
104
|
rspec (~> 3.0)
|
|
105
|
+
rubocop (= 0.79.0)
|
|
80
106
|
|
|
81
107
|
BUNDLED WITH
|
|
82
|
-
1.
|
|
108
|
+
2.1.0
|
data/README.md
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
# Consyncful
|
|
2
|
-
[](https://travis-ci.com/boost/consyncful)
|
|
1
|
+
# Consyncful [](https://travis-ci.org/boost/consyncful)
|
|
3
2
|
|
|
4
3
|
Contentful -> local database synchronisation for Rails
|
|
5
4
|
|
|
@@ -90,7 +89,7 @@ To run a syncronization process run:
|
|
|
90
89
|
|
|
91
90
|
The first time you run this it will download all the contentful content, it will then check every 15 seconds for changes to the content and update/delete records in the database when changes are made in contentful.
|
|
92
91
|
|
|
93
|
-
If you want to
|
|
92
|
+
If you want to syncronise from scratch run:
|
|
94
93
|
|
|
95
94
|
$ rake consyncful:refresh
|
|
96
95
|
|
|
@@ -135,6 +134,24 @@ Because all contentful models are stored as polymorphic subtypes of Consyncful::
|
|
|
135
134
|
Consyncful::Base.where(title: 'a title') # [ #<ModelName>, #<OtherModelName> ]
|
|
136
135
|
```
|
|
137
136
|
|
|
137
|
+
### Sync callbacks
|
|
138
|
+
|
|
139
|
+
You may want to attach some application logic to happen before or after a sync run, for example to update caches or something.
|
|
140
|
+
|
|
141
|
+
Callbacks can be registered using:
|
|
142
|
+
|
|
143
|
+
```ruby
|
|
144
|
+
Consyncful::Sync.before_run do
|
|
145
|
+
# do something before the run
|
|
146
|
+
end
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
```ruby
|
|
150
|
+
Consyncful::Sync.after_run do |updated_ids|
|
|
151
|
+
# invalidate cache for updated_ids, or something
|
|
152
|
+
end
|
|
153
|
+
```
|
|
154
|
+
|
|
138
155
|
## Limitations
|
|
139
156
|
|
|
140
157
|
### Locales
|
data/Rakefile
CHANGED
data/bin/console
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
2
3
|
|
|
3
|
-
require
|
|
4
|
-
require
|
|
4
|
+
require 'bundler/setup'
|
|
5
|
+
require 'consyncful'
|
|
5
6
|
|
|
6
7
|
# You can add fixtures and/or initialization code here to make experimenting
|
|
7
8
|
# with your gem easier. You can also use a different console, if you like.
|
|
@@ -10,5 +11,5 @@ require "consyncful"
|
|
|
10
11
|
# require "pry"
|
|
11
12
|
# Pry.start
|
|
12
13
|
|
|
13
|
-
require
|
|
14
|
+
require 'irb'
|
|
14
15
|
IRB.start(__FILE__)
|
data/consyncful.gemspec
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
|
|
2
|
-
lib = File.expand_path(
|
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
|
-
require
|
|
5
|
+
require 'consyncful/version'
|
|
5
6
|
|
|
6
7
|
Gem::Specification.new do |spec|
|
|
7
|
-
spec.name =
|
|
8
|
+
spec.name = 'consyncful'
|
|
8
9
|
spec.version = Consyncful::VERSION
|
|
9
|
-
spec.authors = [
|
|
10
|
-
spec.email = [
|
|
10
|
+
spec.authors = ['Andy Anastasiadis-Gray', 'Montgomery Anderson']
|
|
11
|
+
spec.email = ['andy@boost.co.nz', 'montgomery@boost.co.nz']
|
|
11
12
|
|
|
12
|
-
spec.summary =
|
|
13
|
-
|
|
14
|
-
spec.license =
|
|
13
|
+
spec.summary = 'Contentful to local database synchronisation for Rails'
|
|
14
|
+
spec.homepage = 'https://github.com/boost/consyncful'
|
|
15
|
+
spec.license = 'MIT'
|
|
15
16
|
|
|
16
17
|
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
|
17
18
|
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
|
@@ -25,15 +26,18 @@ Gem::Specification.new do |spec|
|
|
|
25
26
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
|
26
27
|
f.match(%r{^(test|spec|features)/})
|
|
27
28
|
end
|
|
28
|
-
spec.bindir =
|
|
29
|
+
spec.bindir = 'exe'
|
|
29
30
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
30
|
-
spec.require_paths = [
|
|
31
|
+
spec.require_paths = ['lib']
|
|
31
32
|
|
|
32
|
-
spec.add_development_dependency
|
|
33
|
-
spec.add_development_dependency
|
|
34
|
-
spec.add_development_dependency
|
|
33
|
+
spec.add_development_dependency 'bundler', '~> 2'
|
|
34
|
+
spec.add_development_dependency 'database_cleaner'
|
|
35
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
|
36
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
|
37
|
+
spec.add_development_dependency 'rubocop', '0.79.0'
|
|
35
38
|
|
|
36
|
-
spec.add_dependency
|
|
37
|
-
spec.add_dependency
|
|
38
|
-
spec.add_dependency '
|
|
39
|
+
spec.add_dependency 'contentful', ['>=2.11.1', '<3.0.0']
|
|
40
|
+
spec.add_dependency 'hooks', '>=0.4.1'
|
|
41
|
+
spec.add_dependency 'mongoid', ['>=7.0.2', '<8.0.0']
|
|
42
|
+
spec.add_dependency 'rainbow'
|
|
39
43
|
end
|
data/docker-compose.yml
CHANGED
data/lib/consyncful.rb
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'consyncful/version'
|
|
2
4
|
|
|
3
5
|
require 'mongoid'
|
|
4
6
|
require 'contentful'
|
|
5
7
|
|
|
6
|
-
require
|
|
7
|
-
require
|
|
8
|
+
require 'consyncful/base'
|
|
9
|
+
require 'consyncful/sync'
|
|
8
10
|
|
|
9
|
-
require
|
|
11
|
+
require 'consyncful/railtie' if defined?(Rails)
|
|
10
12
|
|
|
11
13
|
module Consyncful
|
|
12
14
|
class << self
|
|
@@ -32,7 +34,7 @@ module Consyncful
|
|
|
32
34
|
DEFAULT_CLIENT_OPTIONS = {
|
|
33
35
|
reuse_entries: true,
|
|
34
36
|
api_url: 'cdn.contentful.com'
|
|
35
|
-
}
|
|
37
|
+
}.freeze
|
|
36
38
|
|
|
37
39
|
def self.client
|
|
38
40
|
@client ||= begin
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Consyncful
|
|
4
|
+
# Takes a mapped item from contentful and applies it to the local storage.
|
|
5
|
+
class PersistedItem
|
|
6
|
+
DEFAULT_LOCALE = 'en-NZ'
|
|
7
|
+
|
|
8
|
+
def initialize(item, sync_id, stats)
|
|
9
|
+
@item = item
|
|
10
|
+
@sync_id = sync_id
|
|
11
|
+
@stats = stats
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def persist
|
|
15
|
+
puts Rainbow("syncing: #{@item.id}").yellow
|
|
16
|
+
if @item.deletion?
|
|
17
|
+
delete_model(@item.id, @stats)
|
|
18
|
+
else
|
|
19
|
+
create_or_update_model(@item, @sync_id, @stats)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
def delete_model(id, stats)
|
|
26
|
+
Base.find_by(id: id).destroy
|
|
27
|
+
stats.record_deleted
|
|
28
|
+
rescue Mongoid::Errors::DocumentNotFound
|
|
29
|
+
puts Rainbow("Deleted record not found: #{id}").yellow
|
|
30
|
+
nil
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def create_or_update_model(item, sync_id, stats)
|
|
34
|
+
return if item.type.nil?
|
|
35
|
+
|
|
36
|
+
instance = find_or_initialize_item(item)
|
|
37
|
+
update_stats(instance, stats)
|
|
38
|
+
|
|
39
|
+
reset_fields(instance)
|
|
40
|
+
|
|
41
|
+
item.mapped_fields(DEFAULT_LOCALE).each do |field, value|
|
|
42
|
+
instance[field] = value
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
instance[:sync_id] = sync_id
|
|
46
|
+
|
|
47
|
+
instance.save
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def find_or_initialize_item(item)
|
|
51
|
+
model_class(item.type).find_or_initialize_by(id: item.id)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def update_stats(instance, stats)
|
|
55
|
+
if instance.persisted?
|
|
56
|
+
stats.record_updated
|
|
57
|
+
else
|
|
58
|
+
stats.record_added
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def model_class(type)
|
|
63
|
+
Base.model_map[type] || Base
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def reset_fields(instance)
|
|
67
|
+
instance.attributes.each do |field_name, _value|
|
|
68
|
+
next if field_name.in? %w[_id _type]
|
|
69
|
+
|
|
70
|
+
instance[field_name] = nil
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
data/lib/consyncful/stats.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'rainbow'
|
|
4
|
+
|
|
3
5
|
module Consyncful
|
|
4
6
|
class Stats
|
|
5
7
|
def initialize
|
|
@@ -23,9 +25,9 @@ module Consyncful
|
|
|
23
25
|
end
|
|
24
26
|
|
|
25
27
|
def print_stats
|
|
26
|
-
puts "Added: #{@stats[:records_added]}, \
|
|
28
|
+
puts Rainbow("Added: #{@stats[:records_added]}, \
|
|
27
29
|
updated: #{@stats[:records_updated]}, \
|
|
28
|
-
deleted: #{@stats[:records_deleted]}".blue
|
|
30
|
+
deleted: #{@stats[:records_deleted]}").blue
|
|
29
31
|
end
|
|
30
32
|
end
|
|
31
33
|
end
|
data/lib/consyncful/sync.rb
CHANGED
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require '
|
|
3
|
+
require 'rainbow'
|
|
4
4
|
require 'consyncful/item_mapper'
|
|
5
|
+
require 'consyncful/persisted_item'
|
|
5
6
|
require 'consyncful/stats'
|
|
6
|
-
|
|
7
|
-
class String
|
|
8
|
-
include Term::ANSIColor
|
|
9
|
-
end
|
|
7
|
+
require 'hooks'
|
|
10
8
|
|
|
11
9
|
module Consyncful
|
|
12
10
|
class Sync
|
|
13
11
|
include Mongoid::Document
|
|
12
|
+
include Hooks
|
|
13
|
+
|
|
14
|
+
define_hook :before_run
|
|
15
|
+
define_hook :after_run
|
|
14
16
|
|
|
15
17
|
DEFAULT_LOCALE = 'en-NZ'
|
|
16
18
|
|
|
@@ -21,109 +23,73 @@ module Consyncful
|
|
|
21
23
|
last || new
|
|
22
24
|
end
|
|
23
25
|
|
|
24
|
-
def self.
|
|
26
|
+
def self.fresh
|
|
25
27
|
destroy_all
|
|
26
|
-
|
|
28
|
+
latest
|
|
27
29
|
end
|
|
28
30
|
|
|
29
|
-
def
|
|
30
|
-
|
|
31
|
-
|
|
31
|
+
def drop_stale
|
|
32
|
+
stale = Base.where(:sync_id.ne => id, :sync_id.exists => true)
|
|
33
|
+
puts Rainbow("Dropping #{stale.count} records that haven't been touched in this sync").red
|
|
34
|
+
stale.destroy
|
|
32
35
|
end
|
|
33
36
|
|
|
34
37
|
def run
|
|
38
|
+
run_hook :before_run
|
|
39
|
+
|
|
35
40
|
stats = Consyncful::Stats.new
|
|
36
41
|
load_all_models
|
|
37
42
|
|
|
38
43
|
sync = start_sync
|
|
39
44
|
|
|
40
|
-
sync_items(sync, stats)
|
|
45
|
+
changed_ids = sync_items(sync, stats)
|
|
41
46
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
47
|
+
drop_stale
|
|
48
|
+
|
|
49
|
+
update_run(sync.next_sync_url)
|
|
45
50
|
stats.print_stats
|
|
51
|
+
|
|
52
|
+
run_hook :after_run, changed_ids
|
|
46
53
|
end
|
|
47
54
|
|
|
48
55
|
private
|
|
49
56
|
|
|
50
57
|
def load_all_models
|
|
51
58
|
return unless defined? Rails
|
|
59
|
+
|
|
52
60
|
Rails.application.eager_load!
|
|
53
61
|
end
|
|
54
62
|
|
|
63
|
+
def update_run(next_url)
|
|
64
|
+
self.next_url = next_url
|
|
65
|
+
self.last_run_at = Time.current
|
|
66
|
+
save
|
|
67
|
+
end
|
|
68
|
+
|
|
55
69
|
def start_sync
|
|
56
70
|
if next_url.present?
|
|
57
|
-
puts "Starting update, last update: #{last_run_at} (#{(Time.current - last_run_at).round(3)}s ago)".blue
|
|
71
|
+
puts Rainbow("Starting update, last update: #{last_run_at} (#{(Time.current - last_run_at).round(3)}s ago)").blue
|
|
58
72
|
Consyncful.client.sync(next_url)
|
|
59
73
|
else
|
|
60
|
-
puts 'Starting full refresh'.blue
|
|
74
|
+
puts Rainbow('Starting full refresh').blue
|
|
61
75
|
Consyncful.client.sync(initial: true)
|
|
62
76
|
end
|
|
63
77
|
end
|
|
64
78
|
|
|
65
79
|
def sync_items(sync, stats)
|
|
80
|
+
ids = []
|
|
66
81
|
sync.each_page do |page|
|
|
67
82
|
page.items.each do |item|
|
|
68
|
-
sync_item(ItemMapper.new(item), stats)
|
|
83
|
+
ids << sync_item(ItemMapper.new(item), stats)
|
|
69
84
|
end
|
|
70
85
|
end
|
|
86
|
+
ids
|
|
71
87
|
end
|
|
72
88
|
|
|
73
89
|
def sync_item(item, stats)
|
|
74
|
-
puts "syncing: #{item.id}".yellow
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
else
|
|
78
|
-
create_or_update_model(item, stats)
|
|
79
|
-
end
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
def delete_model(id, stats)
|
|
83
|
-
Base.find_by(id: id).destroy
|
|
84
|
-
stats.record_deleted
|
|
85
|
-
rescue Mongoid::Errors::DocumentNotFound
|
|
86
|
-
puts "Deleted record not found: #{id}".yellow
|
|
87
|
-
nil
|
|
88
|
-
end
|
|
89
|
-
|
|
90
|
-
def create_or_update_model(item, stats)
|
|
91
|
-
return if item.type.nil?
|
|
92
|
-
|
|
93
|
-
instance = find_or_initialize_item(item)
|
|
94
|
-
update_stats(instance, stats)
|
|
95
|
-
|
|
96
|
-
reset_fields(instance)
|
|
97
|
-
|
|
98
|
-
item.mapped_fields(DEFAULT_LOCALE).each do |field, value|
|
|
99
|
-
instance[field] = value
|
|
100
|
-
end
|
|
101
|
-
|
|
102
|
-
instance.save
|
|
103
|
-
end
|
|
104
|
-
|
|
105
|
-
def find_or_initialize_item(item)
|
|
106
|
-
model_class(item.type).find_or_initialize_by(id: item.id)
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
def update_stats(instance, stats)
|
|
110
|
-
if instance.persisted?
|
|
111
|
-
stats.record_updated
|
|
112
|
-
else
|
|
113
|
-
stats.record_added
|
|
114
|
-
end
|
|
115
|
-
end
|
|
116
|
-
|
|
117
|
-
def model_class(type)
|
|
118
|
-
Base.model_map[type] || Base
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
def reset_fields(instance)
|
|
122
|
-
instance.attributes.each do |field_name, _value|
|
|
123
|
-
next if field_name.in? %w[_id _type]
|
|
124
|
-
|
|
125
|
-
instance[field_name] = nil
|
|
126
|
-
end
|
|
90
|
+
puts Rainbow("syncing: #{item.id}").yellow
|
|
91
|
+
PersistedItem.new(item, id, stats).persist
|
|
92
|
+
item.id
|
|
127
93
|
end
|
|
128
94
|
end
|
|
129
95
|
end
|
|
@@ -9,7 +9,7 @@ namespace :consyncful do
|
|
|
9
9
|
Consyncful::Sync.fresh.run
|
|
10
10
|
end
|
|
11
11
|
|
|
12
|
-
task :sync, [:seconds] => [
|
|
12
|
+
task :sync, [:seconds] => %i[environment update_model_names] do |_task, args|
|
|
13
13
|
seconds = args[:seconds].to_i
|
|
14
14
|
seconds = 15 if seconds.zero?
|
|
15
15
|
loop do
|
|
@@ -19,12 +19,19 @@ namespace :consyncful do
|
|
|
19
19
|
end
|
|
20
20
|
|
|
21
21
|
task update_model_names: [:environment] do
|
|
22
|
-
Rails.application.
|
|
23
|
-
|
|
22
|
+
if Object.const_defined?('Zeitwerk::Loader') && Rails.application.config.autoloader.to_s == 'zeitwerk'
|
|
23
|
+
Zeitwerk::Loader.eager_load_all
|
|
24
|
+
else
|
|
25
|
+
Rails.application.eager_load!
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
puts Rainbow('Updating model names:').blue
|
|
29
|
+
|
|
24
30
|
Consyncful::Base.model_map.each do |contentful_name, constant|
|
|
25
|
-
puts "#{contentful_name}: #{constant}".yellow
|
|
31
|
+
puts Rainbow("#{contentful_name}: #{constant}").yellow
|
|
26
32
|
Consyncful::Base.where(contentful_type: contentful_name).update_all(_type: constant.to_s)
|
|
27
33
|
end
|
|
34
|
+
|
|
28
35
|
Consyncful::Base.where(:contentful_type.nin => Consyncful::Base.model_map.keys).update_all(_type: 'Consyncful::Base')
|
|
29
36
|
end
|
|
30
37
|
end
|
data/lib/consyncful/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: consyncful
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Andy Anastasiadis-Gray
|
|
8
8
|
- Montgomery Anderson
|
|
9
|
-
autorequire:
|
|
9
|
+
autorequire:
|
|
10
10
|
bindir: exe
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date:
|
|
12
|
+
date: 2021-02-03 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: bundler
|
|
@@ -17,28 +17,42 @@ dependencies:
|
|
|
17
17
|
requirements:
|
|
18
18
|
- - "~>"
|
|
19
19
|
- !ruby/object:Gem::Version
|
|
20
|
-
version: '
|
|
20
|
+
version: '2'
|
|
21
21
|
type: :development
|
|
22
22
|
prerelease: false
|
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
|
24
24
|
requirements:
|
|
25
25
|
- - "~>"
|
|
26
26
|
- !ruby/object:Gem::Version
|
|
27
|
-
version: '
|
|
27
|
+
version: '2'
|
|
28
|
+
- !ruby/object:Gem::Dependency
|
|
29
|
+
name: database_cleaner
|
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
|
31
|
+
requirements:
|
|
32
|
+
- - ">="
|
|
33
|
+
- !ruby/object:Gem::Version
|
|
34
|
+
version: '0'
|
|
35
|
+
type: :development
|
|
36
|
+
prerelease: false
|
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
38
|
+
requirements:
|
|
39
|
+
- - ">="
|
|
40
|
+
- !ruby/object:Gem::Version
|
|
41
|
+
version: '0'
|
|
28
42
|
- !ruby/object:Gem::Dependency
|
|
29
43
|
name: rake
|
|
30
44
|
requirement: !ruby/object:Gem::Requirement
|
|
31
45
|
requirements:
|
|
32
46
|
- - "~>"
|
|
33
47
|
- !ruby/object:Gem::Version
|
|
34
|
-
version: '
|
|
48
|
+
version: '13.0'
|
|
35
49
|
type: :development
|
|
36
50
|
prerelease: false
|
|
37
51
|
version_requirements: !ruby/object:Gem::Requirement
|
|
38
52
|
requirements:
|
|
39
53
|
- - "~>"
|
|
40
54
|
- !ruby/object:Gem::Version
|
|
41
|
-
version: '
|
|
55
|
+
version: '13.0'
|
|
42
56
|
- !ruby/object:Gem::Dependency
|
|
43
57
|
name: rspec
|
|
44
58
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -54,47 +68,75 @@ dependencies:
|
|
|
54
68
|
- !ruby/object:Gem::Version
|
|
55
69
|
version: '3.0'
|
|
56
70
|
- !ruby/object:Gem::Dependency
|
|
57
|
-
name:
|
|
71
|
+
name: rubocop
|
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
|
73
|
+
requirements:
|
|
74
|
+
- - '='
|
|
75
|
+
- !ruby/object:Gem::Version
|
|
76
|
+
version: 0.79.0
|
|
77
|
+
type: :development
|
|
78
|
+
prerelease: false
|
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
80
|
+
requirements:
|
|
81
|
+
- - '='
|
|
82
|
+
- !ruby/object:Gem::Version
|
|
83
|
+
version: 0.79.0
|
|
84
|
+
- !ruby/object:Gem::Dependency
|
|
85
|
+
name: contentful
|
|
58
86
|
requirement: !ruby/object:Gem::Requirement
|
|
59
87
|
requirements:
|
|
60
88
|
- - ">="
|
|
61
89
|
- !ruby/object:Gem::Version
|
|
62
|
-
version:
|
|
90
|
+
version: 2.11.1
|
|
63
91
|
- - "<"
|
|
64
92
|
- !ruby/object:Gem::Version
|
|
65
|
-
version:
|
|
93
|
+
version: 3.0.0
|
|
66
94
|
type: :runtime
|
|
67
95
|
prerelease: false
|
|
68
96
|
version_requirements: !ruby/object:Gem::Requirement
|
|
69
97
|
requirements:
|
|
70
98
|
- - ">="
|
|
71
99
|
- !ruby/object:Gem::Version
|
|
72
|
-
version:
|
|
100
|
+
version: 2.11.1
|
|
73
101
|
- - "<"
|
|
74
102
|
- !ruby/object:Gem::Version
|
|
75
|
-
version:
|
|
103
|
+
version: 3.0.0
|
|
76
104
|
- !ruby/object:Gem::Dependency
|
|
77
|
-
name:
|
|
105
|
+
name: hooks
|
|
78
106
|
requirement: !ruby/object:Gem::Requirement
|
|
79
107
|
requirements:
|
|
80
108
|
- - ">="
|
|
81
109
|
- !ruby/object:Gem::Version
|
|
82
|
-
version:
|
|
110
|
+
version: 0.4.1
|
|
111
|
+
type: :runtime
|
|
112
|
+
prerelease: false
|
|
113
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
114
|
+
requirements:
|
|
115
|
+
- - ">="
|
|
116
|
+
- !ruby/object:Gem::Version
|
|
117
|
+
version: 0.4.1
|
|
118
|
+
- !ruby/object:Gem::Dependency
|
|
119
|
+
name: mongoid
|
|
120
|
+
requirement: !ruby/object:Gem::Requirement
|
|
121
|
+
requirements:
|
|
122
|
+
- - ">="
|
|
123
|
+
- !ruby/object:Gem::Version
|
|
124
|
+
version: 7.0.2
|
|
83
125
|
- - "<"
|
|
84
126
|
- !ruby/object:Gem::Version
|
|
85
|
-
version:
|
|
127
|
+
version: 8.0.0
|
|
86
128
|
type: :runtime
|
|
87
129
|
prerelease: false
|
|
88
130
|
version_requirements: !ruby/object:Gem::Requirement
|
|
89
131
|
requirements:
|
|
90
132
|
- - ">="
|
|
91
133
|
- !ruby/object:Gem::Version
|
|
92
|
-
version:
|
|
134
|
+
version: 7.0.2
|
|
93
135
|
- - "<"
|
|
94
136
|
- !ruby/object:Gem::Version
|
|
95
|
-
version:
|
|
137
|
+
version: 8.0.0
|
|
96
138
|
- !ruby/object:Gem::Dependency
|
|
97
|
-
name:
|
|
139
|
+
name: rainbow
|
|
98
140
|
requirement: !ruby/object:Gem::Requirement
|
|
99
141
|
requirements:
|
|
100
142
|
- - ">="
|
|
@@ -107,7 +149,7 @@ dependencies:
|
|
|
107
149
|
- - ">="
|
|
108
150
|
- !ruby/object:Gem::Version
|
|
109
151
|
version: '0'
|
|
110
|
-
description:
|
|
152
|
+
description:
|
|
111
153
|
email:
|
|
112
154
|
- andy@boost.co.nz
|
|
113
155
|
- montgomery@boost.co.nz
|
|
@@ -117,6 +159,9 @@ extra_rdoc_files: []
|
|
|
117
159
|
files:
|
|
118
160
|
- ".gitignore"
|
|
119
161
|
- ".rspec"
|
|
162
|
+
- ".rubocop.yml"
|
|
163
|
+
- ".rubocop_todo.yml"
|
|
164
|
+
- ".ruby-version"
|
|
120
165
|
- ".travis.yml"
|
|
121
166
|
- Gemfile
|
|
122
167
|
- Gemfile.lock
|
|
@@ -131,16 +176,17 @@ files:
|
|
|
131
176
|
- lib/consyncful.rb
|
|
132
177
|
- lib/consyncful/base.rb
|
|
133
178
|
- lib/consyncful/item_mapper.rb
|
|
179
|
+
- lib/consyncful/persisted_item.rb
|
|
134
180
|
- lib/consyncful/railtie.rb
|
|
135
181
|
- lib/consyncful/stats.rb
|
|
136
182
|
- lib/consyncful/sync.rb
|
|
137
183
|
- lib/consyncful/tasks/consyncful.rake
|
|
138
184
|
- lib/consyncful/version.rb
|
|
139
|
-
homepage:
|
|
185
|
+
homepage: https://github.com/boost/consyncful
|
|
140
186
|
licenses:
|
|
141
187
|
- MIT
|
|
142
188
|
metadata: {}
|
|
143
|
-
post_install_message:
|
|
189
|
+
post_install_message:
|
|
144
190
|
rdoc_options: []
|
|
145
191
|
require_paths:
|
|
146
192
|
- lib
|
|
@@ -155,9 +201,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
155
201
|
- !ruby/object:Gem::Version
|
|
156
202
|
version: '0'
|
|
157
203
|
requirements: []
|
|
158
|
-
rubyforge_project:
|
|
159
|
-
rubygems_version: 2.7.6
|
|
160
|
-
signing_key:
|
|
204
|
+
rubyforge_project:
|
|
205
|
+
rubygems_version: 2.7.6.2
|
|
206
|
+
signing_key:
|
|
161
207
|
specification_version: 4
|
|
162
208
|
summary: Contentful to local database synchronisation for Rails
|
|
163
209
|
test_files: []
|