consyncful 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +13 -0
- data/.rspec +3 -0
- data/.travis.yml +8 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +82 -0
- data/LICENSE.txt +21 -0
- data/README.md +156 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/consyncful.gemspec +39 -0
- data/dev_data/.keep +0 -0
- data/docker-compose.yml +8 -0
- data/lib/consyncful/base.rb +32 -0
- data/lib/consyncful/item_mapper.rb +80 -0
- data/lib/consyncful/stats.rb +31 -0
- data/lib/consyncful/sync.rb +120 -0
- data/lib/consyncful/version.rb +3 -0
- data/lib/consyncful.rb +43 -0
- metadata +159 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: c489939e52bc185ec4c807d7f0d30b0347ec78d6acdc22c5469da2c78a749405
|
4
|
+
data.tar.gz: 34b24e8195a6f3be6d3a0753511959fefb815e21e87f2b1cc8ad67317d4c90ed
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ab76e80b28c06aa855cf375868200226e48205a0093b70400e94ff1dcf5499bf47fb6970f5477193038028254901637c8a73759a04a15eeeb667beb054c2f5e8
|
7
|
+
data.tar.gz: e3f9401264b593c4b11d1cd1fe1cbef34ad2c44ae79b727b027d84bf4980eb284e85bec8b0e3d67642f2b81cf076bb75cb9aa055b16670b6deb9b9d024c3b2b1
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
consyncful (0.1.0)
|
5
|
+
contentful (>= 2.11.1, < 3.0.0)
|
6
|
+
mongoid (>= 7.0.2, < 8.0.0)
|
7
|
+
term-ansicolor
|
8
|
+
|
9
|
+
GEM
|
10
|
+
remote: https://rubygems.org/
|
11
|
+
specs:
|
12
|
+
activemodel (5.2.2)
|
13
|
+
activesupport (= 5.2.2)
|
14
|
+
activesupport (5.2.2)
|
15
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
16
|
+
i18n (>= 0.7, < 2)
|
17
|
+
minitest (~> 5.1)
|
18
|
+
tzinfo (~> 1.1)
|
19
|
+
addressable (2.6.0)
|
20
|
+
public_suffix (>= 2.0.2, < 4.0)
|
21
|
+
bson (4.4.2)
|
22
|
+
concurrent-ruby (1.1.4)
|
23
|
+
contentful (2.11.1)
|
24
|
+
http (> 0.8, < 4.0)
|
25
|
+
multi_json (~> 1)
|
26
|
+
diff-lcs (1.3)
|
27
|
+
domain_name (0.5.20180417)
|
28
|
+
unf (>= 0.0.5, < 1.0.0)
|
29
|
+
http (3.3.0)
|
30
|
+
addressable (~> 2.3)
|
31
|
+
http-cookie (~> 1.0)
|
32
|
+
http-form_data (~> 2.0)
|
33
|
+
http_parser.rb (~> 0.6.0)
|
34
|
+
http-cookie (1.0.3)
|
35
|
+
domain_name (~> 0.5)
|
36
|
+
http-form_data (2.1.1)
|
37
|
+
http_parser.rb (0.6.0)
|
38
|
+
i18n (1.5.3)
|
39
|
+
concurrent-ruby (~> 1.0)
|
40
|
+
minitest (5.11.3)
|
41
|
+
mongo (2.7.0)
|
42
|
+
bson (>= 4.4.2, < 5.0.0)
|
43
|
+
mongoid (7.0.2)
|
44
|
+
activemodel (>= 5.1, < 6.0.0)
|
45
|
+
mongo (>= 2.5.1, < 3.0.0)
|
46
|
+
multi_json (1.13.1)
|
47
|
+
public_suffix (3.0.3)
|
48
|
+
rake (10.5.0)
|
49
|
+
rspec (3.8.0)
|
50
|
+
rspec-core (~> 3.8.0)
|
51
|
+
rspec-expectations (~> 3.8.0)
|
52
|
+
rspec-mocks (~> 3.8.0)
|
53
|
+
rspec-core (3.8.0)
|
54
|
+
rspec-support (~> 3.8.0)
|
55
|
+
rspec-expectations (3.8.2)
|
56
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
57
|
+
rspec-support (~> 3.8.0)
|
58
|
+
rspec-mocks (3.8.0)
|
59
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
60
|
+
rspec-support (~> 3.8.0)
|
61
|
+
rspec-support (3.8.0)
|
62
|
+
term-ansicolor (1.7.1)
|
63
|
+
tins (~> 1.0)
|
64
|
+
thread_safe (0.3.6)
|
65
|
+
tins (1.20.2)
|
66
|
+
tzinfo (1.2.5)
|
67
|
+
thread_safe (~> 0.1)
|
68
|
+
unf (0.1.4)
|
69
|
+
unf_ext
|
70
|
+
unf_ext (0.0.7.5)
|
71
|
+
|
72
|
+
PLATFORMS
|
73
|
+
ruby
|
74
|
+
|
75
|
+
DEPENDENCIES
|
76
|
+
bundler (~> 1.16)
|
77
|
+
consyncful!
|
78
|
+
rake (~> 10.0)
|
79
|
+
rspec (~> 3.0)
|
80
|
+
|
81
|
+
BUNDLED WITH
|
82
|
+
1.16.1
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2019 Andy Anastasiadis-Gray
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,156 @@
|
|
1
|
+
# Consyncful
|
2
|
+
[![Build Status](https://travis-ci.com/boost/consyncful.svg?branch=master)](https://travis-ci.com/boost/consyncful)
|
3
|
+
|
4
|
+
Contentful -> local database synchronisation for Rails
|
5
|
+
|
6
|
+
Requesting complicated models from the Contentful Delivery API in Rails applications is often
|
7
|
+
too slow, and makes testing applications painful. Consyncful uses Contentful's syncronisation API
|
8
|
+
to keep a local copy of the entire content in a Mongo database up to date.
|
9
|
+
|
10
|
+
Once the content is availble locally, finding and interact with contentful data is as easy as
|
11
|
+
using [Mongoid](https://docs.mongodb.com/mongoid/current/tutorials/mongoid-documents/) ODM.
|
12
|
+
|
13
|
+
This gem doesn't provide any intergration with the management api or any way to update contentful models from the local store. It is strictly read only.
|
14
|
+
|
15
|
+
## Why do I have to use MongoDB?
|
16
|
+
|
17
|
+
Consyncful currently only supports Mongoid ODM because models have dynamic schemas. And that's all we've had a chance to work out so far. :)
|
18
|
+
The same pattern might be able to be extended to work with ActiveRecord, but having to migrate the local database as well as your contentful content type's seems tedious.
|
19
|
+
|
20
|
+
## Installation
|
21
|
+
|
22
|
+
Add this line to your application's Gemfile:
|
23
|
+
|
24
|
+
```ruby
|
25
|
+
gem 'consyncful'
|
26
|
+
```
|
27
|
+
|
28
|
+
And then execute:
|
29
|
+
|
30
|
+
$ bundle
|
31
|
+
|
32
|
+
If you don't already use mongoid, generate a mongoid.yml by running:
|
33
|
+
|
34
|
+
$ rake g mongoid:config
|
35
|
+
|
36
|
+
Add an initializer:
|
37
|
+
Consyncful uses [contentful.rb](https://github.com/contentful/contentful.rb) so client options are as documented there.
|
38
|
+
```ruby
|
39
|
+
Consyncful.configure do |config|
|
40
|
+
config.locale = 'en-NZ'
|
41
|
+
config.contentful_client_options = {
|
42
|
+
api_url: 'cdn.contentful.com',
|
43
|
+
space: 'space_id',
|
44
|
+
access_token: 'ACCESS TOKEN',
|
45
|
+
environment: 'master', # optional
|
46
|
+
logger: Logger.new(STDOUT) # optional for debugging
|
47
|
+
}
|
48
|
+
end
|
49
|
+
```
|
50
|
+
|
51
|
+
## Usage
|
52
|
+
|
53
|
+
### Creating contentful models in your rails app
|
54
|
+
|
55
|
+
Create models by inheriting from `Consyncful::Base`
|
56
|
+
|
57
|
+
```ruby
|
58
|
+
class ModelName < Consyncful::Base
|
59
|
+
contentful_model_name 'contentfulTypeName'
|
60
|
+
end
|
61
|
+
```
|
62
|
+
|
63
|
+
Model fields will be dynamicly assigned, but mongoid dynamic fields are not accessible if the entry has an empty field. If you want the accessor methods to be reliably available for fields it is recommended to define the fields in the model:
|
64
|
+
|
65
|
+
```ruby
|
66
|
+
class ModelName < Consyncful::Base
|
67
|
+
contentful_model_name 'contentfulTypeName'
|
68
|
+
|
69
|
+
field :title
|
70
|
+
field :is_awesome, type: Boolean
|
71
|
+
end
|
72
|
+
```
|
73
|
+
|
74
|
+
Contentful reference fields are a bit special compared with standard mongoid associations, Consyncful provides the following helpers to set up the correct relationships:
|
75
|
+
|
76
|
+
```ruby
|
77
|
+
class ModelWithReferences < Consyncful::Base
|
78
|
+
contentful_model_name 'contentfulTypeName'
|
79
|
+
|
80
|
+
references_one :thing
|
81
|
+
references_many :other_things
|
82
|
+
end
|
83
|
+
```
|
84
|
+
|
85
|
+
### Syncronizing contentful data
|
86
|
+
|
87
|
+
To run a syncronization process run:
|
88
|
+
|
89
|
+
$ rake consyncful:sync
|
90
|
+
|
91
|
+
The first time you run this it will download all the contentful content, it will then check every 15 seconds for changes to the content and update/delete records in the database when changes are made in contentful.
|
92
|
+
|
93
|
+
If you want to delete everything and start syncronising from scratch run:
|
94
|
+
|
95
|
+
$ rake consyncful:refresh
|
96
|
+
|
97
|
+
It is recommended to refresh your data if you change model names.
|
98
|
+
|
99
|
+
Now you've synced your data, it is all available via your rails models
|
100
|
+
|
101
|
+
### Finding and interacting with models
|
102
|
+
|
103
|
+
#### Querying
|
104
|
+
Models are available using standard mongoid [queries](https://docs.mongodb.com/mongoid/current/tutorials/mongoid-queries/).
|
105
|
+
|
106
|
+
```ruby
|
107
|
+
instance = ModelName.find_by(instance: 'foo')
|
108
|
+
|
109
|
+
instance.is_awesome # true
|
110
|
+
```
|
111
|
+
|
112
|
+
#### References
|
113
|
+
References work like you woule expect:
|
114
|
+
|
115
|
+
```ruby
|
116
|
+
|
117
|
+
instance = ModelWithReferences.find('contentfulID')
|
118
|
+
|
119
|
+
instance.thing # returns the referenced thing
|
120
|
+
instance.other_things # all the referenced things, polymorphic, so might be different types
|
121
|
+
```
|
122
|
+
|
123
|
+
**Except**:
|
124
|
+
`references_many` associations return objects in a different order from how they are ordered in contentful. If you want them in the order they appare in contentful, use the `.in_order` helper:
|
125
|
+
|
126
|
+
```ruby
|
127
|
+
instance.other_things.in_order # ordered the same as in contentful
|
128
|
+
```
|
129
|
+
|
130
|
+
#### Finding entrys from different content types
|
131
|
+
|
132
|
+
Because all contentful models are stored as polymorphic subtypes of Consyncful::Base, you can query all entries without knowing what type you are looking for:
|
133
|
+
|
134
|
+
```ruby
|
135
|
+
Consyncful::Base.where(title: 'a title') # [ #<ModelName>, #<OtherModelName> ]
|
136
|
+
```
|
137
|
+
|
138
|
+
## Limitations
|
139
|
+
|
140
|
+
### Locales
|
141
|
+
|
142
|
+
Current Consyncful only uses one globally configured locale to map the data to the database.
|
143
|
+
|
144
|
+
## Development
|
145
|
+
|
146
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
147
|
+
|
148
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
149
|
+
|
150
|
+
## Contributing
|
151
|
+
|
152
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/boost/consyncful.
|
153
|
+
|
154
|
+
## License
|
155
|
+
|
156
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "consyncful"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/consyncful.gemspec
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require "consyncful/version"
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "consyncful"
|
8
|
+
spec.version = Consyncful::VERSION
|
9
|
+
spec.authors = ["Andy Anastasiadis-Gray"]
|
10
|
+
spec.email = ["andy@boost.co.nz"]
|
11
|
+
|
12
|
+
spec.summary = %q{Contentful to local database synchronisation for Rails}
|
13
|
+
# spec.homepage = "TODO: Put your gem's website or public repo URL here."
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
17
|
+
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
18
|
+
# if spec.respond_to?(:metadata)
|
19
|
+
# spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
|
20
|
+
# else
|
21
|
+
# raise "RubyGems 2.0 or newer is required to protect against " \
|
22
|
+
# "public gem pushes."
|
23
|
+
# end
|
24
|
+
|
25
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
26
|
+
f.match(%r{^(test|spec|features)/})
|
27
|
+
end
|
28
|
+
spec.bindir = "exe"
|
29
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
30
|
+
spec.require_paths = ["lib"]
|
31
|
+
|
32
|
+
spec.add_development_dependency "bundler", "~> 1.16"
|
33
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
34
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
35
|
+
|
36
|
+
spec.add_dependency "mongoid", [">=7.0.2", "<8.0.0"]
|
37
|
+
spec.add_dependency "contentful", [">=2.11.1", "<3.0.0"]
|
38
|
+
spec.add_dependency 'term-ansicolor'
|
39
|
+
end
|
data/dev_data/.keep
ADDED
File without changes
|
data/docker-compose.yml
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Consyncful
|
4
|
+
class Base
|
5
|
+
include Mongoid::Document
|
6
|
+
include Mongoid::Attributes::Dynamic
|
7
|
+
|
8
|
+
store_in collection: 'contentful_models'
|
9
|
+
|
10
|
+
cattr_accessor :model_map
|
11
|
+
|
12
|
+
def self.contentful_model_name(name)
|
13
|
+
self.model_map ||= {}
|
14
|
+
|
15
|
+
self.model_map[name] = self
|
16
|
+
end
|
17
|
+
|
18
|
+
# rubocop:disable Lint/NestedMethodDefinition
|
19
|
+
def self.references_many(name)
|
20
|
+
has_and_belongs_to_many name.to_sym, class_name: 'Consyncful::Base', inverse_of: nil do
|
21
|
+
def in_order
|
22
|
+
_target.to_a.sort_by { |a| _base[foreign_key].index(a.id) }
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
# rubocop:enable Lint/NestedMethodDefinition
|
27
|
+
|
28
|
+
def self.references_one(name)
|
29
|
+
belongs_to name.to_sym, optional: true, class_name: 'Consyncful::Base'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Consyncful
|
4
|
+
class ItemMapper
|
5
|
+
def initialize(item)
|
6
|
+
@item = item
|
7
|
+
end
|
8
|
+
|
9
|
+
def deletion?
|
10
|
+
@item.is_a?(Contentful::DeletedEntry) || @item.is_a?(Contentful::DeletedAsset)
|
11
|
+
end
|
12
|
+
|
13
|
+
def type
|
14
|
+
if @item.type == 'Entry'
|
15
|
+
@item.content_type.id
|
16
|
+
elsif @item.type == 'Asset'
|
17
|
+
'asset'
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def id
|
22
|
+
@item.id
|
23
|
+
end
|
24
|
+
|
25
|
+
def mapped_fields(locale)
|
26
|
+
fields = generic_fields
|
27
|
+
|
28
|
+
@item.fields_with_locales.each do |field, value_with_locales|
|
29
|
+
value = value_with_locales[locale.to_sym]
|
30
|
+
next if value.is_a? Contentful::File # it is special
|
31
|
+
|
32
|
+
assign_field(fields, field, value)
|
33
|
+
end
|
34
|
+
|
35
|
+
fields[:file] = raw_file(locale) if type == 'asset'
|
36
|
+
|
37
|
+
fields
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def generic_fields
|
43
|
+
fields = {}
|
44
|
+
fields[:created_at] = @item.created_at
|
45
|
+
fields[:updated_at] = @item.updated_at
|
46
|
+
fields[:revision] = @item.revision
|
47
|
+
fields[:contentful_type] = type
|
48
|
+
fields[:synced_at] = Time.current
|
49
|
+
fields
|
50
|
+
end
|
51
|
+
|
52
|
+
def raw_file(locale)
|
53
|
+
file_json = @item.raw.fetch('fields', {}).fetch('file', nil)
|
54
|
+
file_json[locale]
|
55
|
+
end
|
56
|
+
|
57
|
+
def reference_value?(value)
|
58
|
+
single_reference?(value) || many_reference?(value)
|
59
|
+
end
|
60
|
+
|
61
|
+
def single_reference?(value)
|
62
|
+
value.is_a?(Contentful::BaseResource)
|
63
|
+
end
|
64
|
+
|
65
|
+
def many_reference?(value)
|
66
|
+
value.is_a?(Array) && single_reference?(value.first)
|
67
|
+
end
|
68
|
+
|
69
|
+
def assign_field(hash, field, value)
|
70
|
+
if single_reference?(value)
|
71
|
+
hash[ActiveSupport::Inflector.foreign_key(field).to_sym] = value.id
|
72
|
+
elsif many_reference?(value)
|
73
|
+
ids_field_name = field.to_s.singularize + '_ids' # fk field name
|
74
|
+
hash[ids_field_name.to_sym] = value.map(&:id)
|
75
|
+
else
|
76
|
+
hash[field] = value
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Consyncful
|
4
|
+
class Stats
|
5
|
+
def initialize
|
6
|
+
@stats = {
|
7
|
+
records_added: 0,
|
8
|
+
records_updated: 0,
|
9
|
+
records_deleted: 0
|
10
|
+
}
|
11
|
+
end
|
12
|
+
|
13
|
+
def record_added
|
14
|
+
@stats[:records_added] += 1
|
15
|
+
end
|
16
|
+
|
17
|
+
def record_updated
|
18
|
+
@stats[:records_updated] += 1
|
19
|
+
end
|
20
|
+
|
21
|
+
def record_deleted
|
22
|
+
@stats[:records_deleted] += 1
|
23
|
+
end
|
24
|
+
|
25
|
+
def print_stats
|
26
|
+
puts "Added: #{@stats[:records_added]}, \
|
27
|
+
updated: #{@stats[:records_updated]}, \
|
28
|
+
deleted: #{@stats[:records_deleted]}".blue
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'term/ansicolor'
|
4
|
+
require 'consyncful/item_mapper'
|
5
|
+
require 'consyncful/stats'
|
6
|
+
|
7
|
+
class String
|
8
|
+
include Term::ANSIColor
|
9
|
+
end
|
10
|
+
|
11
|
+
module Consyncful
|
12
|
+
class Sync
|
13
|
+
include Mongoid::Document
|
14
|
+
# include ActionView::Helpers::DateHelper
|
15
|
+
|
16
|
+
DEFAULT_LOCALE = 'en-NZ'
|
17
|
+
|
18
|
+
field :next_url
|
19
|
+
field :last_run_at, type: DateTime
|
20
|
+
|
21
|
+
def self.latest
|
22
|
+
last || new
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.reset
|
26
|
+
destroy_all
|
27
|
+
Base.destroy_all
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.fresh
|
31
|
+
reset
|
32
|
+
latest
|
33
|
+
end
|
34
|
+
|
35
|
+
def run
|
36
|
+
stats = Consyncful::Stats.new
|
37
|
+
load_all_models
|
38
|
+
|
39
|
+
sync = start_sync
|
40
|
+
|
41
|
+
sync_items(sync, stats)
|
42
|
+
|
43
|
+
self.next_url = sync.next_sync_url
|
44
|
+
self.last_run_at = Time.current
|
45
|
+
save
|
46
|
+
stats.print_stats
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
def load_all_models
|
52
|
+
return unless defined? Rails
|
53
|
+
Rails.application.eager_load!
|
54
|
+
end
|
55
|
+
|
56
|
+
def start_sync
|
57
|
+
if next_url.present?
|
58
|
+
puts "Starting update, last update: #{last_run_at} (#{(Time.current - last_run_at).round(3)}s ago)".blue
|
59
|
+
Consyncful.client.sync(next_url)
|
60
|
+
else
|
61
|
+
puts 'Starting full refresh'.blue
|
62
|
+
Consyncful.client.sync(initial: true)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def sync_items(sync, stats)
|
67
|
+
sync.each_page do |page|
|
68
|
+
page.items.each do |item|
|
69
|
+
sync_item(ItemMapper.new(item), stats)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def sync_item(item, stats)
|
75
|
+
puts "syncing: #{item.id}".yellow
|
76
|
+
if item.deletion?
|
77
|
+
delete_model(item.id, stats)
|
78
|
+
else
|
79
|
+
create_or_update_model(item, stats)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def delete_model(id, stats)
|
84
|
+
Base.find_by(id: id).destroy
|
85
|
+
stats.record_deleted
|
86
|
+
rescue Mongoid::Errors::DocumentNotFound
|
87
|
+
puts "Deleted record not found: #{id}".yellow
|
88
|
+
nil
|
89
|
+
end
|
90
|
+
|
91
|
+
def create_or_update_model(item, stats)
|
92
|
+
return if item.type.nil?
|
93
|
+
|
94
|
+
instance = find_or_initialize_item(item)
|
95
|
+
update_stats(instance, stats)
|
96
|
+
|
97
|
+
item.mapped_fields(DEFAULT_LOCALE).each do |field, value|
|
98
|
+
instance[field] = value
|
99
|
+
end
|
100
|
+
|
101
|
+
instance.save
|
102
|
+
end
|
103
|
+
|
104
|
+
def find_or_initialize_item(item)
|
105
|
+
model_class(item.type).find_or_initialize_by(id: item.id)
|
106
|
+
end
|
107
|
+
|
108
|
+
def update_stats(instance, stats)
|
109
|
+
if instance.persisted?
|
110
|
+
stats.record_updated
|
111
|
+
else
|
112
|
+
stats.record_added
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def model_class(type)
|
117
|
+
Base.model_map[type] || Base
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
data/lib/consyncful.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
require "consyncful/version"
|
2
|
+
|
3
|
+
require 'mongoid'
|
4
|
+
require 'contentful'
|
5
|
+
|
6
|
+
require "consyncful/base"
|
7
|
+
require "consyncful/sync"
|
8
|
+
|
9
|
+
module Consyncful
|
10
|
+
class << self
|
11
|
+
attr_accessor :configuration
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.configure
|
15
|
+
self.configuration ||= Configuration.new
|
16
|
+
yield(configuration)
|
17
|
+
end
|
18
|
+
|
19
|
+
class Configuration
|
20
|
+
attr_accessor :contentful_client_options, :locale
|
21
|
+
|
22
|
+
def initialize
|
23
|
+
@contentful_client_options = {
|
24
|
+
api_url: 'cdn.contentful.com'
|
25
|
+
}
|
26
|
+
@locale = 'en-US'
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
DEFAULT_CLIENT_OPTIONS = {
|
31
|
+
reuse_entries: true,
|
32
|
+
api_url: 'cdn.contentful.com'
|
33
|
+
}
|
34
|
+
|
35
|
+
def self.client
|
36
|
+
@client ||= begin
|
37
|
+
options = Consyncful.configuration.contentful_client_options
|
38
|
+
options.reverse_merge!(DEFAULT_CLIENT_OPTIONS)
|
39
|
+
|
40
|
+
Contentful::Client.new(options)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
metadata
ADDED
@@ -0,0 +1,159 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: consyncful
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Andy Anastasiadis-Gray
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2019-02-22 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.16'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.16'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: mongoid
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 7.0.2
|
62
|
+
- - "<"
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: 8.0.0
|
65
|
+
type: :runtime
|
66
|
+
prerelease: false
|
67
|
+
version_requirements: !ruby/object:Gem::Requirement
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
version: 7.0.2
|
72
|
+
- - "<"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: 8.0.0
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: contentful
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: 2.11.1
|
82
|
+
- - "<"
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: 3.0.0
|
85
|
+
type: :runtime
|
86
|
+
prerelease: false
|
87
|
+
version_requirements: !ruby/object:Gem::Requirement
|
88
|
+
requirements:
|
89
|
+
- - ">="
|
90
|
+
- !ruby/object:Gem::Version
|
91
|
+
version: 2.11.1
|
92
|
+
- - "<"
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: 3.0.0
|
95
|
+
- !ruby/object:Gem::Dependency
|
96
|
+
name: term-ansicolor
|
97
|
+
requirement: !ruby/object:Gem::Requirement
|
98
|
+
requirements:
|
99
|
+
- - ">="
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
102
|
+
type: :runtime
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - ">="
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '0'
|
109
|
+
description:
|
110
|
+
email:
|
111
|
+
- andy@boost.co.nz
|
112
|
+
executables: []
|
113
|
+
extensions: []
|
114
|
+
extra_rdoc_files: []
|
115
|
+
files:
|
116
|
+
- ".gitignore"
|
117
|
+
- ".rspec"
|
118
|
+
- ".travis.yml"
|
119
|
+
- Gemfile
|
120
|
+
- Gemfile.lock
|
121
|
+
- LICENSE.txt
|
122
|
+
- README.md
|
123
|
+
- Rakefile
|
124
|
+
- bin/console
|
125
|
+
- bin/setup
|
126
|
+
- consyncful.gemspec
|
127
|
+
- dev_data/.keep
|
128
|
+
- docker-compose.yml
|
129
|
+
- lib/consyncful.rb
|
130
|
+
- lib/consyncful/base.rb
|
131
|
+
- lib/consyncful/item_mapper.rb
|
132
|
+
- lib/consyncful/stats.rb
|
133
|
+
- lib/consyncful/sync.rb
|
134
|
+
- lib/consyncful/version.rb
|
135
|
+
homepage:
|
136
|
+
licenses:
|
137
|
+
- MIT
|
138
|
+
metadata: {}
|
139
|
+
post_install_message:
|
140
|
+
rdoc_options: []
|
141
|
+
require_paths:
|
142
|
+
- lib
|
143
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
144
|
+
requirements:
|
145
|
+
- - ">="
|
146
|
+
- !ruby/object:Gem::Version
|
147
|
+
version: '0'
|
148
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0'
|
153
|
+
requirements: []
|
154
|
+
rubyforge_project:
|
155
|
+
rubygems_version: 2.7.6
|
156
|
+
signing_key:
|
157
|
+
specification_version: 4
|
158
|
+
summary: Contentful to local database synchronisation for Rails
|
159
|
+
test_files: []
|