consyncful 1.0.2 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +1 -0
- data/.gitignore +2 -0
- data/Gemfile +10 -7
- data/Gemfile.lock +105 -3
- data/README.md +168 -66
- data/app/controllers/consyncful/webhook_controller.rb +51 -0
- data/config/routes.rb +5 -0
- data/lib/consyncful/configuration.rb +24 -2
- data/lib/consyncful/engine.rb +12 -0
- data/lib/consyncful/item_mapper.rb +7 -3
- data/lib/consyncful/sync.rb +18 -0
- data/lib/consyncful/sync_runner.rb +43 -0
- data/lib/consyncful/tasks/consyncful.rake +10 -6
- data/lib/consyncful/version.rb +1 -1
- data/lib/consyncful.rb +10 -2
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8c8c5bb5e0b223ac438fbb3c5be517570fcdcfc7950f573ac2a4f30240275863
|
4
|
+
data.tar.gz: 65f81747941edf52d0477bcd03cc52d4114aebf52c1090a4cb257e21f3df01f6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b54be82b13267054ecd039d91f3eb4ee5984fdbb07be161c2a113e532944bbdc8de2679e7d0729fcf842421bbdd67807229a47e00c76a25bf58b6688920e7555
|
7
|
+
data.tar.gz: 854fdb89c3456c32235ec8b7bb9ce509c87a875c58866f183a25992ad00327798d4fcbd21b657d929f9f9857e8aad513e6f515882b8e27d51640016abe43c2dc
|
data/.github/workflows/ci.yml
CHANGED
data/.gitignore
CHANGED
data/Gemfile
CHANGED
@@ -3,20 +3,23 @@
|
|
3
3
|
source 'https://rubygems.org'
|
4
4
|
|
5
5
|
git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
|
6
|
-
|
7
|
-
# gem 'activemodel', '~> 7'
|
8
|
-
# gem 'activesupport', '~> 7'
|
9
|
-
# gem 'mongoid', '~> 8'
|
10
|
-
|
11
6
|
# Specify your gem's dependencies in consyncful.gemspec
|
12
7
|
gemspec
|
13
8
|
|
14
9
|
group :development do
|
15
10
|
gem 'bundler', '~> 2'
|
16
|
-
gem 'database_cleaner-mongoid'
|
17
11
|
gem 'rake', '~> 13.0'
|
18
|
-
gem 'rspec', '~> 3.0'
|
19
12
|
gem 'rubocop'
|
20
13
|
gem 'rubocop-rake'
|
21
14
|
gem 'rubocop-rspec'
|
22
15
|
end
|
16
|
+
|
17
|
+
group :development, :test do
|
18
|
+
gem 'combustion', '~> 1.3'
|
19
|
+
gem 'rspec', '~> 3.13'
|
20
|
+
gem 'rspec-rails', '~> 6.1'
|
21
|
+
end
|
22
|
+
|
23
|
+
group :test do
|
24
|
+
gem 'database_cleaner-mongoid', '~> 2.0'
|
25
|
+
end
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
consyncful (1.
|
4
|
+
consyncful (1.1.1)
|
5
5
|
contentful (>= 2.11.1, < 3.0.0)
|
6
6
|
hooks (>= 0.4.1)
|
7
7
|
mongoid (>= 7.0.2)
|
@@ -10,6 +10,23 @@ PATH
|
|
10
10
|
GEM
|
11
11
|
remote: https://rubygems.org/
|
12
12
|
specs:
|
13
|
+
actionpack (7.2.2.1)
|
14
|
+
actionview (= 7.2.2.1)
|
15
|
+
activesupport (= 7.2.2.1)
|
16
|
+
nokogiri (>= 1.8.5)
|
17
|
+
racc
|
18
|
+
rack (>= 2.2.4, < 3.2)
|
19
|
+
rack-session (>= 1.0.1)
|
20
|
+
rack-test (>= 0.6.3)
|
21
|
+
rails-dom-testing (~> 2.2)
|
22
|
+
rails-html-sanitizer (~> 1.6)
|
23
|
+
useragent (~> 0.16)
|
24
|
+
actionview (7.2.2.1)
|
25
|
+
activesupport (= 7.2.2.1)
|
26
|
+
builder (~> 3.1)
|
27
|
+
erubi (~> 1.11)
|
28
|
+
rails-dom-testing (~> 2.2)
|
29
|
+
rails-html-sanitizer (~> 1.6)
|
13
30
|
activemodel (7.2.2.1)
|
14
31
|
activesupport (= 7.2.2.1)
|
15
32
|
activesupport (7.2.2.1)
|
@@ -31,18 +48,27 @@ GEM
|
|
31
48
|
benchmark (0.4.0)
|
32
49
|
bigdecimal (3.1.9)
|
33
50
|
bson (5.0.2)
|
51
|
+
builder (3.3.0)
|
52
|
+
combustion (1.5.0)
|
53
|
+
activesupport (>= 3.0.0)
|
54
|
+
railties (>= 3.0.0)
|
55
|
+
thor (>= 0.14.6)
|
34
56
|
concurrent-ruby (1.3.5)
|
35
57
|
connection_pool (2.5.3)
|
36
58
|
contentful (2.17.1)
|
37
59
|
http (> 0.8, < 6.0)
|
38
60
|
multi_json (~> 1)
|
61
|
+
crass (1.0.6)
|
39
62
|
database_cleaner-core (2.0.1)
|
40
63
|
database_cleaner-mongoid (2.0.1)
|
41
64
|
database_cleaner-core (~> 2.0.0)
|
42
65
|
mongoid
|
66
|
+
date (3.4.1)
|
43
67
|
diff-lcs (1.6.2)
|
44
68
|
domain_name (0.6.20240107)
|
45
69
|
drb (2.2.3)
|
70
|
+
erb (5.0.2)
|
71
|
+
erubi (1.13.1)
|
46
72
|
ffi (1.17.2)
|
47
73
|
ffi (1.17.2-aarch64-linux-gnu)
|
48
74
|
ffi (1.17.2-aarch64-linux-musl)
|
@@ -70,6 +96,11 @@ GEM
|
|
70
96
|
http-form_data (2.3.0)
|
71
97
|
i18n (1.14.7)
|
72
98
|
concurrent-ruby (~> 1.0)
|
99
|
+
io-console (0.8.1)
|
100
|
+
irb (1.15.2)
|
101
|
+
pp (>= 0.6.0)
|
102
|
+
rdoc (>= 4.0.0)
|
103
|
+
reline (>= 0.4.2)
|
73
104
|
json (2.12.2)
|
74
105
|
language_server-protocol (3.17.0.5)
|
75
106
|
lint_roller (1.1.0)
|
@@ -77,6 +108,10 @@ GEM
|
|
77
108
|
ffi-compiler (~> 1.0)
|
78
109
|
rake (~> 13.0)
|
79
110
|
logger (1.7.0)
|
111
|
+
loofah (2.24.1)
|
112
|
+
crass (~> 1.0.2)
|
113
|
+
nokogiri (>= 1.12.0)
|
114
|
+
mini_portile2 (2.8.9)
|
80
115
|
minitest (5.25.5)
|
81
116
|
mongo (2.21.1)
|
82
117
|
base64
|
@@ -87,16 +122,69 @@ GEM
|
|
87
122
|
mongo (>= 2.18.0, < 3.0.0)
|
88
123
|
ruby2_keywords (~> 0.0.5)
|
89
124
|
multi_json (1.15.0)
|
125
|
+
nokogiri (1.18.9)
|
126
|
+
mini_portile2 (~> 2.8.2)
|
127
|
+
racc (~> 1.4)
|
128
|
+
nokogiri (1.18.9-aarch64-linux-gnu)
|
129
|
+
racc (~> 1.4)
|
130
|
+
nokogiri (1.18.9-aarch64-linux-musl)
|
131
|
+
racc (~> 1.4)
|
132
|
+
nokogiri (1.18.9-arm-linux-gnu)
|
133
|
+
racc (~> 1.4)
|
134
|
+
nokogiri (1.18.9-arm-linux-musl)
|
135
|
+
racc (~> 1.4)
|
136
|
+
nokogiri (1.18.9-arm64-darwin)
|
137
|
+
racc (~> 1.4)
|
138
|
+
nokogiri (1.18.9-x86_64-darwin)
|
139
|
+
racc (~> 1.4)
|
140
|
+
nokogiri (1.18.9-x86_64-linux-gnu)
|
141
|
+
racc (~> 1.4)
|
142
|
+
nokogiri (1.18.9-x86_64-linux-musl)
|
143
|
+
racc (~> 1.4)
|
90
144
|
parallel (1.27.0)
|
91
145
|
parser (3.3.8.0)
|
92
146
|
ast (~> 2.4.1)
|
93
147
|
racc
|
148
|
+
pp (0.6.2)
|
149
|
+
prettyprint
|
150
|
+
prettyprint (0.2.0)
|
94
151
|
prism (1.4.0)
|
152
|
+
psych (5.2.6)
|
153
|
+
date
|
154
|
+
stringio
|
95
155
|
public_suffix (6.0.2)
|
96
156
|
racc (1.8.1)
|
157
|
+
rack (3.1.16)
|
158
|
+
rack-session (2.1.1)
|
159
|
+
base64 (>= 0.1.0)
|
160
|
+
rack (>= 3.0.0)
|
161
|
+
rack-test (2.2.0)
|
162
|
+
rack (>= 1.3)
|
163
|
+
rackup (2.2.1)
|
164
|
+
rack (>= 3)
|
165
|
+
rails-dom-testing (2.3.0)
|
166
|
+
activesupport (>= 5.0.0)
|
167
|
+
minitest
|
168
|
+
nokogiri (>= 1.6)
|
169
|
+
rails-html-sanitizer (1.6.2)
|
170
|
+
loofah (~> 2.21)
|
171
|
+
nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
|
172
|
+
railties (7.2.2.1)
|
173
|
+
actionpack (= 7.2.2.1)
|
174
|
+
activesupport (= 7.2.2.1)
|
175
|
+
irb (~> 1.13)
|
176
|
+
rackup (>= 1.0.0)
|
177
|
+
rake (>= 12.2)
|
178
|
+
thor (~> 1.0, >= 1.2.2)
|
179
|
+
zeitwerk (~> 2.6)
|
97
180
|
rainbow (3.1.1)
|
98
181
|
rake (13.2.1)
|
182
|
+
rdoc (6.14.2)
|
183
|
+
erb
|
184
|
+
psych (>= 4.0.0)
|
99
185
|
regexp_parser (2.10.0)
|
186
|
+
reline (0.6.2)
|
187
|
+
io-console (~> 0.5)
|
100
188
|
rspec (3.13.0)
|
101
189
|
rspec-core (~> 3.13.0)
|
102
190
|
rspec-expectations (~> 3.13.0)
|
@@ -109,6 +197,14 @@ GEM
|
|
109
197
|
rspec-mocks (3.13.4)
|
110
198
|
diff-lcs (>= 1.2.0, < 2.0)
|
111
199
|
rspec-support (~> 3.13.0)
|
200
|
+
rspec-rails (6.1.5)
|
201
|
+
actionpack (>= 6.1)
|
202
|
+
activesupport (>= 6.1)
|
203
|
+
railties (>= 6.1)
|
204
|
+
rspec-core (~> 3.13)
|
205
|
+
rspec-expectations (~> 3.13)
|
206
|
+
rspec-mocks (~> 3.13)
|
207
|
+
rspec-support (~> 3.13)
|
112
208
|
rspec-support (3.13.3)
|
113
209
|
rubocop (1.75.7)
|
114
210
|
json (~> 2.3)
|
@@ -133,12 +229,16 @@ GEM
|
|
133
229
|
ruby-progressbar (1.13.0)
|
134
230
|
ruby2_keywords (0.0.5)
|
135
231
|
securerandom (0.4.1)
|
232
|
+
stringio (3.1.7)
|
233
|
+
thor (1.4.0)
|
136
234
|
tzinfo (2.0.6)
|
137
235
|
concurrent-ruby (~> 1.0)
|
138
236
|
uber (0.0.15)
|
139
237
|
unicode-display_width (3.1.4)
|
140
238
|
unicode-emoji (~> 4.0, >= 4.0.4)
|
141
239
|
unicode-emoji (4.0.4)
|
240
|
+
useragent (0.16.11)
|
241
|
+
zeitwerk (2.7.3)
|
142
242
|
|
143
243
|
PLATFORMS
|
144
244
|
aarch64-linux-gnu
|
@@ -155,10 +255,12 @@ PLATFORMS
|
|
155
255
|
|
156
256
|
DEPENDENCIES
|
157
257
|
bundler (~> 2)
|
258
|
+
combustion (~> 1.3)
|
158
259
|
consyncful!
|
159
|
-
database_cleaner-mongoid
|
260
|
+
database_cleaner-mongoid (~> 2.0)
|
160
261
|
rake (~> 13.0)
|
161
|
-
rspec (~> 3.
|
262
|
+
rspec (~> 3.13)
|
263
|
+
rspec-rails (~> 6.1)
|
162
264
|
rubocop
|
163
265
|
rubocop-rake
|
164
266
|
rubocop-rspec
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Consyncful
|
2
2
|
|
3
|
-
Contentful ->
|
3
|
+
[Contentful](https://www.contentful.com/) -> MongoDB synchronisation for Rails.
|
4
4
|
|
5
5
|
Requesting complicated models from the Contentful Delivery API in Rails applications is often too slow, and makes testing applications painful. Consyncful uses Contentful's synchronisation API to keep a local, up-to-date copy of the entire content in a Mongo database.
|
6
6
|
|
@@ -8,23 +8,45 @@ Once the content is available locally, finding and interact with contentful data
|
|
8
8
|
|
9
9
|
This gem doesn't provide any integration with the management API, or any way to update Contentful models from the local store. It is strictly read only.
|
10
10
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
11
|
+
```mermaid
|
12
|
+
flowchart TD
|
13
|
+
CF["Contentful (Delivery API / Sync API)"]
|
14
|
+
SY["rake consyncful:sync"]
|
15
|
+
DB["MongoDB (single collection)"]
|
16
|
+
APP["Rails models (Mongoid, subclass Consyncful::Base)"]
|
17
|
+
|
18
|
+
CF -->|sync| SY
|
19
|
+
SY -->|writes| DB
|
20
|
+
APP -->|queries| DB
|
21
|
+
```
|
22
|
+
|
23
|
+
## Contents
|
24
|
+
- [Setup](#setup)
|
25
|
+
- [Installation](#installation)
|
26
|
+
- [Configuration options](#configuration-options)
|
27
|
+
- [Creating contentful models in your Rails app](#creating-contentful-models-in-your-rails-app)
|
28
|
+
- [Synchronizing contentful data](#synchronizing-contentful-data)
|
29
|
+
- [Continuous sync](#continuous-sync-either-mode)
|
30
|
+
- [Refresh from scratch](#refresh-from-scratch)
|
31
|
+
- [Enabling webhook mode](#enabling-webhook-mode)
|
19
32
|
- [Sync callbacks](#sync-callbacks)
|
33
|
+
- [Sync specific contents using contentful tags](#sync-specific-contents-using-contentful-tag)
|
34
|
+
- [Finding and interacting with models](#finding-and-interacting-with-models)
|
35
|
+
- [Querying](#querying)
|
36
|
+
- [References](#references)
|
37
|
+
- [Finding entries from different content types](#finding-entries-from-different-content-types)
|
20
38
|
- [Using Locales for specific fields](#using-locales-for-specific-fields)
|
21
|
-
- [
|
22
|
-
|
39
|
+
- [Preserving Contentful timestamps](#preserving-contentful-timestamps)
|
40
|
+
- [MongoDB Configuration](#mongodb-configuration)
|
41
|
+
- [Choosing the Mongo Database](#choosing-the-mongo-database)
|
42
|
+
- [Why MongoDB?](#why-mongodb)
|
23
43
|
- [Development](#development)
|
24
44
|
- [Contributing](#contributing)
|
25
45
|
- [License](#license)
|
26
46
|
|
27
|
-
##
|
47
|
+
## Setup
|
48
|
+
|
49
|
+
### Installation
|
28
50
|
|
29
51
|
Add this line to your application's Gemfile:
|
30
52
|
|
@@ -40,16 +62,15 @@ If you don't already use Mongoid, generate a mongoid.yml by running:
|
|
40
62
|
|
41
63
|
$ rake g mongoid:config
|
42
64
|
|
43
|
-
|
65
|
+
Create `config/initializers/consyncful.rb`. An example with common configuration is:
|
44
66
|
|
45
|
-
Consyncful uses [contentful.rb](https://github.com/contentful/contentful.rb); client options are as documented there. Sync options are documented in the [Content Delivery Sync API docs](https://www.contentful.com/developers/docs/references/content-delivery-api/#/reference/synchronization).
|
46
67
|
```rb
|
47
68
|
Consyncful.configure do |config|
|
48
69
|
config.locale = 'en-NZ'
|
49
70
|
config.contentful_client_options = {
|
50
71
|
api_url: 'cdn.contentful.com',
|
51
72
|
space: 'space_id',
|
52
|
-
access_token: '
|
73
|
+
access_token: 'ACCESS_TOKEN',
|
53
74
|
environment: 'master', # optional
|
54
75
|
logger: Logger.new(STDOUT) # optional for debugging
|
55
76
|
}
|
@@ -61,9 +82,28 @@ Consyncful.configure do |config|
|
|
61
82
|
end
|
62
83
|
```
|
63
84
|
|
64
|
-
|
85
|
+
> [!NOTE]
|
86
|
+
> Consyncful uses the official [contentful.rb](https://github.com/contentful/contentful.rb) client. Any `contentful_client_options` you set are passed through to this library unchanged. Similary, settings in `contentful_sync_options` map to the parameters in Contentful’s [Content Delivery Sync API](https://www.contentful.com/developers/docs/references/content-delivery-api/#/reference/synchronization).
|
87
|
+
|
88
|
+
|
89
|
+
### Configuration options
|
90
|
+
|
91
|
+
| Option | Description | Default |
|
92
|
+
| --- | --- | --- |
|
93
|
+
| `sync_mode` | How syncing is triggered: `:poll` (periodic polling used by default) or `:webhook` (sync runs when a webhook is received from contentful). | `:poll` |
|
94
|
+
| `contentful_client_options` | Options passed through to the `contentful.rb` client. Defaults include `reuse_entries: true`, `api_url: 'cdn.contentful.com'`. | `{}` (merged with defaults) |
|
95
|
+
| `contentful_sync_options` | Contentful Sync API parameters (e.g., `limit`, `type`). Defaults include `limit: 100`, `type: 'all'`. | `{}` (merged with defaults) |
|
96
|
+
| `locale` | Default locale when mapping fields. | `'en-NZ'` |
|
97
|
+
| `content_tags` | Only store entries that have **any** of these tags. | `[]` |
|
98
|
+
| `ignore_content_tags` | Ignore entries with **any** of these tags. | `[]` |
|
99
|
+
| `preserve_contentful_timestamps` | Adds `contentful_created_at` and `contentful_updated_at` to models. | `false` |
|
100
|
+
| `mongo_client` | Mongoid client to use (from `mongoid.yml`). | `:default` |
|
101
|
+
| `mongo_collection` | MongoDB collection name for all entries. | `'contentful_models'` |
|
102
|
+
| `webhook_authentication_enabled` | Require Basic Auth for the webhook endpoint (only relevant when `sync_mode: :webhook`, enabled by default). | `true` |
|
103
|
+
| `webhook_user` | Username for webhook Basic Auth (when enabled). | `nil` |
|
104
|
+
| `webhook_password` | Password for webhook Basic Auth (when enabled). | `nil` |
|
65
105
|
|
66
|
-
|
106
|
+
## Creating contentful models in your Rails application
|
67
107
|
|
68
108
|
Create models by inheriting from `Consyncful::Base`
|
69
109
|
|
@@ -95,25 +135,123 @@ class ModelWithReferences < Consyncful::Base
|
|
95
135
|
end
|
96
136
|
```
|
97
137
|
|
98
|
-
|
138
|
+
## Synchronizing contentful data
|
99
139
|
|
100
|
-
|
140
|
+
`Consyncful` supports **two sync modes**:
|
141
|
+
- **Polling (default)** — checks Contentful on an interval and syncs changes.
|
142
|
+
- **Webhook** — Contentful calls your app; the worker syncs when a webhook arrives.
|
143
|
+
|
144
|
+
### Continuous sync (either mode)
|
145
|
+
Run the same task in both modes — the behaviour depends on your configuration:
|
101
146
|
|
102
147
|
$ rake consyncful:sync
|
103
148
|
|
104
|
-
|
149
|
+
- **Polling mode**: after the initial full sync, the worker polls every **15s** (configurable) and applies changes it finds.
|
150
|
+
- **Webhook mode**: after the initial full sync, the worker **does not poll**. It waits for a webhook signal and then runs a sync.
|
151
|
+
|
152
|
+
|
153
|
+
> [!NOTE]
|
154
|
+
> The first time you run this it will download all the Contentful content.
|
105
155
|
|
106
|
-
|
156
|
+
### Refresh from scratch
|
157
|
+
|
158
|
+
If you want to resynchronize everything (e.g., after model/content type renames), run:
|
107
159
|
|
108
160
|
$ rake consyncful:refresh
|
109
161
|
|
162
|
+
This performs a full rebuild of data from contentful.
|
163
|
+
|
110
164
|
It is recommended to refresh your data if you change model names.
|
111
165
|
|
112
166
|
Now you've synced your data, it is all available via your Rails models.
|
113
167
|
|
114
|
-
###
|
168
|
+
### Enabling webhook mode
|
169
|
+
|
170
|
+
> [!TIP]
|
171
|
+
> **Webhook mode is recommended on limited plans**.
|
172
|
+
> Polling makes API requests on every interval (default ~15s), which can quickly add up and exhaust quotas on lower-tier Contentful plans.
|
173
|
+
> **Webhook mode** only syncs when Contentful sends an event, dramatically reducing API calls. If you’re hitting rate limits—or want to avoid them—switch to `:webhook`.
|
174
|
+
> If you are hitting API rate limits and need to use polling, consider increasing the interval to reduce load.
|
175
|
+
|
176
|
+
|
177
|
+
#### 1. Set the sync mode to webhook
|
178
|
+
|
179
|
+
```
|
180
|
+
# e.g. config/initializers/consyncful.rb
|
181
|
+
Consyncful.configure do |c|
|
182
|
+
c.sync_mode = :webhook
|
183
|
+
end
|
184
|
+
```
|
185
|
+
|
186
|
+
#### 2. Mount the webhooks controller:
|
187
|
+
Expose the engine so Contentful can POST to it
|
188
|
+
```
|
189
|
+
# config/routes.rb
|
190
|
+
mount Consyncful::Engine, at: "/consyncful"
|
191
|
+
```
|
192
|
+
The webhook endpoint lives under this mount (e.g. `/consyncful/trigger_sync`).
|
193
|
+
|
194
|
+
#### 3. Authentication (recommended)
|
195
|
+
Webhook authentication is **on by default**:
|
196
|
+
```
|
197
|
+
Consyncful.configure do |c|
|
198
|
+
c.webhook_authentication_required = true # default
|
199
|
+
c.webhook_user = ENV["CONSYNCFUL_WEBHOOK_USER"]
|
200
|
+
c.webhook_password = ENV["CONSYNCFUL_WEBHOOK_PASSWORD"]
|
201
|
+
end
|
202
|
+
```
|
203
|
+
To accept webhooks **without** auth (not recommended), explicitly disable it:
|
204
|
+
```
|
205
|
+
c.webhook_authentication_required = false
|
206
|
+
```
|
207
|
+
|
208
|
+
#### 4. Create the webhook in Contentful
|
209
|
+
In your Contentful space/environment, add a webhook that points to your mounted route (e.g. `https://your-app.example.com/consyncful/trigger_sync`) and select which events should trigger a sync (publish/unpublish, entries, assets, etc.). See Contentful documents here for information on setting up a webhook: [Configuring a webhook](https://www.contentful.com/developers/docs/webhooks/configure-webhook/)
|
210
|
+
|
211
|
+
> [!IMPORTANT]
|
212
|
+
> If your application is behind global authentication, VPN, or an allowlist, Contentful won’t be able to reach the webhook endpoint. Ensure that `POST` requests from Contentful can reach your mounted path (e.g. `/consyncful/...`). In many setups this means adding an ingress rule or route exemption for the webhook path. Keeping webhook authentication **enabled** (default) is recommended; configure matching credentials in the Contentful webhook.
|
213
|
+
|
214
|
+
### Sync callbacks
|
215
|
+
|
216
|
+
You may want to attach some application logic to happen before or after a sync run, for example to update caches.
|
217
|
+
|
218
|
+
Callbacks can be registered using:
|
219
|
+
|
220
|
+
```ruby
|
221
|
+
Consyncful::Sync.before_run do
|
222
|
+
# do something before the run
|
223
|
+
end
|
224
|
+
```
|
225
|
+
|
226
|
+
```ruby
|
227
|
+
Consyncful::Sync.after_run do |updated_ids|
|
228
|
+
# invalidate cache for updated_ids, or something
|
229
|
+
end
|
230
|
+
```
|
231
|
+
|
232
|
+
### Sync specific contents using [Contentful Tag](https://www.contentful.com/help/tags/)
|
233
|
+
You can configure Consyncful to sync or ignore specific contents using Contentful Tag.
|
234
|
+
|
235
|
+
```rb
|
236
|
+
Consyncful.configure do |config|
|
237
|
+
# Any contents tagged with 'myTag' will be stored in the database.
|
238
|
+
# Other contents without 'myTag' would be ignored.
|
239
|
+
config.content_tags = ['myTag'] # defaults to []
|
240
|
+
end
|
241
|
+
```
|
242
|
+
|
243
|
+
Also, you can ignore contents with specific Tags.
|
244
|
+
|
245
|
+
```rb
|
246
|
+
Consyncful.configure do |config|
|
247
|
+
# Any contents tagged with 'ignoreTag' won't be stored in the database.
|
248
|
+
config.ignore_content_tags = ['ignoreTag'] # defaults to []
|
249
|
+
end
|
250
|
+
```
|
115
251
|
|
116
|
-
|
252
|
+
## Finding and interacting with models
|
253
|
+
|
254
|
+
### Querying
|
117
255
|
Models are available using standard Mongoid [queries](https://docs.mongodb.com/mongoid/current/tutorials/mongoid-queries/).
|
118
256
|
|
119
257
|
```ruby
|
@@ -122,7 +260,7 @@ instance = ModelName.find_by(instance: 'foo')
|
|
122
260
|
instance.is_awesome # true
|
123
261
|
```
|
124
262
|
|
125
|
-
|
263
|
+
### References
|
126
264
|
References work like you would expect:
|
127
265
|
|
128
266
|
```ruby
|
@@ -140,7 +278,7 @@ instance.other_things # all the referenced things, polymorphic, so might be diff
|
|
140
278
|
instance.other_things.in_order # ordered the same as in Contentful
|
141
279
|
```
|
142
280
|
|
143
|
-
|
281
|
+
### Finding entries from different content types
|
144
282
|
|
145
283
|
Because all Contentful models are stored as polymorphic subtypes of `Consyncful::Base`, you can query all entries without knowing what type you are looking for:
|
146
284
|
|
@@ -148,24 +286,6 @@ Because all Contentful models are stored as polymorphic subtypes of `Consyncful:
|
|
148
286
|
Consyncful::Base.where(title: 'a title') # [ #<ModelName>, #<OtherModelName> ]
|
149
287
|
```
|
150
288
|
|
151
|
-
### Sync callbacks
|
152
|
-
|
153
|
-
You may want to attach some application logic to happen before or after a sync run, for example to update caches.
|
154
|
-
|
155
|
-
Callbacks can be registered using:
|
156
|
-
|
157
|
-
```ruby
|
158
|
-
Consyncful::Sync.before_run do
|
159
|
-
# do something before the run
|
160
|
-
end
|
161
|
-
```
|
162
|
-
|
163
|
-
```ruby
|
164
|
-
Consyncful::Sync.after_run do |updated_ids|
|
165
|
-
# invalidate cache for updated_ids, or something
|
166
|
-
end
|
167
|
-
```
|
168
|
-
|
169
289
|
### Using Locales for specific fields
|
170
290
|
|
171
291
|
If fields have multiple locales then the default locale will be mapped to the field name. Additional locales will have a suffix (lower snake case) on the field name. e.g title (default), title_mi_nz (New Zealand Maori mi-NZ)
|
@@ -183,29 +303,11 @@ Consyncful.configure do |config|
|
|
183
303
|
end
|
184
304
|
```
|
185
305
|
|
186
|
-
|
187
|
-
You can configure Consyncful to sync or ignore specific contents using Contentful Tag.
|
188
|
-
|
189
|
-
```rb
|
190
|
-
Consyncful.configure do |config|
|
191
|
-
# Any contents tagged with 'myTag' will be stored in the database.
|
192
|
-
# Other contents without 'myTag' would be ignored.
|
193
|
-
config.content_tags = ['myTag'] # defaults to []
|
194
|
-
end
|
195
|
-
```
|
196
|
-
|
197
|
-
Also, you can ignore contents with specific Tags.
|
198
|
-
|
199
|
-
```rb
|
200
|
-
Consyncful.configure do |config|
|
201
|
-
# Any contents tagged with 'ignoreTag' won't be stored in the database.
|
202
|
-
config.ignore_content_tags = ['ignoreTag'] # defaults to []
|
203
|
-
end
|
204
|
-
```
|
306
|
+
## MongoDB Configuration
|
205
307
|
|
206
|
-
###
|
308
|
+
### Choosing the Mongo Database
|
207
309
|
|
208
|
-
You can
|
310
|
+
You can configure which Mongoid client Consyncful uses, as well as the name of the collection where entries are stored. This is useful if you want Consyncful data to live in a separate MongoDB database from your application-specific database.
|
209
311
|
|
210
312
|
```rb
|
211
313
|
Consyncful.configure do |config|
|
@@ -214,9 +316,9 @@ Consyncful.configure do |config|
|
|
214
316
|
end
|
215
317
|
```
|
216
318
|
|
217
|
-
### Why
|
319
|
+
### Why MongoDB?
|
218
320
|
|
219
|
-
Consyncful currently only supports Mongoid ODM because models
|
321
|
+
Consyncful currently only supports the Mongoid ODM because models require dynamic schemas. Extending support to ActiveRecord could be possible in the future, but it would also require maintaining database migrations alongside Contentful content type changes—which adds complexity we wanted to avoid.
|
220
322
|
|
221
323
|
## Development
|
222
324
|
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Consyncful
|
4
|
+
# The Consyncful::WebhookController is responsible for handling incoming
|
5
|
+
# webhook requests that can trigger synchronization jobs within Consyncful.
|
6
|
+
#
|
7
|
+
# Features:
|
8
|
+
# - Only responds to requests if `sync_mode` is configured as `:webhook`.
|
9
|
+
# - Optionally requires HTTP Basic authentication if
|
10
|
+
# `webhook_authentication_required` is enabled in configuration.
|
11
|
+
# - Exposes a single endpoint (`trigger_sync`) that signals a sync process
|
12
|
+
# through `Consyncful::Sync.signal_webhook!`.
|
13
|
+
#
|
14
|
+
# Security:
|
15
|
+
# - Uses `ActionController::HttpAuthentication::Basic` to enforce
|
16
|
+
# authentication when enabled.
|
17
|
+
# - Compares provided credentials with configured values using
|
18
|
+
# `ActiveSupport::SecurityUtils.secure_compare` to prevent timing attacks.
|
19
|
+
#
|
20
|
+
# Responses:
|
21
|
+
# - Returns `404 Not Found` if webhooks are not enabled.
|
22
|
+
# - Returns `202 Accepted` after signaling a sync.
|
23
|
+
class WebhookController < ActionController::API
|
24
|
+
include ActionController::HttpAuthentication::Basic::ControllerMethods
|
25
|
+
before_action :authenticate, if: -> { Consyncful.configuration.webhook_authentication_required && use_webhooks? }
|
26
|
+
|
27
|
+
def trigger_sync
|
28
|
+
return head :not_found unless use_webhooks?
|
29
|
+
|
30
|
+
Consyncful::Sync.signal_webhook!
|
31
|
+
head :accepted
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
def use_webhooks?
|
37
|
+
Consyncful.configuration.sync_mode == :webhook
|
38
|
+
end
|
39
|
+
|
40
|
+
def authenticate
|
41
|
+
config = Consyncful.configuration
|
42
|
+
authenticate_or_request_with_http_basic('Consyncful: Authenticate to Trigger Sync') do |username, password|
|
43
|
+
secure_compare(username, config.webhook_user) && secure_compare(password, config.webhook_password)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def secure_compare(value, expected)
|
48
|
+
ActiveSupport::SecurityUtils.secure_compare(value.to_s, expected.to_s)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
data/config/routes.rb
ADDED
@@ -2,7 +2,18 @@
|
|
2
2
|
|
3
3
|
# Handles Rails configurations for Consyncful
|
4
4
|
module Consyncful
|
5
|
-
|
5
|
+
# Provides configuration options for Consyncful, including:
|
6
|
+
# - Contentful API client and sync options
|
7
|
+
# - MongoDB client and collection settings
|
8
|
+
# - Locale and content tag filtering
|
9
|
+
# - Sync mode (poll or webhook)
|
10
|
+
# - Webhook authentication credentials
|
11
|
+
#
|
12
|
+
# This class is typically accessed and customized via
|
13
|
+
# Consyncful.configure do |config|
|
14
|
+
# config.locale = 'en-NZ'
|
15
|
+
# config.mongo_collection = 'my_models'
|
16
|
+
# end
|
6
17
|
class Configuration
|
7
18
|
attr_accessor :contentful_client_options,
|
8
19
|
:contentful_sync_options,
|
@@ -11,9 +22,15 @@ module Consyncful
|
|
11
22
|
:mongo_collection,
|
12
23
|
:content_tags,
|
13
24
|
:ignore_content_tags,
|
14
|
-
:preserve_contentful_timestamps
|
25
|
+
:preserve_contentful_timestamps,
|
26
|
+
:sync_mode,
|
27
|
+
:webhook_authentication_required,
|
28
|
+
:webhook_user,
|
29
|
+
:webhook_password
|
15
30
|
|
31
|
+
# rubocop:disable Metrics/MethodLength
|
16
32
|
def initialize
|
33
|
+
@sync_mode = :poll
|
17
34
|
@contentful_client_options = {}
|
18
35
|
@contentful_sync_options = {}
|
19
36
|
@locale = 'en-NZ'
|
@@ -22,7 +39,12 @@ module Consyncful
|
|
22
39
|
@content_tags = []
|
23
40
|
@ignore_content_tags = []
|
24
41
|
@preserve_contentful_timestamps = false
|
42
|
+
|
43
|
+
@webhook_authentication_required = true
|
44
|
+
@webhook_user = nil
|
45
|
+
@webhook_password = nil
|
25
46
|
end
|
47
|
+
# rubocop:enable Metrics/MethodLength
|
26
48
|
|
27
49
|
def initial_sync_options
|
28
50
|
options = { initial: true }
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Consyncful
|
4
|
+
# Rails engine for Consyncful.
|
5
|
+
#
|
6
|
+
# This isolates the Consyncful namespace and allows the gem
|
7
|
+
# to provide its own routes, controllers, and configuration
|
8
|
+
# within a Rails application without clashing with the host app.
|
9
|
+
class Engine < ::Rails::Engine
|
10
|
+
isolate_namespace Consyncful
|
11
|
+
end
|
12
|
+
end
|
@@ -14,8 +14,12 @@ module Consyncful
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def excluded_by_tag?
|
17
|
-
|
18
|
-
|
17
|
+
config = Consyncful.configuration
|
18
|
+
content_tags = config.content_tags
|
19
|
+
ignore_content_tags = config.ignore_content_tags
|
20
|
+
|
21
|
+
return (content_tags & item_tag_ids).empty? if content_tags.any?
|
22
|
+
return (ignore_content_tags & item_tag_ids).any? if ignore_content_tags.any?
|
19
23
|
|
20
24
|
false
|
21
25
|
end
|
@@ -65,7 +69,7 @@ module Consyncful
|
|
65
69
|
|
66
70
|
@item.fields_with_locales.each do |field, value_with_locales|
|
67
71
|
value_with_locales.each do |locale_code, value|
|
68
|
-
next if value.is_a? Contentful::File # assets are
|
72
|
+
next if value.is_a? Contentful::File # assets are handled below
|
69
73
|
|
70
74
|
field_name = localized_field_name(field, locale_code, default_locale)
|
71
75
|
field_name, value = mapped_field_entry_for(field_name, value)
|
data/lib/consyncful/sync.rb
CHANGED
@@ -27,10 +27,28 @@ module Consyncful
|
|
27
27
|
field :next_url
|
28
28
|
field :last_run_at, type: DateTime
|
29
29
|
|
30
|
+
field :webhook_pending, type: Boolean, default: false
|
31
|
+
|
30
32
|
def self.latest
|
31
33
|
last || new
|
32
34
|
end
|
33
35
|
|
36
|
+
##
|
37
|
+
# Signal that a webhook has been received and a sync should be triggered
|
38
|
+
def self.signal_webhook!
|
39
|
+
latest.set(webhook_pending: true)
|
40
|
+
true
|
41
|
+
end
|
42
|
+
|
43
|
+
##
|
44
|
+
# Consume the webhook signal and set webhook_pending to false
|
45
|
+
def self.consume_webhook_signal!
|
46
|
+
return false unless latest.webhook_pending?
|
47
|
+
|
48
|
+
latest.set(webhook_pending: false)
|
49
|
+
true
|
50
|
+
end
|
51
|
+
|
34
52
|
##
|
35
53
|
# Delete the previous sync chains from database and create a fresh one.
|
36
54
|
# Used to completely resync all items from Contentful.
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Consyncful
|
4
|
+
# The SyncRunner is responsible for continuously executing Contentful sync
|
5
|
+
# jobs at a configurable interval or in response to webhook signals.
|
6
|
+
#
|
7
|
+
# Modes:
|
8
|
+
# - :poll — runs the sync every N seconds (default 15)
|
9
|
+
# - :webhook — waits for webhook signals and triggers a sync when received
|
10
|
+
#
|
11
|
+
# Behavior:
|
12
|
+
# - Starts with an initial sync (`Consyncful::Sync.latest.run`).
|
13
|
+
# - In poll mode, sleeps for the configured interval and then re-runs sync.
|
14
|
+
# - In webhook mode, listens for webhook signals and runs sync immediately.
|
15
|
+
class SyncRunner
|
16
|
+
DEFAULT_INTERVAL = 15
|
17
|
+
VALID_MODES = %i[poll webhook].freeze
|
18
|
+
|
19
|
+
def initialize(seconds: nil, mode: nil)
|
20
|
+
@interval = seconds || DEFAULT_INTERVAL
|
21
|
+
@mode = validate_mode(mode)
|
22
|
+
end
|
23
|
+
|
24
|
+
def run
|
25
|
+
current_sync = Consyncful::Sync.latest
|
26
|
+
current_sync.run # Run initial sync
|
27
|
+
|
28
|
+
loop do
|
29
|
+
sleep(@interval)
|
30
|
+
current_sync.run if @mode == :poll || Consyncful::Sync.consume_webhook_signal!
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
def validate_mode(value)
|
37
|
+
sym = value.to_sym
|
38
|
+
return sym if VALID_MODES.include?(sym)
|
39
|
+
|
40
|
+
raise ArgumentError, "Unknown sync mode: #{sym.inspect} (expected :poll or :webhook)"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -1,28 +1,32 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
namespace :consyncful do
|
4
|
+
desc 'Run a one-time sync of the latest Contentful data into the app'
|
4
5
|
task update: [:environment] do
|
5
6
|
Consyncful::Sync.latest.run
|
6
7
|
end
|
7
8
|
|
9
|
+
desc 'Run a one-time full refresh of all Contentful data into the app (bypasses caching)'
|
8
10
|
task refresh: [:environment] do
|
9
11
|
Consyncful::Sync.fresh.run
|
10
12
|
end
|
11
13
|
|
14
|
+
desc 'Continuously sync Contentful data. Default: poll every N seconds (default: 15)'
|
12
15
|
task :sync, [:seconds] => %i[environment update_model_names] do |_task, args|
|
16
|
+
require 'consyncful/sync_runner'
|
13
17
|
Signal.trap('TERM') do
|
14
18
|
puts Rainbow("Graceful shutdown PID=#{Process.pid}").red
|
15
19
|
exit 0
|
16
20
|
end
|
17
21
|
|
18
|
-
seconds = args[:seconds]
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
end
|
22
|
+
seconds = args[:seconds]
|
23
|
+
mode = Consyncful.configuration&.sync_mode || :poll
|
24
|
+
puts "mode=#{mode.inspect} interval=#{(seconds || 15).inspect}s"
|
25
|
+
|
26
|
+
Consyncful::SyncRunner.new(seconds: seconds, mode: mode).run
|
24
27
|
end
|
25
28
|
|
29
|
+
desc 'Update stored model_type fields based on Contentful type mappings'
|
26
30
|
task update_model_names: [:environment] do
|
27
31
|
if Rails.autoloaders.zeitwerk_enabled?
|
28
32
|
Zeitwerk::Loader.eager_load_all
|
data/lib/consyncful/version.rb
CHANGED
data/lib/consyncful.rb
CHANGED
@@ -1,9 +1,17 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
# External dependencies
|
4
4
|
require 'mongoid'
|
5
5
|
require 'contentful'
|
6
|
+
|
7
|
+
# Internal library files
|
8
|
+
require 'consyncful/version'
|
6
9
|
require 'consyncful/configuration'
|
7
10
|
require 'consyncful/base'
|
8
11
|
require 'consyncful/sync'
|
9
|
-
|
12
|
+
|
13
|
+
# Rails integration (only load if Rails is present)
|
14
|
+
if defined?(Rails)
|
15
|
+
require 'consyncful/railtie'
|
16
|
+
require 'consyncful/engine'
|
17
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: consyncful
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andy Anastasiadis-Gray
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2025-
|
11
|
+
date: 2025-08-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: contentful
|
@@ -90,19 +90,23 @@ files:
|
|
90
90
|
- LICENSE.txt
|
91
91
|
- README.md
|
92
92
|
- Rakefile
|
93
|
+
- app/controllers/consyncful/webhook_controller.rb
|
93
94
|
- bin/console
|
94
95
|
- bin/setup
|
96
|
+
- config/routes.rb
|
95
97
|
- consyncful.gemspec
|
96
98
|
- dev_data/.keep
|
97
99
|
- docker-compose.yml
|
98
100
|
- lib/consyncful.rb
|
99
101
|
- lib/consyncful/base.rb
|
100
102
|
- lib/consyncful/configuration.rb
|
103
|
+
- lib/consyncful/engine.rb
|
101
104
|
- lib/consyncful/item_mapper.rb
|
102
105
|
- lib/consyncful/persisted_item.rb
|
103
106
|
- lib/consyncful/railtie.rb
|
104
107
|
- lib/consyncful/stats.rb
|
105
108
|
- lib/consyncful/sync.rb
|
109
|
+
- lib/consyncful/sync_runner.rb
|
106
110
|
- lib/consyncful/tasks/consyncful.rake
|
107
111
|
- lib/consyncful/version.rb
|
108
112
|
homepage: https://github.com/boost/consyncful
|