wcc-contentful 0.4.0.pre.alpha → 1.0.0.pre.rc3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. checksums.yaml +5 -5
  2. data/Guardfile +43 -0
  3. data/README.md +246 -11
  4. data/Rakefile +5 -0
  5. data/app/controllers/wcc/contentful/webhook_controller.rb +25 -24
  6. data/app/jobs/wcc/contentful/webhook_enable_job.rb +36 -2
  7. data/config/routes.rb +1 -1
  8. data/doc +1 -0
  9. data/lib/tasks/download_schema.rake +12 -0
  10. data/lib/wcc/contentful.rb +70 -16
  11. data/lib/wcc/contentful/active_record_shim.rb +72 -0
  12. data/lib/wcc/contentful/configuration.rb +177 -46
  13. data/lib/wcc/contentful/content_type_indexer.rb +14 -0
  14. data/lib/wcc/contentful/downloads_schema.rb +112 -0
  15. data/lib/wcc/contentful/engine.rb +33 -14
  16. data/lib/wcc/contentful/event.rb +171 -0
  17. data/lib/wcc/contentful/events.rb +41 -0
  18. data/lib/wcc/contentful/exceptions.rb +3 -0
  19. data/lib/wcc/contentful/indexed_representation.rb +2 -2
  20. data/lib/wcc/contentful/instrumentation.rb +31 -0
  21. data/lib/wcc/contentful/link.rb +28 -0
  22. data/lib/wcc/contentful/link_visitor.rb +122 -0
  23. data/lib/wcc/contentful/middleware.rb +7 -0
  24. data/lib/wcc/contentful/middleware/store.rb +158 -0
  25. data/lib/wcc/contentful/middleware/store/caching_middleware.rb +114 -0
  26. data/lib/wcc/contentful/model.rb +37 -3
  27. data/lib/wcc/contentful/model_builder.rb +1 -0
  28. data/lib/wcc/contentful/model_methods.rb +40 -15
  29. data/lib/wcc/contentful/model_singleton_methods.rb +47 -30
  30. data/lib/wcc/contentful/rake.rb +4 -0
  31. data/lib/wcc/contentful/rspec.rb +46 -0
  32. data/lib/wcc/contentful/services.rb +61 -27
  33. data/lib/wcc/contentful/simple_client.rb +81 -25
  34. data/lib/wcc/contentful/simple_client/management.rb +43 -10
  35. data/lib/wcc/contentful/simple_client/response.rb +61 -22
  36. data/lib/wcc/contentful/simple_client/typhoeus_adapter.rb +17 -17
  37. data/lib/wcc/contentful/store.rb +7 -66
  38. data/lib/wcc/contentful/store/README.md +85 -0
  39. data/lib/wcc/contentful/store/base.rb +34 -119
  40. data/lib/wcc/contentful/store/cdn_adapter.rb +71 -12
  41. data/lib/wcc/contentful/store/factory.rb +186 -0
  42. data/lib/wcc/contentful/store/instrumentation.rb +55 -0
  43. data/lib/wcc/contentful/store/interface.rb +82 -0
  44. data/lib/wcc/contentful/store/memory_store.rb +27 -24
  45. data/lib/wcc/contentful/store/postgres_store.rb +268 -101
  46. data/lib/wcc/contentful/store/postgres_store/schema_1.sql +73 -0
  47. data/lib/wcc/contentful/store/postgres_store/schema_2.sql +21 -0
  48. data/lib/wcc/contentful/store/query.rb +246 -0
  49. data/lib/wcc/contentful/store/query/interface.rb +63 -0
  50. data/lib/wcc/contentful/store/rspec_examples.rb +48 -0
  51. data/lib/wcc/contentful/store/rspec_examples/basic_store.rb +629 -0
  52. data/lib/wcc/contentful/store/rspec_examples/include_param.rb +283 -0
  53. data/lib/wcc/contentful/store/rspec_examples/nested_queries.rb +342 -0
  54. data/lib/wcc/contentful/sync_engine.rb +181 -0
  55. data/lib/wcc/contentful/test.rb +7 -0
  56. data/lib/wcc/contentful/test/attributes.rb +56 -0
  57. data/lib/wcc/contentful/test/double.rb +76 -0
  58. data/lib/wcc/contentful/test/factory.rb +101 -0
  59. data/lib/wcc/contentful/version.rb +1 -1
  60. data/wcc-contentful.gemspec +23 -11
  61. metadata +299 -116
  62. data/Gemfile +0 -6
  63. data/app/jobs/wcc/contentful/delayed_sync_job.rb +0 -63
  64. data/lib/wcc/contentful/client_ext.rb +0 -28
  65. data/lib/wcc/contentful/graphql.rb +0 -14
  66. data/lib/wcc/contentful/graphql/builder.rb +0 -177
  67. data/lib/wcc/contentful/graphql/types.rb +0 -54
  68. data/lib/wcc/contentful/simple_client/http_adapter.rb +0 -24
  69. data/lib/wcc/contentful/store/lazy_cache_store.rb +0 -161
@@ -0,0 +1,73 @@
1
+ CREATE TABLE IF NOT EXISTS wcc_contentful_schema_version (
2
+ version integer PRIMARY KEY,
3
+ updated_at timestamp DEFAULT now()
4
+ );
5
+
6
+ START TRANSACTION;
7
+
8
+ CREATE TABLE IF NOT EXISTS contentful_raw (
9
+ -- The Contentful 'sys'->'id'
10
+ id varchar PRIMARY KEY,
11
+ -- The contentful entry
12
+ data jsonb,
13
+ -- Every ID that this entry links to in 'fields'->*->[each locale]->'sys'->'id'
14
+ links text[]
15
+ );
16
+ CREATE INDEX IF NOT EXISTS contentful_raw_value_type ON contentful_raw ((data->'sys'->>'type'));
17
+ CREATE INDEX IF NOT EXISTS contentful_raw_value_content_type ON contentful_raw ((data->'sys'->'contentType'->'sys'->>'id'));
18
+
19
+ -- Insert or update a Contentful entry by it's ID
20
+ CREATE or replace FUNCTION "fn_contentful_upsert_entry"(_id varchar, _data jsonb, _links text[]) RETURNS jsonb AS $$
21
+ DECLARE
22
+ prev jsonb;
23
+ BEGIN
24
+ SELECT data, links FROM contentful_raw WHERE id = _id INTO prev;
25
+ INSERT INTO contentful_raw (id, data, links) values (_id, _data, _links)
26
+ ON CONFLICT (id) DO
27
+ UPDATE
28
+ SET data = _data,
29
+ links = _links;
30
+ RETURN prev;
31
+ END;
32
+ $$ LANGUAGE 'plpgsql';
33
+
34
+ -- Joins the entries table to itself by all the linked entries down to depth 5.
35
+ -- Each entry has a row for each downstream entry in it's tree.
36
+ -- Example:
37
+ -- | id | included_id | depth |
38
+ -- | page1 | page2 | 1 |
39
+ -- | page1 | subpage2 | 2 | -- through page2
40
+ -- | page1 | asset1 | 1 |
41
+ -- | page2 | subpage2 | 1 |
42
+ -- ...
43
+ CREATE MATERIALIZED VIEW IF NOT EXISTS contentful_raw_includes_ids_jointable AS
44
+ WITH RECURSIVE includes (root_id, depth) AS (
45
+ SELECT t.id as root_id, 0, t.id, t.links FROM contentful_raw t
46
+ UNION ALL
47
+ SELECT l.root_id, l.depth + 1, r.id, r.links
48
+ FROM includes l, contentful_raw r
49
+ WHERE r.id = ANY(l.links) AND l.depth < 5
50
+ )
51
+ SELECT root_id as id, id as included_id, min(depth)
52
+ FROM includes
53
+ GROUP BY root_id, id;
54
+
55
+ CREATE INDEX IF NOT EXISTS contentful_raw_includes_ids_jointable_id ON contentful_raw_includes_ids_jointable (id);
56
+ CREATE UNIQUE INDEX IF NOT EXISTS contentful_raw_includes_ids_jointable_id_included_id ON contentful_raw_includes_ids_jointable (id, included_id);
57
+
58
+ -- Uses the contentful_raw_includes_ids_jointable to join the entries table to itself,
59
+ -- aggregating the included entries into an array.
60
+ -- Example:
61
+ -- | id | data | includes |
62
+ -- | page1 | jsonb | {page2 jsonb, subpage2 jsonb, asset1 jsonb} |
63
+ CREATE OR REPLACE VIEW contentful_raw_includes AS
64
+ SELECT t.id, t.data, array_remove(array_agg(r_incl.data), NULL) as includes
65
+ FROM contentful_raw t
66
+ LEFT JOIN contentful_raw_includes_ids_jointable incl ON t.id = incl.id
67
+ LEFT JOIN contentful_raw r_incl ON r_incl.id = incl.included_id
68
+ GROUP BY t.id, t.data;
69
+
70
+ INSERT INTO wcc_contentful_schema_version
71
+ VALUES (1);
72
+
73
+ COMMIT;
@@ -0,0 +1,21 @@
1
+ START TRANSACTION;
2
+
3
+ -- Convert a jsonb array or jsonb value to a jsonb array
4
+ CREATE or replace FUNCTION "fn_contentful_jsonb_any_to_jsonb_array"(potential_arr jsonb) RETURNS jsonb AS $$
5
+ DECLARE
6
+ result jsonb;
7
+ BEGIN
8
+ SELECT
9
+ CASE
10
+ WHEN jsonb_typeof(potential_arr) = 'array' THEN potential_arr
11
+ ELSE jsonb_build_array(potential_arr)
12
+ END
13
+ INTO result;
14
+ RETURN result;
15
+ END;
16
+ $$ LANGUAGE 'plpgsql';
17
+
18
+
19
+ INSERT INTO wcc_contentful_schema_version
20
+ VALUES (2);
21
+ COMMIT;
@@ -0,0 +1,246 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../../contentful'
4
+ require_relative './query/interface'
5
+
6
+ module WCC::Contentful::Store
7
+ # The default query object returned by Stores that extend WCC::Contentful::Store::Base.
8
+ # It exposes several chainable query methods to apply query filters.
9
+ # Enumerating the query executes it, caching the result.
10
+ class Query
11
+ include WCC::Contentful::Store::Query::Interface
12
+ include Enumerable
13
+
14
+ # by default all enumerable methods delegated to the to_enum method
15
+ delegate(*(Enumerable.instance_methods - Module.instance_methods), to: :to_enum)
16
+
17
+ # except count, which should not iterate the lazy enumerator
18
+ delegate :count, to: :result_set
19
+
20
+ # Executes the query against the store and memoizes the resulting enumerable.
21
+ # Subclasses can override this to provide a more efficient implementation.
22
+ def to_enum
23
+ @to_enum ||=
24
+ result_set.lazy.map { |row| resolve_includes(row, @options[:include]) }
25
+ end
26
+
27
+ attr_reader :store, :content_type, :conditions
28
+
29
+ def initialize(store, content_type:, conditions: nil, options: nil, **extra)
30
+ @store = store
31
+ @content_type = content_type
32
+ @conditions = conditions || []
33
+ @options = options || {}
34
+ @extra = extra
35
+ end
36
+
37
+ # Returns a new chained Query that has a new condition. The new condition
38
+ # represents the WHERE comparison being applied here. The underlying store
39
+ # implementation translates this condition statement into an appropriate
40
+ # query against the datastore.
41
+ #
42
+ # @example
43
+ # query = query.apply_operator(:gt, :timestamp, '2019-01-01', context)
44
+ # # in a SQL based store, the query now contains a condition like:
45
+ # # WHERE table.'timestamp' > '2019-01-01'
46
+ #
47
+ # @operator one of WCC::Contentful::Store::Query::Interface::OPERATORS
48
+ # @field The path through the fields of the content type that we are querying against.
49
+ # Can be an array, symbol, or dotted-notation path specification.
50
+ # @expected The expected value to compare the field's value against.
51
+ # @context A context object optionally containing `context[:locale]`
52
+ def apply_operator(operator, field, expected, context = nil)
53
+ raise ArgumentError, "Operator #{operator} not supported" unless respond_to?(operator)
54
+
55
+ field = field.to_s if field.is_a? Symbol
56
+ path = field.is_a?(Array) ? field : field.split('.')
57
+
58
+ path = self.class.normalize_condition_path(path, context)
59
+
60
+ _append_condition(
61
+ Condition.new(path, operator, expected)
62
+ )
63
+ end
64
+
65
+ WCC::Contentful::Store::Query::Interface::OPERATORS.each do |op|
66
+ # @see #apply_operator
67
+ define_method(op) do |field, expected, context = nil|
68
+ apply_operator(op, field, expected, context)
69
+ end
70
+ end
71
+
72
+ # Called with a filter object by {Base#find_by} in order to apply the filter.
73
+ # The filter in this case is a hash where the keys are paths and the values
74
+ # are expectations.
75
+ # @see #apply_operator
76
+ def apply(filter, context = nil)
77
+ self.class.flatten_filter_hash(filter).reduce(self) do |query, cond|
78
+ query.apply_operator(cond[:op], cond[:path], cond[:expected], context)
79
+ end
80
+ end
81
+
82
+ # Override this to provide a result set from the Query object itself
83
+ # rather than from calling #execute in the store.
84
+ def result_set
85
+ @result_set ||= store.execute(self)
86
+ end
87
+
88
+ private
89
+
90
+ def _append_condition(condition)
91
+ self.class.new(
92
+ store,
93
+ content_type: content_type,
94
+ conditions: conditions + [condition],
95
+ options: @options,
96
+ **@extra
97
+ )
98
+ end
99
+
100
+ # naive implementation recursively descends the graph to turns links into
101
+ # the actual entry data. If the result set from #execute returns a tuple,
102
+ # it tries to pull links from the second column in the tuple. This allows
103
+ # a store implementation to return ex. `SELECT entry, includes FROM...`
104
+ # Otherwise, if the store does not return a tuple or does not have an includes
105
+ # column, it calls {Base#find} for each link and so it is very inefficient.
106
+ def resolve_includes(row, depth)
107
+ entry = row.try(:entry) || row.try(:[], 0) || row
108
+ includes = row.try(:includes) || row.try(:[], 1)
109
+ return entry unless entry && depth && depth > 0
110
+
111
+ WCC::Contentful::LinkVisitor.new(entry, :Link, :Asset,
112
+ # Walk all the links except for the leaf nodes
113
+ depth: depth - 1).map! do |val|
114
+ resolve_link(val, includes)
115
+ end
116
+ end
117
+
118
+ # Returns the resolved link if it exists in the includes hash, or returns
119
+ # the link hash.
120
+ def resolve_link(val, includes)
121
+ return val unless val.is_a?(Hash) && val.dig('sys', 'type') == 'Link'
122
+
123
+ id = val.dig('sys', 'id')
124
+ included =
125
+ if includes
126
+ includes[id]
127
+ else
128
+ @store.find(id)
129
+ end
130
+
131
+ included || val
132
+ end
133
+
134
+ class << self
135
+ def op?(key)
136
+ Interface::OPERATORS.include?(key.to_sym)
137
+ end
138
+
139
+ # Turns a hash into a flat array of individual conditions, where each
140
+ # element can be passed as params to apply_operator
141
+ def flatten_filter_hash(hash, path = [])
142
+ hash.flat_map do |(k, v)|
143
+ k = k.to_s
144
+ if k.include?('.')
145
+ k, *rest = k.split('.')
146
+ v = { rest.join('.') => v }
147
+ end
148
+
149
+ if v.is_a? Hash
150
+ flatten_filter_hash(v, path + [k])
151
+ elsif op?(k)
152
+ { path: path, op: k.to_sym, expected: v }
153
+ else
154
+ { path: path + [k], op: :eq, expected: v }
155
+ end
156
+ end
157
+ end
158
+
159
+ def known_locales
160
+ @known_locales = WCC::Contentful.locales.keys
161
+ end
162
+ RESERVED_NAMES = %w[fields sys].freeze
163
+
164
+ # Takes a path array in non-normal form and inserts 'sys', 'fields',
165
+ # and the current locale as appropriate to normalize it.
166
+ # rubocop:disable Metrics/BlockNesting
167
+ def normalize_condition_path(path, context = nil)
168
+ context_locale = context[:locale] if context.present?
169
+ context_locale ||= 'en-US'
170
+
171
+ rev_path = path.reverse
172
+ new_path = []
173
+
174
+ current_tuple = []
175
+ current_locale_was_inferred = false
176
+ until rev_path.empty? && current_tuple.empty?
177
+ raise ArgumentError, "Query too complex: #{path.join('.')}" if new_path.length > 7
178
+
179
+ case current_tuple.length
180
+ when 0
181
+ # expect a locale
182
+ current_tuple <<
183
+ if known_locales.include?(rev_path[0])
184
+ current_locale_was_inferred = false
185
+ rev_path.shift
186
+ else
187
+ # infer locale
188
+ current_locale_was_inferred = true
189
+ context_locale
190
+ end
191
+ when 1
192
+ # expect a path
193
+ current_tuple << rev_path.shift
194
+ when 2
195
+ # expect 'sys' or 'fields'
196
+ current_tuple <<
197
+ if RESERVED_NAMES.include?(rev_path[0])
198
+ rev_path.shift
199
+ else
200
+ # infer 'sys' or 'fields'
201
+ current_tuple.last == 'id' ? 'sys' : 'fields'
202
+ end
203
+
204
+ if current_tuple.last == 'sys' && current_locale_was_inferred
205
+ # remove the inferred current locale
206
+ current_tuple.shift
207
+ end
208
+ new_path << current_tuple
209
+ current_tuple = []
210
+ end
211
+ end
212
+
213
+ new_path.flat_map { |x| x }.reverse.freeze
214
+ end
215
+ # rubocop:enable Metrics/BlockNesting
216
+ end
217
+
218
+ Condition =
219
+ Struct.new(:path, :op, :expected) do
220
+ LINK_KEYS = %w[id type linkType].freeze
221
+
222
+ def path_tuples
223
+ @path_tuples ||=
224
+ [].tap do |arr|
225
+ remaining = path.dup
226
+ until remaining.empty?
227
+ locale = nil
228
+ link_sys = nil
229
+ link_field = nil
230
+
231
+ sys_or_fields = remaining.shift
232
+ field = remaining.shift
233
+ locale = remaining.shift if sys_or_fields == 'fields'
234
+
235
+ if remaining[0] == 'sys' && LINK_KEYS.include?(remaining[1])
236
+ link_sys = remaining.shift
237
+ link_field = remaining.shift
238
+ end
239
+
240
+ arr << [sys_or_fields, field, locale, link_sys, link_field].compact
241
+ end
242
+ end
243
+ end
244
+ end
245
+ end
246
+ end
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ class WCC::Contentful::Store::Query
4
+ # This module represents the common interface of queries that must be returned
5
+ # by a store's #find_all implementation.
6
+ # It is documentation ONLY and does not add functionality.
7
+ #
8
+ # This is distinct from WCC::Contentful::Store::Query, because certain helpers
9
+ # exposed publicly by that abstract class are not part of the actual interface
10
+ # and can change without a major version update.
11
+ module Interface
12
+ include Enumerable
13
+
14
+ # The set of operators that can be applied to a query. Not all stores
15
+ # implement all operators. At a bare minimum a store must implement #eq.
16
+ OPERATORS = %i[
17
+ eq
18
+ ne
19
+ all
20
+ in
21
+ nin
22
+ exists
23
+ lt
24
+ lte
25
+ gt
26
+ gte
27
+ query
28
+ match
29
+ ].freeze
30
+
31
+ WCC::Contentful::Store::Query::Interface::OPERATORS.each do |op|
32
+ # @see #apply_operator
33
+ define_method(op) do |_field, _expected, _context = nil|
34
+ raise NotImplementedError, "#{self.class} does not implement ##{op}"
35
+ end
36
+ end
37
+
38
+ # Applies an equality condition to the query. The underlying store
39
+ # translates this into a '==' check.
40
+ #
41
+ # sig {abstract.params(
42
+ # field: T.any(T::String),
43
+ # expected: T.untyped,
44
+ # context: T.nilable(T::Hash[T.untyped, T.untyped])
45
+ # ).returns(T.self_type)}
46
+ def eq(_field, _expected, _context = nil)
47
+ raise NotImplementedError, "#{self.class} does not implement #eq"
48
+ end
49
+
50
+ # Called with a filter object in order to apply the filter.
51
+ # The filter in this case is a hash where the keys are paths and the values
52
+ # are expectations.
53
+ #
54
+ # sig {abstract.params(
55
+ # field: T.any(T::String),
56
+ # expected: T.untyped,
57
+ # context: T.nilable(T::Hash[T.untyped, T.untyped])
58
+ # ).returns(T.self_type)}
59
+ def apply(_filter, _context = nil)
60
+ raise NotImplementedError, "#{self.class} does not implement #apply"
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative './rspec_examples/basic_store'
4
+ require_relative './rspec_examples/nested_queries'
5
+ require_relative './rspec_examples/include_param'
6
+
7
+ # rubocop:disable Style/BlockDelimiters
8
+
9
+ # These shared examples are included to help you implement a new store from scratch.
10
+ # To get started implementing your store, require this file and then include the
11
+ # shared examples in your RSpec block.
12
+ #
13
+ # The shared examples take a hash which describes the feature set that this store
14
+ # implements. All the additional features start out in the 'pending' state,
15
+ # once you've implemented that feature in your store then you can switch them
16
+ # to `true`.
17
+ #
18
+ # [:nested_queries] - This feature allows queries that reference a field on a
19
+ # linked object, example: `Player.find_by(team: { slug: '/dallas-cowboys' })`.
20
+ # This becomes essentially a JOIN. For reference see the Postgres store.
21
+ # [:include_param] - This feature defines how the store respects the `include: n`
22
+ # key in the Options hash. Some stores can make use of this parameter to get
23
+ # all linked entries of an object in a single query.
24
+ # If your store does not respect the include parameter, then the Model layer
25
+ # will be calling #find a lot in order to resolve linked entries.
26
+ #
27
+ # @example
28
+ # require 'wcc/contentful/store/rspec_examples'
29
+ # RSpec.describe MyStore do
30
+ # subject { MyStore.new }
31
+ #
32
+ # it_behaves_like 'contentful store', {
33
+ # # nested_queries: true,
34
+ # # include_param: true
35
+ # }
36
+ #
37
+ RSpec.shared_examples 'contentful store' do |feature_set|
38
+ feature_set = {
39
+ nested_queries: 'pending',
40
+ include_param: 'pending'
41
+ }.merge(feature_set&.symbolize_keys || {})
42
+
43
+ include_examples 'basic store'
44
+ include_examples 'supports nested queries', feature_set[:nested_queries]
45
+ include_examples 'supports include param', feature_set[:include_param]
46
+ end
47
+
48
+ # rubocop:enable Style/BlockDelimiters