wcc-contentful 1.1.2 → 1.3.0

data/lib/wcc/contentful/store/cdn_adapter.rb

@@ -3,7 +3,7 @@
 module WCC::Contentful::Store
   class CDNAdapter
     include WCC::Contentful::Store::Interface
-    # Note: CDNAdapter should not instrument store events cause it's not a store.
+    # NOTE: CDNAdapter should not instrument store events cause it's not a store.
 
     attr_writer :client, :preview_client
 
@@ -72,9 +72,13 @@ module WCC::Contentful::Store
       delegate :count, to: :response
 
       def to_enum
-        return response.items unless @options[:include]
+        return response.each_page.flat_map(&:page_items) unless @options[:include]
 
-        response.items.map { |e| resolve_includes(e, @options[:include]) }
+        response.each_page
+          .flat_map { |page| page.page_items.each_with_object(page).to_a }
+          .map do |e, page|
+            resolve_includes(e, page.includes, depth: @options[:include])
+          end
       end
 
       def initialize(store, client:, relation:, options: nil, **extra)
@@ -160,18 +164,18 @@ module WCC::Contentful::Store
           end
       end
 
-      def resolve_includes(entry, depth)
+      def resolve_includes(entry, includes, depth:)
         return entry unless entry && depth && depth > 0
 
         # Dig links out of response.includes and insert them into the entry
         WCC::Contentful::LinkVisitor.new(entry, :Link, depth: depth - 1).map! do |val|
-          resolve_link(val)
+          resolve_link(val, includes)
         end
       end
 
-      def resolve_link(val)
+      def resolve_link(val, includes)
         return val unless val.is_a?(Hash) && val.dig('sys', 'type') == 'Link'
-        return val unless included = response.includes[val.dig('sys', 'id')]
+        return val unless included = includes[val.dig('sys', 'id')]
 
         included
       end

data/lib/wcc/contentful/store/factory.rb

@@ -90,7 +90,7 @@ module WCC::Contentful::Store
         next if m[0].respond_to?(:call)
 
         raise ArgumentError, "The middleware '#{m[0]&.try(:name) || m[0]}' cannot be applied!  " \
-          'It must respond to :call'
+                             'It must respond to :call'
       end
 
       validate_store!(store)

data/lib/wcc/contentful/store/memory_store.rb

@@ -9,13 +9,15 @@ module WCC::Contentful::Store
   class MemoryStore < Base
     def initialize
       super
+
+      @mutex = Concurrent::ReentrantReadWriteLock.new
       @hash = {}
     end
 
     def set(key, value)
       value = value.deep_dup.freeze
       ensure_hash value
-      mutex.with_write_lock do
+      @mutex.with_write_lock do
         old = @hash[key]
         @hash[key] = value
         old
@@ -23,17 +25,17 @@ module WCC::Contentful::Store
     end
 
     def delete(key)
-      mutex.with_write_lock do
+      @mutex.with_write_lock do
         @hash.delete(key)
       end
     end
 
     def keys
-      mutex.with_read_lock { @hash.keys }
+      @mutex.with_read_lock { @hash.keys }
     end
 
     def find(key, **_options)
-      mutex.with_read_lock do
+      @mutex.with_read_lock do
         @hash[key]
       end
     end
@@ -41,11 +43,12 @@ module WCC::Contentful::Store
     SUPPORTED_OPS = %i[eq ne in nin].freeze
 
     def execute(query)
-      (query.conditions.map(&:op) - SUPPORTED_OPS).each do |op|
-        raise ArgumentError, "Operator :#{op} not supported"
+      if bad_op = (query.conditions.map(&:op) - SUPPORTED_OPS).first
+        raise ArgumentError, "Operator :#{bad_op} not supported"
       end
 
-      relation = mutex.with_read_lock { @hash.values }
+      # Since @hash.values returns a new array, we only need to lock here
+      relation = @mutex.with_read_lock { @hash.values }
 
       # relation is an enumerable that we apply conditions to in the form of
       #  Enumerable#select and Enumerable#reject.

data/lib/wcc/contentful/store/postgres_store.rb

@@ -23,18 +23,22 @@ module WCC::Contentful::Store
       connection_options ||= { dbname: 'postgres' }
       pool_options ||= {}
       @connection_pool = PostgresStore.build_connection_pool(connection_options, pool_options)
-      @dirty = false
+      @dirty = Concurrent::AtomicBoolean.new
+      @mutex = Mutex.new
     end
 
     def set(key, value)
       ensure_hash value
+
       result =
-        @connection_pool.with do |conn|
-          conn.exec_prepared('upsert_entry', [
-                               key,
-                               value.to_json,
-                               quote_array(extract_links(value))
-                             ])
+        _instrument 'upsert_entry' do
+          @connection_pool.with do |conn|
+            conn.exec_prepared('upsert_entry', [
+                                 key,
+                                 value.to_json,
+                                 quote_array(extract_links(value))
+                               ])
+          end
         end
 
       previous_value =
@@ -46,18 +50,22 @@ module WCC::Contentful::Store
         end
 
       if views_need_update?(value, previous_value)
-        # mark dirty - we need to refresh the materialized view
-        unless mutex.with_read_lock { @dirty }
-          _instrument 'mark_dirty'
-          mutex.with_write_lock { @dirty = true }
-        end
+        # Mark the views as needing to be refreshed, they will be refreshed on the next query.
+        was_dirty = @dirty.make_true
+        # Send out an instrumentation event if we are the thread that marked it dirty
+        # (make_true returns true if the value changed)
+        _instrument 'mark_dirty' if was_dirty
       end
 
       previous_value
     end
 
     def keys
-      result = @connection_pool.with { |conn| conn.exec_prepared('select_ids') }
+      result =
+        _instrument 'select_ids' do
+          @connection_pool.with { |conn| conn.exec_prepared('select_ids') }
+        end
+
       arr = []
       result.each { |r| arr << r['id'].strip }
       arr
@@ -66,14 +74,21 @@ module WCC::Contentful::Store
     end
 
     def delete(key)
-      result = @connection_pool.with { |conn| conn.exec_prepared('delete_by_id', [key]) }
+      result =
+        _instrument 'delete_by_id', key: key do
+          @connection_pool.with { |conn| conn.exec_prepared('delete_by_id', [key]) }
+        end
+
       return if result.num_tuples == 0
 
       JSON.parse(result.getvalue(0, 1))
     end
 
     def find(key, **_options)
-      result = @connection_pool.with { |conn| conn.exec_prepared('select_entry', [key]) }
+      result =
+        _instrument 'select_entry', key: key do
+          @connection_pool.with { |conn| conn.exec_prepared('select_entry', [key]) }
+        end
       return if result.num_tuples == 0
 
       JSON.parse(result.getvalue(0, 1))
@@ -90,22 +105,21 @@ module WCC::Contentful::Store
     end
 
     def exec_query(statement, params = [])
-      if mutex.with_read_lock { @dirty }
-        was_dirty =
-          mutex.with_write_lock do
-            was_dirty = @dirty
-            @dirty = false
-            was_dirty
-          end
-
-        if was_dirty
-          _instrument 'refresh_views' do
-            @connection_pool.with { |conn| conn.exec_prepared('refresh_views_concurrently') }
+      if @dirty.true?
+        # Only one thread should call refresh_views_concurrently but all should wait for it to finish.
+        @mutex.synchronize do
+          # We have to check again b/c another thread may have gotten here first
+          if @dirty.true?
+            _instrument 'refresh_views' do
+              @connection_pool.with { |conn| conn.exec_prepared('refresh_views_concurrently') }
+            end
+            # Mark that the views have been refreshed.
+            @dirty.make_false
           end
         end
       end
 
-      logger&.debug('[PostgresStore] ' + statement + "\n" + params.inspect)
+      logger&.debug("[PostgresStore] #{statement} #{params.inspect}")
       _instrument 'exec' do
         @connection_pool.with { |conn| conn.exec(statement, params) }
       end
@@ -231,16 +245,12 @@ module WCC::Contentful::Store
         end
 
         statement =
-          select_statement +
-          " FROM #{table} AS t \n" +
-          joins.join("\n") + "\n" +
-          statement +
-          (limit_statement || '')
+          "#{select_statement} FROM #{table} AS t \n#{joins.join("\n")}\n#{statement}#{limit_statement || ''}"
 
         [statement, params]
       end
 
-      def _eq(path, expected, params)
+      def _eq(path, expected, params) # rubocop:disable Layout/LineContinuationLeadingSpace
         return " AND t.id = $#{push_param(expected, params)}" if path == %w[sys id]
 
         if path[3] == 'sys'
@@ -248,12 +258,12 @@ module WCC::Contentful::Store
           # into it to detect whether it contains `{ "sys": { "id" => expected } }`
           expected = { 'sys' => { path[4] => expected } }.to_json
           return ' AND fn_contentful_jsonb_any_to_jsonb_array(t.data->' \
-            "#{quote_parameter_path(path.take(3))}) @> " \
-            "jsonb_build_array($#{push_param(expected, params)}::jsonb)"
+                 "#{quote_parameter_path(path.take(3))}) @> " \
+                 "jsonb_build_array($#{push_param(expected, params)}::jsonb)"
         end
 
-        " AND t.data->#{quote_parameter_path(path)}" \
-          " @> to_jsonb($#{push_param(expected, params)})"
+        " AND t.data->#{quote_parameter_path(path)} " \
+          "@> to_jsonb($#{push_param(expected, params)})"
       end
 
       PARAM_TYPES = {
@@ -292,7 +302,7 @@ module WCC::Contentful::Store
       def push_join(_path, joins)
         table_alias = "s#{joins.length}"
         joins << "JOIN contentful_raw AS #{table_alias} ON " \
-          "#{table_alias}.id=ANY(t.links)"
+                 "#{table_alias}.id=ANY(t.links)"
         table_alias
       end
     end
@@ -326,8 +336,8 @@ module WCC::Contentful::Store
       end
 
       def schema_ensured?(conn)
-        result = conn.exec('SELECT version FROM wcc_contentful_schema_version' \
-          ' ORDER BY version DESC LIMIT 1')
+        result = conn.exec('SELECT version FROM wcc_contentful_schema_version ' \
+                           'ORDER BY version DESC LIMIT 1')
         return false if result.num_tuples == 0
 
         result[0]['version'].to_i >= EXPECTED_VERSION
@@ -339,8 +349,8 @@ module WCC::Contentful::Store
       def ensure_schema(conn)
         result =
           begin
-            conn.exec('SELECT version FROM wcc_contentful_schema_version' \
-          ' ORDER BY version DESC')
+            conn.exec('SELECT version FROM wcc_contentful_schema_version ' \
+                      'ORDER BY version DESC')
           rescue PG::UndefinedTable
             []
           end

data/lib/wcc/contentful/store/query.rb

@@ -36,7 +36,7 @@ module WCC::Contentful::Store
 
     FALSE_VALUES = [
       false, 0,
-      '0', :"0",
+      '0', :'0',
       'f', :f,
       'F', :F,
       'false', :false, # rubocop:disable Lint/BooleanSymbol
@@ -237,7 +237,7 @@ module WCC::Contentful::Store
 
     Condition =
       Struct.new(:path, :op, :expected) do
-        LINK_KEYS = %w[id type linkType].freeze
+        LINK_KEYS = %w[id type linkType].freeze # rubocop:disable Lint/ConstantDefinitionInBlock
 
         def path_tuples
           @path_tuples ||=

data/lib/wcc/contentful/store/rspec_examples/include_param.rb

@@ -15,10 +15,12 @@ RSpec.shared_examples 'supports include param' do |feature_set|
         'fields' => {
           'name' => { 'en-US' => 'root' },
           'link' => { 'en-US' => make_link_to('deep1') },
-          'links' => { 'en-US' => [
-            make_link_to('shallow3'),
-            make_link_to('deep2')
-          ] }
+          'links' => {
+            'en-US' => [
+              make_link_to('shallow3'),
+              make_link_to('deep2')
+            ]
+          }
         }
       }
     end

data/lib/wcc/contentful/store/rspec_examples/operators.rb

@@ -40,7 +40,7 @@ RSpec.shared_examples 'operators' do |feature_set|
     end
   end
 
-  supported_operators.each do |op, value|
+  supported_operators.each do |op, value| # rubocop:disable Style/CombinableLoops
     next unless value
 
     it_behaves_like "supports :#{op} operator" do

data/lib/wcc/contentful/sync_engine.rb

@@ -31,15 +31,18 @@ module WCC::Contentful
       (@state&.dup || token_wrapper_factory(nil)).freeze
     end
 
-    attr_reader :store
-    attr_reader :client
+    attr_reader :store, :client, :options
 
     def should_sync?
       store&.index?
     end
 
-    def initialize(state: nil, store: nil, client: nil, key: nil)
-      @state_key = key || "sync:#{object_id}"
+    def initialize(client: nil, store: nil, state: nil, **options)
+      @options = {
+        key: "sync:#{object_id}"
+      }.merge!(options).freeze
+
+      @state_key = @options[:key] || "sync:#{object_id}"
       @client = client || WCC::Contentful::Services.instance.client
       @mutex = Mutex.new
 
@@ -53,9 +56,7 @@ module WCC::Contentful
       if state
         @state = token_wrapper_factory(state)
         raise ArgumentError, ':state param must be a String or Hash' unless @state.is_a? Hash
-        unless @state.dig('sys', 'type') == 'token'
-          raise ArgumentError, ':state param must be of sys.type = "token"'
-        end
+        raise ArgumentError, ':state param must be of sys.type = "token"' unless @state.dig('sys', 'type') == 'token'
       end
       raise ArgumentError, 'either :state or :store must be provided' unless @state || @store
     end
@@ -75,21 +76,23 @@ module WCC::Contentful
 
       @mutex.synchronize do
         @state ||= read_state || token_wrapper_factory(nil)
-        next_sync_token = @state['token']
-
-        sync_resp = client.sync(sync_token: next_sync_token)
-        sync_resp.items.each do |item|
-          id = item.dig('sys', 'id')
-          id_found ||= id == up_to_id
-
-          store.index(item) if store&.index?
-          event = WCC::Contentful::Event.from_raw(item, source: self)
-          yield(event) if block_given?
-          emit_event(event)
-          all_events << event
-        end
+        sync_token = @state['token']
+
+        next_sync_token =
+          client.sync(sync_token: sync_token) do |item|
+            id = item.dig('sys', 'id')
+            id_found ||= id == up_to_id
+
+            store.index(item) if store&.index?
+            event = WCC::Contentful::Event.from_raw(item, source: self)
+            yield(event) if block_given?
+            emit_event(event)
+
+            # Only keep the "sys" not the content in case we have a large space
+            all_events << WCC::Contentful::Event.from_raw(item.slice('sys'), source: self)
+          end
 
-        @state = @state.merge('token' => sync_resp.next_sync_token)
+        @state = @state.merge('token' => next_sync_token)
         write_state
       end
 
@@ -135,17 +138,27 @@ module WCC::Contentful
       # This job uses the Contentful Sync API to update the configured store with
       # the latest data from Contentful.
       class Job < ActiveJob::Base
-        include WCC::Contentful::ServiceAccessors
-
         self.queue_adapter = :async
         queue_as :default
 
+        def configuration
+          @configuration ||= WCC::Contentful.configuration
+        end
+
+        def services
+          @services ||= WCC::Contentful::Services.instance
+        end
+
         def perform(event = nil)
-          return unless sync_engine&.should_sync?
+          return unless services.sync_engine&.should_sync?
 
           up_to_id = nil
-          up_to_id = event[:up_to_id] || event.dig('sys', 'id') if event
-          sync!(up_to_id: up_to_id)
+          retry_count = 0
+          if event
+            up_to_id = event[:up_to_id] || event.dig('sys', 'id')
+            retry_count = event[:retry_count] if event[:retry_count]
+          end
+          sync!(up_to_id: up_to_id, retry_count: retry_count)
         end
 
         # Calls the Contentful Sync API and updates the configured store with the returned
@@ -156,21 +169,32 @@ module WCC::Contentful
         #  If we don't find this ID in the sync data, then drop a job to try
         #  the sync again after a few minutes.
         #
-        def sync!(up_to_id: nil)
-          id_found, count = sync_engine.next(up_to_id: up_to_id)
+        def sync!(up_to_id: nil, retry_count: 0)
+          id_found, count = services.sync_engine.next(up_to_id: up_to_id)
 
-          next_sync_token = sync_engine.state['token']
+          next_sync_token = services.sync_engine.state['token']
 
           logger.info "Synced #{count} entries.  Next sync token:\n  #{next_sync_token}"
-          logger.info "Should enqueue again? [#{!id_found}]"
-          # Passing nil to only enqueue the job 1 more time
-          sync_later!(up_to_id: nil) unless id_found
+          unless id_found
+            if retry_count >= configuration.sync_retry_limit
+              logger.error "Unable to find item with id '#{up_to_id}' on the Sync API.  " \
+                           "Abandoning after #{retry_count} retries."
+            else
+              wait = (2**retry_count) * configuration.sync_retry_wait.seconds
+              logger.info "Unable to find item with id '#{up_to_id}' on the Sync API.  " \
+                          "Retrying after #{wait.inspect} " \
+                          "(#{configuration.sync_retry_limit - retry_count} retries remaining)"
+
+              self.class.set(wait: wait)
+                .perform_later(up_to_id: up_to_id, retry_count: retry_count + 1)
+            end
+          end
           next_sync_token
         end
 
-        # Drops an ActiveJob job to invoke WCC::Contentful.sync! after a given amount
+        # Enqueues an ActiveJob job to invoke WCC::Contentful.sync! after a given amount
         # of time.
-        def sync_later!(up_to_id: nil, wait: 10.minutes)
+        def sync_later!(up_to_id: nil, wait: 10.seconds)
           self.class.set(wait: wait)
             .perform_later(up_to_id: up_to_id)
         end

data/lib/wcc/contentful/sys.rb

@@ -11,7 +11,7 @@ WCC::Contentful::Sys =
     :revision,
     :context
   ) do
-
+    # rubocop:disable Lint/ConstantDefinitionInBlock
     ATTRIBUTES = %i[
       id
       type
@@ -22,6 +22,7 @@ WCC::Contentful::Sys =
       revision
       context
     ].freeze
+    # rubocop:enable Lint/ConstantDefinitionInBlock
 
     undef []=
     ATTRIBUTES.each { |a| __send__(:undef_method, "#{a}=") }

data/lib/wcc/contentful/test/double.rb

@@ -8,9 +8,7 @@ module WCC::Contentful::Test::Double
   # All attributes that are known to be required fields on the content type
   # will return a default value based on the field type.
   def contentful_double(const, **attrs)
-    unless const.respond_to?(:content_type_definition)
-      const = WCC::Contentful::Model.resolve_constant(const.to_s)
-    end
+    const = WCC::Contentful::Model.resolve_constant(const.to_s) unless const.respond_to?(:content_type_definition)
     attrs.symbolize_keys!
 
     bad_attrs = attrs.reject { |a| const.instance_methods.include?(a) }
@@ -45,7 +43,7 @@ module WCC::Contentful::Test::Double
           sys: {
             type: 'Link',
             linkType: 'Space',
-            id: ENV['CONTENTFUL_SPACE_ID']
+            id: ENV.fetch('CONTENTFUL_SPACE_ID', nil)
           }
         },
         id: SecureRandom.urlsafe_base64,
@@ -62,7 +60,7 @@ module WCC::Contentful::Test::Double
         revision: rand(100),
         locale: 'en-US'
       },
-      fields: attrs.each_with_object({}) { |(k, v), h| h[k] = { 'en-US' => v } }
+      fields: attrs.transform_values { |v| { 'en-US' => v } }
     }
 
     double(attrs)

data/lib/wcc/contentful/test/factory.rb

@@ -8,9 +8,7 @@ module WCC::Contentful::Test::Factory
   # All attributes that are known to be required fields on the content type
   # will return a default value based on the field type.
   def contentful_create(const, context = nil, **attrs)
-    unless const.respond_to?(:content_type_definition)
-      const = WCC::Contentful::Model.resolve_constant(const.to_s)
-    end
+    const = WCC::Contentful::Model.resolve_constant(const.to_s) unless const.respond_to?(:content_type_definition)
     attrs = attrs.transform_keys { |a| a.to_s.camelize(:lower) }
 
     id = attrs.delete('id')
@@ -59,7 +57,7 @@ module WCC::Contentful::Test::Factory
         sys: {
           type: 'Link',
           linkType: 'Space',
-          id: ENV['CONTENTFUL_SPACE_ID']
+          id: ENV.fetch('CONTENTFUL_SPACE_ID', nil)
         }
       },
       id: id || SecureRandom.urlsafe_base64,
@@ -86,8 +84,8 @@ module WCC::Contentful::Test::Factory
   end
 
   def contentful_fields(model)
-    WCC::Contentful::Test::Attributes.defaults(model).each_with_object({}) do |(k, v), h|
-      h[k] = { 'en-US' => v }
+    WCC::Contentful::Test::Attributes.defaults(model).transform_values do |v|
+      { 'en-US' => v }
     end
   end
 

data/lib/wcc/contentful/version.rb

@@ -2,6 +2,6 @@
 
 module WCC
   module Contentful
-    VERSION = '1.1.2'
+    VERSION = '1.3.0'
   end
 end

data/lib/wcc/contentful.rb

@@ -3,7 +3,7 @@
 require 'wcc/contentful/version'
 
 require 'active_support'
-require 'active_support/core_ext/object'
+require 'active_support/all'
 
 require 'wcc/contentful/active_record_shim'
 require 'wcc/contentful/configuration'
@@ -46,9 +46,8 @@ module WCC::Contentful
     end
 
     def logger
-      return Rails.logger if defined?(Rails)
-
-      @logger ||= Logger.new(STDERR)
+      ActiveSupport::Deprecation.warn('Use WCC::Contentful::Services.instance.logger instead')
+      WCC::Contentful::Services.instance.logger
     end
   end
 
@@ -82,7 +81,7 @@ module WCC::Contentful
 
     if configuration.update_schema_file == :always ||
         (configuration.update_schema_file == :if_possible && Services.instance.management_client) ||
-        configuration.update_schema_file == :if_missing && !File.exist?(configuration.schema_file)
+        (configuration.update_schema_file == :if_missing && !File.exist?(configuration.schema_file))
 
       begin
         downloader = WCC::Contentful::DownloadsSchema.new
@@ -90,17 +89,15 @@ module WCC::Contentful
       rescue WCC::Contentful::SimpleClient::ApiError => e
         raise InitializationError, e if configuration.update_schema_file == :always
 
-        WCC::Contentful.logger.warn("Unable to download schema from management API - #{e.message}")
+        Services.instance.logger.warn("Unable to download schema from management API - #{e.message}")
       end
     end
 
     content_types =
       begin
-        if File.exist?(configuration.schema_file)
-          JSON.parse(File.read(configuration.schema_file))['contentTypes']
-        end
+        JSON.parse(File.read(configuration.schema_file))['contentTypes'] if File.exist?(configuration.schema_file)
       rescue JSON::ParserError
-        WCC::Contentful.logger.warn("Schema file invalid, ignoring it: #{configuration.schema_file}")
+        Services.instance.warn("Schema file invalid, ignoring it: #{configuration.schema_file}")
         nil
       end
 
@@ -114,13 +111,13 @@ module WCC::Contentful
         content_types = client.content_types(limit: 1000).items if client
       rescue WCC::Contentful::SimpleClient::ApiError => e
         # indicates bad credentials
-        WCC::Contentful.logger.warn("Unable to load content types from API - #{e.message}")
+        Services.instance.logger.warn("Unable to load content types from API - #{e.message}")
       end
     end
 
     unless content_types
-      raise InitializationError, 'Unable to load content types from schema file or API!' \
-        ' Check your access token and space ID.'
+      raise InitializationError, 'Unable to load content types from schema file or API! ' \
+                                 'Check your access token and space ID.'
     end
 
     # Set the schema on the default WCC::Contentful::Model