wcc-contentful 0.4.0.pre.alpha → 1.0.0.pre.rc3

data/lib/wcc/contentful/model_builder.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative './link'
 require_relative './sys'
 
 module WCC::Contentful

data/lib/wcc/contentful/model_methods.rb

@@ -5,6 +5,15 @@
 #
 # @api Model
 module WCC::Contentful::ModelMethods
+  include WCC::Contentful::Instrumentation
+
+  # The set of options keys that are specific to the Model layer and shouldn't
+  # be passed down to the Store layer.
+  MODEL_LAYER_CONTEXT_KEYS = %i[
+    preview
+    backlinks
+  ].freeze
+
   # Resolves all links in an entry to the specified depth.
   #
   # Each link in the entry is recursively retrieved from the store until the given
@@ -22,7 +31,7 @@ module WCC::Contentful::ModelMethods
   #   handled.  `:raise` causes a {WCC::Contentful::CircularReferenceError} to be raised,
   #   `:ignore` will cause the field to remain unresolved, and any other value (or nil)
   #   will cause the field to point to the previously resolved ruby object for that ID.
-  def resolve(depth: 1, fields: nil, context: {}, **options)
+  def resolve(depth: 1, fields: nil, context: sys.context.to_h, **options)
     raise ArgumentError, "Depth must be > 0 (was #{depth})" unless depth && depth > 0
     return self if resolved?(depth: depth, fields: fields)
 
@@ -32,20 +41,26 @@ module WCC::Contentful::ModelMethods
     typedef = self.class.content_type_definition
     links = fields.select { |f| %i[Asset Link].include?(typedef.fields[f].type) }
 
-    raw_links =
-      links.any? do |field_name|
-        raw_value = raw.dig('fields', field_name, sys.locale)
-        if raw_value&.is_a? Array
-          raw_value.any? { |v| v&.dig('sys', 'type') == 'Link' }
-        elsif raw_value
-          raw_value.dig('sys', 'type') == 'Link'
+    raw_link_ids =
+      links.map { |field_name| raw.dig('fields', field_name, sys.locale) }
+        .flat_map do |raw_value|
+          _try_map(raw_value) { |v| v.dig('sys', 'id') if v.dig('sys', 'type') == 'Link' }
+        end
+    raw_link_ids = raw_link_ids.compact
+    backlinked_ids = (context[:backlinks]&.map { |m| m.id } || [])
+
+    has_unresolved_raw_links = (raw_link_ids - backlinked_ids).any?
+    if has_unresolved_raw_links
+      raw =
+        _instrument 'resolve', id: id, depth: depth, backlinks: backlinked_ids do
+          # use include param to do resolution
+          self.class.store(context[:preview])
+            .find_by(content_type: self.class.content_type,
+                     filter: { 'sys.id' => id },
+                     options: context.except(*MODEL_LAYER_CONTEXT_KEYS).merge!({
+                       include: [depth, 10].min
+                     }))
         end
-      end
-    if raw_links
-      # use include param to do resolution
-      raw = self.class.store.find_by(content_type: self.class.content_type,
-                                     filter: { 'sys.id' => id },
-                                     options: { include: [depth, 10].min })
       unless raw
         raise WCC::Contentful::ResolveError, "Cannot find #{self.class.content_type} with ID #{id}"
       end
@@ -118,6 +133,12 @@ module WCC::Contentful::ModelMethods
 
   delegate :to_json, to: :to_h
 
+  protected
+
+  def _instrumentation_event_prefix
+    '.model.contentful.wcc'
+  end
+
   private
 
   def _resolve_field(field_name, depth = 1, context = {}, options = {})
@@ -147,7 +168,10 @@ module WCC::Contentful::ModelMethods
         # instantiate from already resolved raw entry data.
         m = already_resolved ||
           if raw.dig('sys', 'type') == 'Link'
-            WCC::Contentful::Model.find(id, new_context)
+            _instrument 'resolve',
+              id: self.id, depth: depth, backlinks: context[:backlinks]&.map(&:id) do
+              WCC::Contentful::Model.find(id, options: new_context)
+            end
           else
             WCC::Contentful::Model.new_from_raw(raw, new_context)
           end
@@ -158,6 +182,7 @@ module WCC::Contentful::ModelMethods
 
     begin
       val = _try_map(val) { |v| load.call(v) }
+      val = val.compact if val.is_a? Array
 
       instance_variable_set(var_name + '_resolved', val)
     rescue WCC::Contentful::CircularReferenceError

data/lib/wcc/contentful/model_singleton_methods.rb

@@ -4,18 +4,6 @@
 # methods that are not dynamically generated.
 # @api Model
 module WCC::Contentful::ModelSingletonMethods
-  def store(preview = false)
-    if preview
-      if WCC::Contentful::Model.preview_store.nil?
-        raise ArgumentError,
-          'You must include a contentful preview token in your WCC::Contentful.configure block'
-      end
-      WCC::Contentful::Model.preview_store
-    else
-      WCC::Contentful::Model.store
-    end
-  end
-
   # Finds an instance of this content type.
   #
   # @return [nil, WCC::Contentful::Model] An instance of the appropriate model class
@@ -24,8 +12,12 @@ module WCC::Contentful::ModelSingletonMethods
   #   WCC::Contentful::Model::Page.find(id)
   def find(id, options: nil)
     options ||= {}
-    raw = store(options[:preview])
-      .find(id, { hint: type }.merge!(options.except(:preview)))
+    store = store(options[:preview])
+    raw =
+      WCC::Contentful::Instrumentation.instrument 'find.model.contentful.wcc',
+        content_type: content_type, id: id, options: options do
+        store.find(id, { hint: type }.merge!(options.except(:preview)))
+      end
     new(raw, options) if raw.present?
   end
 
@@ -40,16 +32,16 @@ module WCC::Contentful::ModelSingletonMethods
     filter = filter&.dup
     options = filter&.delete(:options) || {}
 
-    if filter.present?
-      filter.transform_keys! { |k| k.to_s.camelize(:lower) }
-      bad_fields = filter.keys.reject { |k| self::FIELDS.include?(k) }
-      raise ArgumentError, "These fields do not exist: #{bad_fields}" unless bad_fields.empty?
-    end
+    filter.transform_keys! { |k| k.to_s.camelize(:lower) } if filter.present?
 
-    query = store(options[:preview])
-      .find_all(content_type: content_type, options: options.except(:preview))
+    store = store(options[:preview])
+    query =
+      WCC::Contentful::Instrumentation.instrument 'find_all.model.contentful.wcc',
+        content_type: content_type, filter: filter, options: options do
+        store.find_all(content_type: content_type, options: options.except(:preview))
+      end
     query = query.apply(filter) if filter.present?
-    query.map { |r| new(r, options) }
+    ModelQuery.new(query, options, self)
   end
 
   # Finds the first instance of this content type matching the given query.
@@ -62,22 +54,47 @@ module WCC::Contentful::ModelSingletonMethods
     filter = filter&.dup
     options = filter&.delete(:options) || {}
 
-    if filter.present?
-      filter.transform_keys! { |k| k.to_s.camelize(:lower) }
-      bad_fields = filter.keys.reject { |k| self::FIELDS.include?(k) }
-      raise ArgumentError, "These fields do not exist: #{bad_fields}" unless bad_fields.empty?
-    end
+    filter.transform_keys! { |k| k.to_s.camelize(:lower) } if filter.present?
 
-    result = store(options[:preview])
-      .find_by(content_type: content_type, filter: filter, options: options.except(:preview))
+    store = store(options[:preview])
+    result =
+      WCC::Contentful::Instrumentation.instrument 'find_by.model.contentful.wcc',
+        content_type: content_type, filter: filter, options: options do
+        store.find_by(content_type: content_type, filter: filter, options: options.except(:preview))
+      end
 
     new(result, options) if result
   end
 
   def inherited(subclass)
-    # only register if it's not already registered
+    # If another different class is already registered for this content type,
+    # don't auto-register this one.
     return if WCC::Contentful::Model.registered?(content_type)
 
     WCC::Contentful::Model.register_for_content_type(content_type, klass: subclass)
   end
+
+  class ModelQuery
+    include Enumerable
+
+    # by default all enumerable methods delegated to the to_enum method
+    delegate(*(Enumerable.instance_methods - Module.instance_methods), to: :to_enum)
+    delegate :each, to: :to_enum
+
+    # except count - because that needs to pull data off the final query obj
+    delegate :count, to: :wrapped_query
+
+    attr_reader :wrapped_query, :options, :klass
+
+    def initialize(wrapped_query, options, klass)
+      @wrapped_query = wrapped_query
+      @options = options
+      @klass = klass
+    end
+
+    def to_enum
+      wrapped_query.to_enum
+        .map { |r| klass.new(r, options) }
+    end
+  end
 end

data/lib/wcc/contentful/rake.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+Dir[File.join(File.dirname(__FILE__), '../../tasks/**/*.rake')]
+  .each { |f| load f }

data/lib/wcc/contentful/rspec.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'wcc/contentful'
+
+require_relative './test'
+
+module WCC::Contentful::RSpec
+  include WCC::Contentful::Test::Double
+  include WCC::Contentful::Test::Factory
+
+  ##
+  # Builds out a fake Contentful entry for the given content type, and then
+  # stubs the Model API to return that content type for `.find` and `.find_by`
+  # query methods.
+  def contentful_stub(content_type, **attrs)
+    const = WCC::Contentful::Model.resolve_constant(content_type.to_s)
+    instance = contentful_create(content_type, **attrs)
+
+    # mimic what's going on inside model_singleton_methods.rb
+    # find, find_by, etc always return a new instance from the same raw
+    allow(WCC::Contentful::Model).to receive(:find)
+      .with(instance.id, any_args) do |_id, keyword_params|
+        options = keyword_params && keyword_params[:options]
+        contentful_create(content_type, options, raw: instance.raw, **attrs)
+      end
+    allow(const).to receive(:find) { |id, options| WCC::Contentful::Model.find(id, **(options || {})) }
+
+    attrs.each do |k, v|
+      allow(const).to receive(:find_by)
+        .with(hash_including(k => v)) do |filter|
+          filter = filter&.dup
+          options = filter&.delete(:options) || {}
+
+          contentful_create(content_type, options, raw: instance.raw, **attrs)
+        end
+    end
+
+    instance
+  end
+end
+
+if defined?(RSpec)
+  RSpec.configure do |config|
+    config.include WCC::Contentful::RSpec
+  end
+end

data/lib/wcc/contentful/services.rb

@@ -1,21 +1,31 @@
 # frozen_string_literal: true
 
-require 'singleton'
-
 module WCC::Contentful
   class Services
-    include Singleton
+    class << self
+      def instance
+        @singleton__instance__ ||= new # rubocop:disable Naming/MemoizedInstanceVariableName
+      end
+    end
+
+    def configuration
+      @configuration ||= WCC::Contentful.configuration
+    end
+
+    def initialize(configuration = nil)
+      @configuration = configuration
+    end
 
     # Gets the data-store which executes the queries run against the dynamic
     # models in the WCC::Contentful::Model namespace.
-    # This is one of the following based on the configured content_delivery method:
+    # This is one of the following based on the configured store method:
     #
     # [:direct] an instance of {WCC::Contentful::Store::CDNAdapter} with a
     #           {WCC::Contentful::SimpleClient::Cdn CDN Client} to access the CDN.
     #
-    # [:lazy_sync] an instance of {WCC::Contentful::Store::LazyCacheStore}
-    #              with the configured ActiveSupport::Cache implementation and a
-    #              {WCC::Contentful::SimpleClient::Cdn CDN Client} for when data
+    # [:lazy_sync] an instance of {WCC::Contentful::Middleware::Store::CachingMiddleware}
+    #              with the configured ActiveSupport::Cache implementation around a
+    #              {WCC::Contentful::Store::CDNAdapter} for when data
     #              cannot be found in the cache.
     #
     # [:eager_sync] an instance of the configured Store type, defined by
@@ -25,12 +35,7 @@ module WCC::Contentful
     def store
       @store ||=
         ensure_configured do |config|
-          WCC::Contentful::Store::Factory.new(
-            config,
-            self,
-            config.content_delivery,
-            config.content_delivery_params
-          ).build_sync_store
+          config.store.build(self)
         end
     end
 
@@ -43,10 +48,9 @@ module WCC::Contentful
         ensure_configured do |config|
           WCC::Contentful::Store::Factory.new(
             config,
-            self,
             :direct,
-            [{ preview: true }]
-          ).build_sync_store
+            :preview
+          ).build(self)
         end
     end
 
@@ -58,10 +62,11 @@ module WCC::Contentful
       @client ||=
         ensure_configured do |config|
           WCC::Contentful::SimpleClient::Cdn.new(
+            **config.connection_options,
             access_token: config.access_token,
             space: config.space,
             default_locale: config.default_locale,
-            adapter: config.http_adapter,
+            connection: config.connection,
             environment: config.environment
           )
         end
@@ -76,10 +81,11 @@ module WCC::Contentful
         ensure_configured do |config|
           if config.preview_token.present?
             WCC::Contentful::SimpleClient::Preview.new(
+              **config.connection_options,
               preview_token: config.preview_token,
               space: config.space,
               default_locale: config.default_locale,
-              adapter: config.http_adapter,
+              connection: config.connection,
               environment: config.environment
             )
           end
@@ -95,27 +101,57 @@ module WCC::Contentful
         ensure_configured do |config|
           if config.management_token.present?
             WCC::Contentful::SimpleClient::Management.new(
+              **config.connection_options,
               management_token: config.management_token,
               space: config.space,
               default_locale: config.default_locale,
-              adapter: config.http_adapter,
+              connection: config.connection,
               environment: config.environment
             )
           end
         end
     end
 
+    # Gets the configured WCC::Contentful::SyncEngine which is responsible for
+    # updating the currently configured store.  The application must periodically
+    # call #next on this instance.  Alternately, the application can mount the
+    # WCC::Contentful::Engine, which will call #next anytime a webhook is received.
+    #
+    # This returns `nil` if the currently configured store does not respond to sync
+    # events.
+    def sync_engine
+      @sync_engine ||=
+        if store.index?
+          SyncEngine.new(
+            store: store,
+            client: client,
+            key: 'sync:token'
+          )
+        end
+    end
+
+    # Gets the configured instrumentation adapter, defaulting to ActiveSupport::Notifications
+    def instrumentation
+      return @instrumentation if @instrumentation
+      return ActiveSupport::Notifications if WCC::Contentful.configuration.nil?
+
+      @instrumentation ||=
+        WCC::Contentful.configuration.instrumentation_adapter ||
+        ActiveSupport::Notifications
+    end
+
     private
 
     def ensure_configured
-      if WCC::Contentful.configuration.nil?
-        raise StandardError, 'WCC::Contentful has not yet been configured!'
-      end
+      raise StandardError, 'WCC::Contentful has not yet been configured!' if configuration.nil?
 
-      yield WCC::Contentful.configuration
+      yield configuration
     end
   end
 
+  SERVICES = (WCC::Contentful::Services.instance_methods -
+      Object.instance_methods)
+
   # Include this module to define accessors for every method defined on the
   # {Services} singleton.
   #
@@ -129,14 +165,12 @@ module WCC::Contentful
   #       store.find(...)
   #
   #       client.entries(...)
+  #
+  #       sync_engine.next
   #     end
   #   end
   # @see Services
   module ServiceAccessors
-    SERVICES = (WCC::Contentful::Services.instance_methods -
-      Object.instance_methods -
-      Singleton.instance_methods)
-
     SERVICES.each do |m|
       define_method m do
         Services.instance.public_send(m)

data/lib/wcc/contentful/simple_client.rb

@@ -2,6 +2,7 @@
 
 require_relative 'simple_client/response'
 require_relative 'simple_client/management'
+require_relative 'instrumentation'
 
 module WCC::Contentful
   # The SimpleClient accesses the Contentful CDN to get JSON responses,
@@ -16,11 +17,13 @@ module WCC::Contentful
   # `get`.  This method returns a WCC::Contentful::SimpleClient::Response
   # that handles paging automatically.
   #
-  # The SimpleClient by default uses 'http' to perform the gets, but any HTTP
-  # client can be injected by passing a proc as the `adapter:` option.
+  # The SimpleClient by default uses 'faraday' to perform the gets, but any HTTP
+  # client adapter be injected by passing the `connection:` option.
   #
   # @api Client
   class SimpleClient
+    include WCC::Contentful::Instrumentation
+
     attr_reader :api_url
     attr_reader :space
 
@@ -30,21 +33,27 @@ module WCC::Contentful
     # @param [String] space The Space ID to access
     # @param [String] access_token A Contentful Access Token to be sent in the Authorization header
     # @param [Hash] options The remaining optional parameters, defined below
-    # @option options [Symbol, Object] adapter The Adapter to use to make requests.
+    # @option options [Symbol, Object] connection The Faraday connection to use to make requests.
     #   Auto-discovered based on what gems are installed if this is not provided.
     # @option options [String] default_locale The locale query param to set by default.
     # @option options [String] environment The contentful environment to access. Defaults to 'master'.
     # @option options [Boolean] no_follow_redirects If true, do not follow 300 level redirects.
+    # @option options [Number] rate_limit_wait_timeout The maximum time to block the thread waiting
+    #   on a rate limit response.  By default will wait for one 429 and then fail on the second 429.
     def initialize(api_url:, space:, access_token:, **options)
       @api_url = URI.join(api_url, '/spaces/', space + '/')
       @space = space
       @access_token = access_token
 
-      @adapter = SimpleClient.load_adapter(options[:adapter])
+      @adapter = SimpleClient.load_adapter(options[:connection])
 
       @options = options
+      @_instrumentation = @options[:instrumentation]
       @query_defaults = {}
       @query_defaults[:locale] = @options[:default_locale] if @options[:default_locale]
+      # default 1.5 so that we retry one time then fail if still rate limited
+      # https://www.contentful.com/developers/docs/references/content-preview-api/#/introduction/api-rate-limits
+      @rate_limit_wait_timeout = @options[:rate_limit_wait_timeout] || 1.5
 
       return unless options[:environment].present?
 
@@ -57,13 +66,17 @@ module WCC::Contentful
     def get(path, query = {})
       url = URI.join(@api_url, path)
 
+      resp =
+        _instrument 'get_http', url: url, query: query do
+          get_http(url, query)
+        end
       Response.new(self,
         { url: url, query: query },
-        get_http(url, query))
+        resp)
     end
 
     ADAPTERS = {
-      http: ['http', '> 1.0', '< 3.0'],
+      faraday: ['faraday', '>= 0.9'],
       typhoeus: ['typhoeus', '~> 1.0']
     }.freeze
 
@@ -80,16 +93,19 @@ module WCC::Contentful
         end
         raise ArgumentError, 'Unable to load adapter!  Please install one of '\
           "#{ADAPTERS.values.map(&:join).join(',')}"
-      when :http
-        require_relative 'simple_client/http_adapter'
-        HttpAdapter.new
+      when :faraday
+        require 'faraday'
+        ::Faraday.new do |faraday|
+          faraday.response :logger, (Rails.logger if defined?(Rails)), { headers: false, bodies: false }
+          faraday.adapter :net_http
+        end
       when :typhoeus
         require_relative 'simple_client/typhoeus_adapter'
         TyphoeusAdapter.new
       else
-        unless adapter.respond_to?(:call)
+        unless adapter.respond_to?(:get)
           raise ArgumentError, "Adapter #{adapter} is not invokeable!  Please "\
-            "pass a proc or use one of #{ADAPTERS.keys}"
+            "pass use one of #{ADAPTERS.keys} or create a Faraday-compatible adapter"
         end
         adapter
       end
@@ -97,7 +113,12 @@ module WCC::Contentful
 
     private
 
-    def get_http(url, query, headers = {}, proxy = {})
+    def _instrumentation_event_prefix
+      # Unify all CDN, Management, Preview notifications under same namespace
+      '.simpleclient.contentful.wcc'
+    end
+
+    def get_http(url, query, headers = {})
       headers = {
         Authorization: "Bearer #{@access_token}"
       }.merge(headers || {})
@@ -105,12 +126,28 @@ module WCC::Contentful
       q = @query_defaults.dup
       q = q.merge(query) if query
 
-      resp = @adapter.call(url, q, headers, proxy)
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      loop do
+        resp = @adapter.get(url, q, headers)
+
+        if [301, 302, 307].include?(resp.status) && !@options[:no_follow_redirects]
+          url = resp.headers['Location']
+          next
+        end
+
+        if resp.status == 429 &&
+            reset = resp.headers['X-Contentful-RateLimit-Reset'].presence
+          reset = reset.to_f
+          _instrument 'rate_limit', start: start, reset: reset, timeout: @rate_limit_wait_timeout
+          now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          if (now - start) + reset < @rate_limit_wait_timeout
+            sleep(reset)
+            next
+          end
+        end
 
-      if [301, 302, 307].include?(resp.code) && !@options[:no_follow_redirects]
-        resp = get_http(resp.headers['location'], nil, headers, proxy)
+        return resp
       end
-      resp
     end
 
     # The CDN SimpleClient accesses 'https://cdn.contentful.com' to get raw
@@ -135,31 +172,46 @@ module WCC::Contentful
 
       # Gets an entry by ID
       def entry(key, query = {})
-        resp = get("entries/#{key}", query)
+        resp =
+          _instrument 'entries', id: key, type: 'Entry', query: query do
+            get("entries/#{key}", query)
+          end
         resp.assert_ok!
       end
 
       # Queries entries with optional query parameters
       def entries(query = {})
-        resp = get('entries', query)
+        resp =
+          _instrument 'entries', type: 'Entry', query: query do
+            get('entries', query)
+          end
         resp.assert_ok!
       end
 
       # Gets an asset by ID
       def asset(key, query = {})
-        resp = get("assets/#{key}", query)
+        resp =
+          _instrument 'entries', type: 'Asset', id: key, query: query do
+            get("assets/#{key}", query)
+          end
         resp.assert_ok!
       end
 
       # Queries assets with optional query parameters
       def assets(query = {})
-        resp = get('assets', query)
+        resp =
+          _instrument 'entries', type: 'Asset', query: query do
+            get('assets', query)
+          end
         resp.assert_ok!
       end
 
       # Queries content types with optional query parameters
       def content_types(query = {})
-        resp = get('content_types', query)
+        resp =
+          _instrument 'content_types', query: query do
+            get('content_types', query)
+          end
         resp.assert_ok!
       end
 
@@ -177,7 +229,11 @@ module WCC::Contentful
             { initial: true }
           end
         query = query.merge(sync_token)
-        resp = SyncResponse.new(get('sync', query))
+        resp =
+          _instrument 'sync', sync_token: sync_token, query: query do
+            get('sync', query)
+          end
+        resp = SyncResponse.new(resp)
         resp.assert_ok!
       end
     end
@@ -186,10 +242,10 @@ module WCC::Contentful
     class Preview < Cdn
       def initialize(space:, preview_token:, **options)
         super(
-          api_url: options[:api_url] || 'https://preview.contentful.com/',
+          **options,
+          api_url: options[:preview_api_url] || 'https://preview.contentful.com/',
           space: space,
-          access_token: preview_token,
-          **options
+          access_token: preview_token
         )
       end