wcc-contentful 0.2.2 → 0.3.0.pre.rc

data/lib/wcc/contentful/simple_client/response.rb

@@ -22,6 +22,24 @@ class WCC::Contentful::SimpleClient
       raw.dig('message') || "#{code}: #{raw_response.message}"
     end
 
+    def next_page?
+      return unless raw.key? 'items'
+
+      raw['items'].length + raw['skip'] < raw['total']
+    end
+
+    def next_page
+      return unless next_page?
+
+      @next_page ||= @client.get(
+        @request[:url],
+        (@request[:query] || {}).merge({
+          skip: raw['items'].length + raw['skip']
+        })
+      )
+      @next_page.assert_ok!
+    end
+
     def initialize(client, request, raw_response)
       @client = client
       @request = request
@@ -31,33 +49,20 @@ class WCC::Contentful::SimpleClient
 
     def assert_ok!
       return self if code >= 200 && code < 300
+
       raise ApiError[code], self
     end
 
     def each_page(&block)
       raise ArgumentError, 'Not a collection response' unless raw['items']
 
-      memoized_pages = (@memoized_pages ||= [self])
       ret =
         Enumerator.new do |y|
-          page_index = 0
-          current_page = self
-          loop do
-            y << current_page
-
-            skip_amt = current_page.raw['items'].length + current_page.raw['skip']
-            break if current_page.raw['items'].empty? || skip_amt >= current_page.raw['total']
-
-            page_index += 1
-            if page_index < memoized_pages.length
-              current_page = memoized_pages[page_index]
-            else
-              current_page = @client.get(
-                @request[:url],
-                (@request[:query] || {}).merge({ skip: skip_amt })
-              )
-              current_page.assert_ok!
-              memoized_pages.push(current_page)
+          y << self
+
+          if next_page?
+            next_page.each_page.each do |page|
+              y << page
             end
           end
         end
@@ -81,8 +86,21 @@ class WCC::Contentful::SimpleClient
 
     def first
       raise ArgumentError, 'Not a collection response' unless raw['items']
+
       raw['items'].first
     end
+
+    def includes
+      @includes ||=
+        raw.dig('includes')&.each_with_object({}) do |(_t, entries), h|
+          entries.each { |e| h[e.dig('sys', 'id')] = e }
+        end || {}
+
+      return @includes unless @next_page
+
+      # This could be more efficient - maybe not worth worrying about
+      @includes.merge(@next_page.includes)
+    end
   end
 
   class SyncResponse < Response
@@ -90,31 +108,36 @@ class WCC::Contentful::SimpleClient
       super(response.client, response.request, response.raw_response)
     end
 
+    def next_page?
+      raw['nextPageUrl'].present?
+    end
+
+    def next_page
+      return unless next_page?
+
+      @next_page ||= SyncResponse.new(@client.get(raw['nextPageUrl']))
+      @next_page.assert_ok!
+    end
+
     def next_sync_token
-      @next_sync_token ||= SyncResponse.parse_sync_token(raw['nextSyncUrl'])
+      # If we haven't grabbed the next page yet, then our next "sync" will be getting
+      # the next page.  We could just as easily call sync again with that token.
+      @next_page&.next_sync_token ||
+        @next_sync_token ||= SyncResponse.parse_sync_token(
+          raw['nextPageUrl'] || raw['nextSyncUrl']
+        )
     end
 
     def each_page
       raise ArgumentError, 'Not a collection response' unless raw['items']
 
-      memoized_pages = (@memoized_pages ||= [self])
       ret =
         Enumerator.new do |y|
-          page_index = 0
-          current_page = self
-          loop do
-            y << current_page
-
-            break if current_page.raw['items'].empty?
-
-            page_index += 1
-            if page_index < memoized_pages.length
-              current_page = memoized_pages[page_index]
-            else
-              current_page = @client.get(raw['nextSyncUrl'])
-              current_page.assert_ok!
-              @next_sync_token = SyncResponse.parse_sync_token(current_page.raw['nextSyncUrl'])
-              memoized_pages.push(current_page)
+          y << self
+
+          if next_page?
+            next_page.each_page.each do |page|
+              y << page
             end
           end
         end
@@ -127,7 +150,8 @@ class WCC::Contentful::SimpleClient
     end
 
     def count
-      raw['items'].length
+      raise NotImplementedError,
+        'Sync does not return an accurate total.  Use #items.count instead.'
     end
 
     def self.parse_sync_token(url)

data/lib/wcc/contentful/simple_client/typhoeus_adapter.rb

@@ -16,6 +16,18 @@ class TyphoeusAdapter
     )
   end
 
+  def post(url, body, headers = {}, proxy = {})
+    raise NotImplementedError, 'Proxying Not Yet Implemented' if proxy[:host]
+
+    TyphoeusAdapter::Response.new(
+      Typhoeus.post(
+        url,
+        body: body.to_json,
+        headers: headers
+      )
+    )
+  end
+
   Response =
     Struct.new(:raw) do
       delegate :body, to: :raw

data/lib/wcc/contentful/store.rb

@@ -1,20 +1,38 @@
-
 # frozen_string_literal: true
 
 require_relative 'store/base'
 require_relative 'store/memory_store'
-require_relative 'store/lazy_cache_store'
 require_relative 'store/cdn_adapter'
+require_relative 'store/lazy_cache_store'
 
-# required dynamically if they select the 'postgres' store option
-# require_relative 'store/postgres_store'
-
+# The "Store" is the middle layer in the WCC::Contentful gem.  It exposes an API
+# that implements the configured content delivery strategy.
+#
+# The different content delivery strategies require different store implementations.
+#
+# direct:: Uses the WCC::Contentful::Store::CDNAdapter to wrap the Contentful CDN,
+#          providing an API consistent with the other stores.  Any query made to
+#          the CDNAdapter will be immediately passed through to the API.
+#          The CDNAdapter does not implement #index because it does not care about
+#          updates coming from the Sync API.
+#
+# lazy_sync:: Uses the Contentful CDN in combination with an ActiveSupport::Cache
+#             implementation in order to respond with the cached data where possible,
+#             saving your CDN quota.  The cache is kept up-to-date via the Sync API
+#             and the WCC::Contentful::DelayedSyncJob.  It is correct, but not complete.
+#
+# eager_sync:: Uses one of the full store implementations to store the entirety
+#              of the Contentful space locally.  All queries are run against this
+#              local copy, which is kept up to date via the Sync API and the
+#              WCC::Contentful::DelayedSyncJob.  The local store is correct and complete.
+#
+# The currently configured store is available on WCC::Contentful::Services.instance.store
 module WCC::Contentful::Store
   SYNC_STORES = {
     memory: ->(_config) { WCC::Contentful::Store::MemoryStore.new },
-    postgres: ->(_config) {
+    postgres: ->(config, *options) {
       require_relative 'store/postgres_store'
-      WCC::Contentful::Store::PostgresStore.new(ENV['POSTGRES_CONNECTION'])
+      WCC::Contentful::Store::PostgresStore.new(config, *options)
     }
   }.freeze
 
@@ -22,10 +40,11 @@ module WCC::Contentful::Store
     eager_sync
     lazy_sync
     direct
+    custom
   ].freeze
 
   Factory =
-    Struct.new(:config, :cdn_method, :content_delivery_params) do
+    Struct.new(:config, :services, :cdn_method, :content_delivery_params) do
       def build_sync_store
         unless respond_to?("build_#{cdn_method}")
           raise ArgumentError, "Don't know how to build content delivery method #{cdn_method}"
@@ -36,38 +55,46 @@ module WCC::Contentful::Store
 
       def validate!
         unless CDN_METHODS.include?(cdn_method)
-          raise ArgumentError, "Please use one of #{CDN_METHODS} for 'content_delivery'"
+          raise ArgumentError, "Please use one of #{CDN_METHODS} instead of #{cdn_method}"
         end
 
         return unless respond_to?("validate_#{cdn_method}")
+
         public_send("validate_#{cdn_method}", config, *content_delivery_params)
       end
 
-      def build_eager_sync(config, store = nil, *_options)
-        puts "store: #{store}"
-        store = SYNC_STORES[store].call(config) if store.is_a?(Symbol)
+      def build_eager_sync(config, store = nil, *options)
+        store = SYNC_STORES[store].call(config, *options) if store.is_a?(Symbol)
         store || MemoryStore.new
       end
 
-      def build_lazy_sync(config, *options)
+      def build_lazy_sync(_config, *options)
         WCC::Contentful::Store::LazyCacheStore.new(
-          config.client,
+          services.client,
           cache: ActiveSupport::Cache.lookup_store(*options)
         )
       end
 
-      def build_direct(config, *options)
+      def build_direct(_config, *options)
         if options.find { |array| array[:preview] == true }
-          CDNAdapter.new(config.preview_client)
+          CDNAdapter.new(services.preview_client)
         else
-          CDNAdapter.new(config.client)
+          CDNAdapter.new(services.client)
         end
       end
 
+      def build_custom(config, *options)
+        store = config.store
+        return store unless store&.respond_to?(:new)
+
+        store.new(config, options)
+      end
+
       def validate_eager_sync(_config, store = nil, *_options)
         return unless store.is_a?(Symbol)
 
-        return if SYNC_STORES.keys.include?(store)
+        return if SYNC_STORES.key?(store)
+
         raise ArgumentError, "Please use one of #{SYNC_STORES.keys}"
       end
     end

data/lib/wcc/contentful/store/base.rb

@@ -1,20 +1,37 @@
-
 # frozen_string_literal: true
 
+# @api Store
 module WCC::Contentful::Store
+  # This is the base class for stores which implement #index, and therefore
+  # must be kept up-to-date via the Sync API.
+  # @abstract At a minimum subclasses should override {#find}, {#find_all}, {#set},
+  #   and #{delete}. As an alternative to overriding set and delete, the subclass
+  #   can override {#index}.  Index is called when a webhook triggers a sync, to
+  #   update the store.
   class Base
+    # Finds an entry by it's ID.  The returned entry is a JSON hash
+    # @abstract Subclasses should implement this at a minimum to provide data
+    #   to the WCC::Contentful::Model API.
     def find(_id)
       raise NotImplementedError, "#{self.class} does not implement #find"
     end
 
+    # Sets the value of the entry with the given ID in the store.
+    # @abstract
     def set(_id, _value)
       raise NotImplementedError, "#{self.class} does not implement #set"
     end
 
+    # Removes the entry by ID from the store.
+    # @abstract
     def delete(_id)
       raise NotImplementedError, "#{self.class} does not implement #delete"
     end
 
+    # Processes a data point received via the Sync API.  This can be a published
+    # entry or asset, or a 'DeletedEntry' or 'DeletedAsset'.  The default
+    # implementation calls into #set and #delete to perform the appropriate
+    # operations in the store.
     def index(json)
       # Subclasses can override to do this in a more performant thread-safe way.
       # Example: postgres_store could do this in a stored procedure for speed
@@ -43,15 +60,30 @@ module WCC::Contentful::Store
       end
     end
 
-    def find_by(content_type:, filter: nil)
+    # Finds the first entry matching the given filter.  A content type is required.
+    #
+    # @param [String] content_type The ID of the content type to search for.
+    # @param [Hash] filter A set of key-value pairs defining filter operations.
+    #  See WCC::Contentful::Store::Base::Query
+    # @param [Hash] options An optional set of additional parameters to the query
+    #  defining for example include depth.  Not all store implementations respect all options.
+    def find_by(content_type:, filter: nil, options: nil)
       # default implementation - can be overridden
-      q = find_all(content_type: content_type)
+      q = find_all(content_type: content_type, options: { limit: 1 }.merge!(options || {}))
       q = q.apply(filter) if filter
       q.first
     end
 
+    # Finds all entries of the given content type.  A content type is required.
+    #
+    # @abstract Subclasses should implement this at a minimum to provide data
+    #   to the {WCC::Contentful::Model} API.
+    # @param [String] content_type The ID of the content type to search for.
+    # @param [Hash] options An optional set of additional parameters to the query
+    #  defining for example include depth.  Not all store implementations respect all options.
+    # @return [Query] A query object that exposes methods to apply filters
     # rubocop:disable Lint/UnusedMethodArgument
-    def find_all(content_type:)
+    def find_all(content_type:, options: nil)
       raise NotImplementedError, "#{self.class} does not implement find_all"
     end
     # rubocop:enable Lint/UnusedMethodArgument
@@ -60,29 +92,117 @@ module WCC::Contentful::Store
       @mutex = Concurrent::ReentrantReadWriteLock.new
     end
 
+    def ensure_hash(val)
+      raise ArgumentError, 'Value must be a Hash' unless val.is_a?(Hash)
+    end
+
     protected
 
     attr_reader :mutex
 
+    # The base class for query objects returned by find_all.  Subclasses should
+    # override the #result method to return an array-like containing the query
+    # results.
     class Query
       delegate :first, to: :result
       delegate :map, to: :result
       delegate :count, to: :result
 
+      OPERATORS = %i[
+        eq
+        ne
+        all
+        in
+        nin
+        exists
+        lt
+        lte
+        gt
+        gte
+        query
+        match
+      ].freeze
+
+      # @abstract Subclasses should provide this in order to fetch the results
+      #   of the query.
       def result
         raise NotImplementedError
       end
 
+      def initialize(store)
+        @store = store
+      end
+
+      # @abstract Subclasses can either override this method to properly respond
+      #   to find_by query objects, or they can define a method for each supported
+      #   operator.  Ex. `#eq`, `#ne`, `#gt`.
+      def apply_operator(operator, field, expected, context = nil)
+        respond_to?(operator) ||
+          raise(ArgumentError, "Operator not implemented: #{operator}")
+
+        public_send(operator, field, expected, context)
+      end
+
+      # Called with a filter object by {Base#find_by} in order to apply the filter.
       def apply(filter, context = nil)
         filter.reduce(self) do |query, (field, value)|
           if value.is_a?(Hash)
-            k = value.keys.first
-            raise ArgumentError, "Filter not implemented: #{value}" unless query.respond_to?(k)
-            query.public_send(k, field, value[k], context)
+            if op?(k = value.keys.first)
+              query.apply_operator(k.to_sym, field.to_s, value[k], context)
+            else
+              query.nested_conditions(field, value, context)
+            end
           else
-            query.eq(field.to_s, value)
+            query.apply_operator(:eq, field.to_s, value)
+          end
+        end
+      end
+
+      protected
+
+      # naive implementation recursively descends the graph to turns links into
+      # the actual entry data.  This calls {Base#find} for each link and so it is
+      # very inefficient.
+      #
+      # @abstract Override this to provide a more efficient implementation for
+      #   a given store.
+      def resolve_includes(entry, depth)
+        return entry unless entry && depth && depth > 0 && fields = entry['fields']
+
+        fields.each do |(_name, locales)|
+          # TODO: handle non-* locale
+          locales.each do |(locale, val)|
+            locales[locale] =
+              if val.is_a? Array
+                val.map { |e| resolve_link(e, depth) }
+              else
+                resolve_link(val, depth)
+              end
           end
         end
+
+        entry
+      end
+
+      def resolve_link(val, depth)
+        return val unless val.is_a?(Hash) && val.dig('sys', 'type') == 'Link'
+        return val unless included = @store.find(val.dig('sys', 'id'))
+
+        resolve_includes(included, depth - 1)
+      end
+
+      private
+
+      def op?(key)
+        OPERATORS.include?(key.to_sym)
+      end
+
+      def sys?(field)
+        field.to_s =~ /sys\./
+      end
+
+      def id?(field)
+        field.to_sym == :id
       end
     end
   end