wcc-contentful 0.4.0.pre.beta → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ff6f351c2042a4a584360d284e9079944fb3345f
-  data.tar.gz: 511988b09e647cf69c7b6c9accf4098fce140d08
+SHA256:
+  metadata.gz: ad34af3480e2e993b8046826ef3ca65b6f28ccf28e57e19cca79061a17766877
+  data.tar.gz: e0e214bf19721287af20502260033f595dcf461d2b5d31fd943c838b3ced204e
 SHA512:
-  metadata.gz: b3fac31b5af8c0f24e9bb91139946819134f5f68d38dfa2b90c70e30aacd6d9933bee1a716f5717b1d1769118875af4eb0f7ca6657fc6b09b736141c6cf4640d
-  data.tar.gz: 30141b535a65ade870b48573efee983310ad4c3ff704332f0cee15ceaf8e033e94d4cfa3bda4f42f36e3812fa7be49bf2d807375fa695231cf50b925cdfd92f8
+  metadata.gz: 1e67eacbb406c6269c0a43b6b7a1284836a357c9b9ff0ffe297ee188aeea6a2c96107b9a6fc127d2e622223f38797528e18e93485575068d7eae4a73a77f9375
+  data.tar.gz: c16f075caa26c9a9a0f0c22fe725eabf0b0c2900f4c58bac33fcb8ad42a10dee09ae257e2a91d71e06e878d1c7b8165dea28b1a5a791480f5c03e328f8635355

data/Guardfile

@@ -3,6 +3,39 @@
 
 # To run, use `bundle exec guard`.
 
+def watch_async(regexp)
+  raise ArgumentError, "No block given" unless block_given?
+  match_queue = Queue.new
+  
+  watch(regexp) do |match|
+    # Producer - add matches to the match queue
+    match_queue << match
+    nil
+  end
+
+  # Consumer - process matches as a batch
+  Thread.new do
+    loop do
+      matches = []
+      matches << match_queue.pop
+
+      loop do
+        begin
+          matches << match_queue.pop(true)
+        rescue ThreadError
+          break
+        end
+      end
+
+      begin
+        yield matches if matches.length > 0
+      rescue StandardError => ex
+        STDERR.puts "Error! #{ex}"
+      end
+    end
+  end
+end
+
 group :red_green_refactor, halt_on_fail: true do
   guard :rspec, cmd: 'bundle exec rspec' do
     require 'guard/rspec/dsl'
@@ -46,6 +79,16 @@ group :red_green_refactor, halt_on_fail: true do
     watch(%r{.+\.rb$})
     watch(%r{(?:.+/)?\.rubocop(?:_todo)?\.yml$}) { |m| File.dirname(m[0]) }
   end
+
+  guard :shell, all_on_start: false do
+    watch_async(%r{app/views/(.+\.html.*\.erb)}) { |matches|
+      
+      matches = matches.map { |m| File.absolute_path(m[0]) }
+      Dir.chdir('..') {
+        system("bundle exec erblint #{matches.join(' ')}")
+      }
+    }
+  end
 end
 
 group :autofix do

data/README.md

@@ -1,11 +1,106 @@
-[![Gem Version](https://badge.fury.io/rb/wcc-contentful.svg)](https://badge.fury.io/rb/wcc-contentful)
-[![CircleCI](https://circleci.com/gh/watermarkchurch/wcc-contentful.svg?style=svg)](https://circleci.com/gh/watermarkchurch/wcc-contentful)
+[![Gem Version](https://badge.fury.io/rb/wcc-contentful.svg)](https://rubygems.org/gems/wcc-contentful)
+[![Build Status](https://circleci.com/gh/watermarkchurch/wcc-contentful.svg?style=svg)](https://circleci.com/gh/watermarkchurch/wcc-contentful)
 [![Coverage Status](https://coveralls.io/repos/github/watermarkchurch/wcc-contentful/badge.svg?branch=master)](https://coveralls.io/github/watermarkchurch/wcc-contentful?branch=master)
 
-Full documentation: https://www.rubydoc.info/github/watermarkchurch/wcc-contentful
+Full documentation: https://watermarkchurch.github.io/wcc-contentful/latest/wcc-contentful/
 
 # WCC::Contentful
 
+An alternative to Contentful's [contentful.rb ruby client](https://github.com/contentful/contentful.rb/), [contentful_model](https://github.com/contentful/contentful_model), and [contentful_rails](https://github.com/contentful/contentful_rails) gems all in one.
+
+Table of Contents:
+
+1. [Why?](#why-did-you-rewrite-the-contentful-ruby-stack)
+2. [Installation](#installation)
+3. [Configuration](#configure)
+4. [Usage](#usage)
+  1. [Model API](#wcccontentfulmodel-api)
+  2. [Store API](#store-api)
+  3. [Direct CDN client](#direct-cdn-api-simpleclient)
+  4. [Accessing the APIs](#accessing-the-apis-within-application-code)
+5. [Architecture](#architecture)
+6. [Test Helpers](#test-helpers)
+7. [Advanced Configuration Example](#advanced-configuration-example)
+8. [Development](#development)
+9. [Contributing](#contributing)
+10. [License](#license)
+
+
+## Why did you rewrite the Contentful ruby stack?
+
+We started working with Contentful almost 3 years ago.  Since that time, Contentful's ruby stack has improved, but there are still a number of pain points that we feel we have addressed better with our gem.  These are:
+
+* [Low-level caching](#low-level-caching)
+* [Better integration with Rails & Rails models](#better-rails-integration)
+* [Automatic pagination and Automatic link resolution](#automatic-pagination-and-link-resolution)
+* [Automatic webhook management](#automatic-webhook-management)
+
+Our gem no longer depends on any of the Contentful gems and interacts directly with the [Contentful CDA](https://www.contentful.com/developers/docs/references/content-delivery-api/) and [Content Management API](https://www.contentful.com/developers/docs/references/content-management-api/) over HTTPS.
+
+### Low-level caching
+
+The wcc-contentful gem enables caching at two levels: the HTTP response using [Faraday HTTP cache middleware](https://github.com/sourcelevel/faraday-http-cache), and at the Entry level using the Rails cache and the [Sync API](https://www.contentful.com/developers/docs/references/content-delivery-api/#/reference/synchronization) to keep it up to date.  We've found these two cache layers to be very effective at reducing both round trip latency to the Content Delivery API, as well as reducing our monthly API request usage. (which reduces our overage charges.  Hooray!)
+
+#### At the request/response level
+By default, the contentful.rb gem requires the [HTTP library](https://rubygems.org/gems/http).  While simple and straightforward to use, it is not as powerful for caching.  We decided to make our client conform to the [Faraday gem's API](https://github.com/lostisland/faraday).  If you prefer not to use Faraday, you can choose to supply your own HTTP adapter that "quacks like" Faraday (see the [TyphoeusAdapter](https://github.com/watermarkchurch/wcc-contentful/blob/master/wcc-contentful/lib/wcc/contentful/simple_client/typhoeus_adapter.rb) for one implementation).
+
+Using Faraday makes it easy to add Middleware.  As an example, our flagship Rails app that powers watermark.org uses the following configuration in Production, which provides us with instrumentation through statsd, logging, and caching:
+```rb
+config.connection = Faraday.new do |builder|
+  builder.use :http_cache,
+    shared_cache: false,
+    store: ActiveSupport::Cache::MemoryStore.new(size: 512.megabytes),
+    logger: Rails.logger,
+    serializer: Marshal,
+    instrumenter: ActiveSupport::Notifications
+
+  builder.use :gzip
+  builder.response :logger, Rails.logger, headers: false, bodies: false if Rails.env.development?
+  builder.request :instrumentation
+  builder.adapter :typhoeus
+end
+```
+
+#### At the Entry level
+
+Our stack has three layers, the middle layer being essentially a cache for individual Entry hashes parsed out of responses from the Delivery API.  We were able to add a caching layer here which stores entries retrieved over the Sync API, and responds to queries with cached versions of local content when possible.  We consider this to be our best innovation on the Contentful ruby stack.
+
+We have successfully created caching layers using Memcached, Postgres, and an in-memory hash.  The architecture allows other caching implementations to be created fairly easily, and we have a set of rspec specs that can verify that a cache store behaves appropriately.  For more information, [see the documentation on the caching modes here](https://watermarkchurch.github.io/wcc-contentful/latest/wcc-contentful/WCC/Contentful/Store.html).
+
+### Better Rails Integration
+
+When we initially got started with the Contentful ruby models, we encountered one problem that was more frustrating than all others: If a field exists in the content model, but the particular entry we're working with does not have that field populated, then accessing that field raised a `NoMethodError`.  This caused us to litter our code with `if defined?(entry.my_field)` which is bad practice.  (Note: this has since been fixed in contentful.rb v2).
+
+We decided it was better to not rely on `method_missing?` (what contentful.rb does), and instead to use `define_method` in an initializer to generate the methods for our models.  This has the advantage that calling `.instance_methods` on a model class includes all the fields present in the content model.
+
+We also took advantage of Rails' naming conventions to automatically infer the content type name based on the class name.  Thus in our code, we have `app/models/page.rb` which defines `class Page << WCC::Contentful::Model::Page`, and is automatically linked to the `page` content type ID.  (Note: this is overridable on a per-model basis)
+
+All our models are automatically generated at startup which improves response times at the expense of initialization time.  In addition, our content model registry allows easy definition of custom models in your `app/models` directory to override fields.  This plays nice with other gems like algoliasearch-rails, which allows you to declaratively manage your Algolia indexes.  Another example from our flagship watermark.org:
+
+```rb
+class Page < WCC::Contentful::Model::Page
+  include AlgoliaSearch
+
+  algoliasearch(index_name: 'pages') do
+    attribute(:title, :slug)
+    ...
+  end
+```
+
+### Automatic Pagination and Link Resolution
+
+Using the `contentful_model` gem, calling `Page.all.load` does not give you all Page entries if there are more than 100.  To get the next page you must call `.paginate` on the response.  By contrast, `Page.find_all` in the `wcc-contentful` gem gives you a [Lazy Enumerator](https://ruby-doc.org/core-2.5.0/Enumerator/Lazy.html).  As you iterate past the 100th entry, the enumerator will automatically fetch the next page.  If you only enumerate 99 entries (say with `.take(99)`), then the second page will never be fetched.
+
+Similarly, if your Page references an asset, say `hero_image`, that field returns a `Link` object rather than the actual `Asset`.  You must either predefine how many links you need using `Page.load_children(3).all.load`, or detect that `hero_image` is a `Link` like `if @page.hero_image.is_a? Contentful::Link` and then call `.resolve` on the link.  We found all of that to be too cumbersome when we are down in a nested partial view template that may be invoked from multiple places.
+
+The `wcc-contentful` gem, by contrast, automatically resolves a link when accessing the associated attribute.  So in our example above, `wcc-contentful` will **always** return a `WCC::Contentful::Asset` when calling `@page.hero_image`, even if it has to execute a query to cdn.contentful.com in order to fetch it.
+
+Warning: This can easily lead to you exhausting your Contentful API quota if you do not carefully tune your cache, which you should be doing anyways!  The default settings will use the Rails cache to try to cache these resolutions, but *you are ultimately responsible for how many queries you execute!*
+
+### Automatic webhook management
+
+The `wcc-contentful` gem, just like `contentful_rails`, provides an Engine to be mounted in your Rails routes file.  Unlike `contentful_rails`, if you also configure `wcc-contentful` with a Contentful Management Token and a public `app_url`, then on startup the `wcc-contentful` engine will reach out to the Contentful Management API and ensure that a webhook is configured to point to your app.  This is one less devops burden on you, and plays very nicely in with Heroku review apps.
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -14,17 +109,30 @@ Add this line to your application's Gemfile:
 gem 'wcc-contentful', require: 'wcc/contentful/rails'
 ```
 
+If you're not using rails, exclude the `require:` parameter.
+
+```ruby
+gem 'wcc-contentful'
+```
+
 And then execute:
 
-    $ bundle
+```
+$ bundle
+```
 
-Or install it yourself as:
+Or install it yourself:
 
-    $ gem install wcc-contentful
+```
+$ gem install wcc-contentful
+```
 
 ## Configure
 
+Put this in an initializer:
+
 ```ruby
+# config/initializers/wcc_contentful.rb
 WCC::Contentful.configure do |config|
   config.access_token = <CONTENTFUL_ACCESS_TOKEN>
   config.space = <CONTENTFUL_SPACE_ID>
@@ -33,13 +141,16 @@ end
 WCC::Contentful.init!
 ```
 
+All configuration options can be found [in the rubydoc under
+WCC::Contentful::Configuration](https://watermarkchurch.github.io/wcc-contentful/latest/wcc-contentful/WCC/Contentful/Configuration) 
+
 ## Usage
 
 ### WCC::Contentful::Model API
 
 The WCC::Contentful::Model API exposes Contentful data as a set of dynamically
 generated Ruby objects.  These objects are based on the content types in your
-Contentful space.  All these objects are generated by WCC::Contentful.init!
+Contentful space.  All these objects are generated by `WCC::Contentful.init!`
 
 The following examples show how to use this API to find entries of the `page`
 content type:
@@ -164,7 +275,8 @@ response.includes
 
 The client handles Paging automatically within the lazy iterator returned by #items.
 This lazy iterator does not respect the `limit` param - that param is only passed
-through to the API to set the page size.
+through to the API to set the page size.  If you truly want a limited subset of
+response items, use [`response.items.take(n)`](https://ruby-doc.org/core-2.5.3/Enumerable.html#method-i-take)
 
 Entries included via the `include` parameter are made available on the #includes
 field.  This is a hash of `<entry ID> => <raw entry>` and makes it easy to grab
@@ -211,6 +323,10 @@ class MyJob < ApplicationJob
 end
 ```
 
+## Architecture
+
+![wcc-contentful diagram](./doc-static/wcc-contentful.png)
+
 ## Test Helpers
 
 To use the test helpers, include the following in your rails_helper.rb:
@@ -239,7 +355,7 @@ instance.other_optional_field
 # => nil
 
 instance.not_a_field
-# NoMethodError: undefined method `not_a_field' for #<Menu:0x00007fbac81ee490>
+# NoMethodError: undefined method `not_a_field' for #<MyContentType:0x00007fbac81ee490>
 
 ##
 # Builds a rspec double of the Contentful model for the given content_type.
@@ -251,6 +367,9 @@ dbl = contentful_double('my-content-type', my_field: 'other-value')
 dbl.my_field
 # => "other-value"
 
+dbl.other_optional_field
+# => nil
+
 dbl.not_a_field
 # => #<Double (anonymous)> received unexpected message :not_a_field with (no args)
 
@@ -270,21 +389,90 @@ MyContentType.find_by(my_field: 'test') == stubbed
 # => true
 ```
 
+## Advanced Configuration Example
 
-## Development
+Here's an example containing all the configuration options, and a sample setup for
+automatic deployment to Heroku.  This is intended to make you aware of what is possible,
+and not as a general recommendation of what your setup should look like.
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+```ruby
+# config/initializers/wcc_contentful.rb
+WCC::Contentful.configure do |config|
+  config.access_token = ENV['CONTENTFUL_ACCESS_TOKEN']
+  config.space = ENV['CONTENTFUL_SPACE_ID']
+  config.environment = ENV['CONTENTFUL_ENVIRONMENT']
+  config.preview_token = ENV['CONTENTFUL_PREVIEW_ACCESS_TOKEN']
+
+  # You may or may not want to provide this to your production server...
+  config.management_token = ENV['CONTENTFUL_MANAGEMENT_TOKEN'] unless Rails.env.production?
+
+  config.app_url = "https://#{ENV['HOSTNAME']}"
+  config.webhook_username = 'my-app-webhook'
+  config.webhook_password = Rails.application.secrets.webhook_password
+  config.webhook_jobs << MyOnWebhookJob
+
+  config.store = :lazy_sync, Rails.cache if Rails.env.production?
+  # config.store = MyCustomStore.new
+
+  # Use a custom Faraday connection
+  config.connection = Faraday.new do |builder|
+    f.request :retry
+    f.request MyFaradayRequestAdapter.new
+    ...
+  end
+  # OR implement some adapter like this to use another HTTP client
+  config.connection = MyNetHttpAdapter.new
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+  config.update_schema_file = :never
+end
+
+WCC::Contentful.init!
+```
+
+For Heroku:
+
+```yaml
+# Procfile
+web: bundle exec rails s
+worker: bundle exec sidekiq
+release: bin/release
+```
+
+```sh
+# bin/release
+#!/bin/sh
+
+set -e
+
+echo "Migrating database..."
+bin/rake db:migrate
+
+echo "Migrating contentful..."
+migrations_to_be_run=$( ... ) # somehow figure this out
+node_modules/.bin/contentful-migration \
+    -s $CONTENTFUL_SPACE_ID -a $CONTENTFUL_MANAGEMENT_TOKEN \
+    -y -p "$migrations_to_be_run"
+
+echo "Updating schema file..."
+rake wcc_contentful:download_schema
+```
+
+All configuration options can be found [in the rubydoc](https://www.rubydoc.info/gems/wcc-contentful/WCC/Contentful/Configuration) under
+{WCC::Contentful::Configuration}
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/watermarkchurch/wcc-contentful. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/watermarkchurch/wcc-contentful.
 
-## License
+The developers at Watermark Community Church have pledged to govern their interactions with each other, with their clients, and with the larger wcc-contentful user community in accordance with the "instruments of good works" from chapter 4 of The Rule of St. Benedict (hereafter: "The Rule"). This code of ethics has proven its mettle in thousands of diverse communities for over 1,500 years, and has served as a baseline for many civil law codes since the time of Charlemagne.
 
-The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+[See the full Code of Ethics](https://github.com/watermarkchurch/wcc-contentful/blob/master/CODE_OF_ETHICS.md)
 
-## Code of Conduct
+## License
 
-Everyone interacting in the WCC::Contentful project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/watermarkchurch/wcc-contentful/blob/master/CODE_OF_CONDUCT.md).
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+task :release do
+  raise StandardError, 'Please run rake release only from the root folder.'
+end

data/app/controllers/wcc/contentful/webhook_controller.rb

@@ -8,6 +8,7 @@ module WCC::Contentful
   # the jobs configured in {WCC::Contentful::Configuration WCC::Contentful::Configuration#webhook_jobs}
   class WebhookController < ApplicationController
     include WCC::Contentful::ServiceAccessors
+    include Wisper::Publisher
 
     before_action :authorize_contentful
     protect_from_forgery unless: -> { request.format.json? }
@@ -17,27 +18,17 @@ module WCC::Contentful
     end
 
     def receive
-      event = params.require('webhook').permit!
-      event.require('sys').require(%w[id type])
-      event = event.to_h
-
-      # Immediately update the store, we may update again later using DelayedSyncJob.
-      store.index(event) if store.respond_to?(:index)
-
-      jobs.each do |job|
-        begin
-          if job.respond_to?(:perform_later)
-            job.perform_later(event)
-          elsif job.respond_to?(:call)
-            job.call(event)
-          else
-            Rails.logger.error "Misconfigured webhook job: #{job} does not respond to " \
-              ':perform_later or :call'
-          end
-        rescue StandardError => e
-          Rails.logger.error "Error in job #{job}: #{e}"
-        end
-      end
+      params.require('sys').require(%w[id type])
+      params.permit('sys', 'fields')
+      event = params.slice('sys', 'fields').permit!.to_h
+
+      return unless check_environment(event)
+
+      # Immediately update the store, we may update again later using SyncEngine::Job.
+      store.index(event) if store.index?
+
+      event = WCC::Contentful::Event.from_raw(event, source: self)
+      emit_event(event)
     end
 
     private
@@ -62,9 +53,19 @@ module WCC::Contentful
       render json: { msg: 'This endpoint only responds to webhooks from Contentful' }, status: 406
     end
 
-    def jobs
-      jobs = [WCC::Contentful::DelayedSyncJob]
-      jobs.push(*WCC::Contentful.configuration.webhook_jobs)
+    def check_environment(event)
+      environment_id = event.dig('sys', 'environment', 'sys', 'id')
+      return true unless environment_id.present?
+
+      configured_environment = WCC::Contentful.configuration.environment.presence || 'master'
+      configured_environment.casecmp(environment_id) == 0
+    end
+
+    def emit_event(event)
+      type = event.dig('sys', 'type')
+      raise ArgumentError, "Unknown event type #{event}" unless type.present?
+
+      broadcast(type, event)
     end
   end
 end

data/app/jobs/wcc/contentful/webhook_enable_job.rb

@@ -7,7 +7,9 @@ module WCC::Contentful
     self.queue_adapter = :async
     queue_as :default
 
-    def perform(args)
+    def perform(args = {})
+      args = default_configuration.merge!(args)
+
       client = WCC::Contentful::SimpleClient::Management.new(
         args
       )
@@ -26,7 +28,8 @@ module WCC::Contentful
         'topics' => [
           '*.publish',
           '*.unpublish'
-        ]
+        ],
+        'filters' => webhook_filters
       }
       body['httpBasicUsername'] = webhook_username if webhook_username.present?
       body['httpBasicPassword'] = webhook_password if webhook_password.present?
@@ -39,5 +42,36 @@ module WCC::Contentful
         raise
       end
     end
+
+    private
+
+    def default_configuration
+      return {} unless config = WCC::Contentful&.configuration
+
+      {
+        management_token: config.management_token,
+        app_url: config.app_url,
+        space: config.space,
+        environment: config.environment,
+        default_locale: config.default_locale,
+        connection: config.connection,
+        webhook_username: config.webhook_username,
+        webhook_password: config.webhook_password
+      }
+    end
+
+    def webhook_filters
+      filters = []
+
+      if (environment_id = WCC::Contentful.configuration&.environment).present?
+        filters << {
+          'equals' => [
+            { 'doc' => 'sys.environment.sys.id' },
+            environment_id
+          ]
+        }
+      end
+      filters
+    end
   end
 end