wcc-contentful 0.4.0.pre.rc → 1.0.0.pre.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: dbe6cdec1d0756eb9c1f8930efb8023ebbf6a758
-  data.tar.gz: 61fadc6215d698b8865dd7ad0a1c2e3c8dd8f26f
+SHA256:
+  metadata.gz: e8c289b7bbddc39150c9ab86fb781e4dcdd740153a6e39906f438caa11740077
+  data.tar.gz: 36d446fb8abe8b7dbed82f68d36280d85528f4ba60a9af2707bbccbdb4a04794
 SHA512:
-  metadata.gz: 3d9ec54fe32c9380048236927fbcfd77a44e3c499327520054b61ae3976a561cdf7a1d7bad916b6f33b839963877f7d1c7e53067cebd8e646c74bb2d335c9606
-  data.tar.gz: aed63f65c37498f9ebee35f768c49c5be763e32032714321aaa8f35eba07d8f10d033082b85401e51cb4e8c964698a11d216eb6af885707ff6ec5bd43cefccb9
+  metadata.gz: a3a992e38c9688f7bc0834bf0629c5df5e6d54fdca044295a534d741ce3fc6c6e4017a9a48ba65c765af91c6ff9733ec5469ecb0aaa5a1302e371c6fd3325de1
+  data.tar.gz: 90aa3d052268446d7d1bb15cea05048b643904026d8406ea274045db6a68b3c395b09e60178a13f028e3238a8a3606c3f63abc58a47ba5c8b822834b6a8c3064

data/Guardfile

@@ -3,6 +3,39 @@
 
 # To run, use `bundle exec guard`.
 
+def watch_async(regexp)
+  raise ArgumentError, "No block given" unless block_given?
+  match_queue = Queue.new
+  
+  watch(regexp) do |match|
+    # Producer - add matches to the match queue
+    match_queue << match
+    nil
+  end
+
+  # Consumer - process matches as a batch
+  Thread.new do
+    loop do
+      matches = []
+      matches << match_queue.pop
+
+      loop do
+        begin
+          matches << match_queue.pop(true)
+        rescue ThreadError
+          break
+        end
+      end
+
+      begin
+        yield matches if matches.length > 0
+      rescue StandardError => ex
+        STDERR.puts "Error! #{ex}"
+      end
+    end
+  end
+end
+
 group :red_green_refactor, halt_on_fail: true do
   guard :rspec, cmd: 'bundle exec rspec' do
     require 'guard/rspec/dsl'
@@ -46,6 +79,16 @@ group :red_green_refactor, halt_on_fail: true do
     watch(%r{.+\.rb$})
     watch(%r{(?:.+/)?\.rubocop(?:_todo)?\.yml$}) { |m| File.dirname(m[0]) }
   end
+
+  guard :shell, all_on_start: false do
+    watch_async(%r{app/views/(.+\.html.*\.erb)}) { |matches|
+      
+      matches = matches.map { |m| File.absolute_path(m[0]) }
+      Dir.chdir('..') {
+        system("bundle exec erblint #{matches.join(' ')}")
+      }
+    }
+  end
 end
 
 group :autofix do

data/README.md

@@ -1,8 +1,8 @@
-[![Gem Version](https://badge.fury.io/rb/wcc-contentful.svg)](https://badge.fury.io/rb/wcc-contentful)
-[![CircleCI](https://circleci.com/gh/watermarkchurch/wcc-contentful.svg?style=svg)](https://circleci.com/gh/watermarkchurch/wcc-contentful)
+[![Gem Version](https://badge.fury.io/rb/wcc-contentful.svg)](https://rubygems.org/gems/wcc-contentful)
+[![Build Status](https://travis-ci.org/watermarkchurch/wcc-contentful.svg?branch=master)](https://travis-ci.org/watermarkchurch/wcc-contentful)
 [![Coverage Status](https://coveralls.io/repos/github/watermarkchurch/wcc-contentful/badge.svg?branch=master)](https://coveralls.io/github/watermarkchurch/wcc-contentful?branch=master)
 
-Full documentation: https://www.rubydoc.info/github/watermarkchurch/wcc-contentful
+Full documentation: https://www.rubydoc.info/gems/wcc-contentful
 
 # WCC::Contentful
 
@@ -14,17 +14,26 @@ Add this line to your application's Gemfile:
 gem 'wcc-contentful', require: 'wcc/contentful/rails'
 ```
 
-And then execute:
+If you're not using rails, exclude the `require:` parameter.
 
-    $ bundle
+```ruby
+gem 'wcc-contentful'
+```
 
+And then execute:
+```
+    $ bundle
+```
 Or install it yourself as:
-
+```
     $ gem install wcc-contentful
+```
 
 ## Configure
 
+Put this in an initializer:
 ```ruby
+# config/initializers/wcc_contentful.rb
 WCC::Contentful.configure do |config|
   config.access_token = <CONTENTFUL_ACCESS_TOKEN>
   config.space = <CONTENTFUL_SPACE_ID>
@@ -33,13 +42,16 @@ end
 WCC::Contentful.init!
 ```
 
+All configuration options can be found [in the rubydoc](https://www.rubydoc.info/gems/wcc-contentful/WCC/Contentful/Configuration) under
+{WCC::Contentful::Configuration}
+
 ## Usage
 
 ### WCC::Contentful::Model API
 
 The WCC::Contentful::Model API exposes Contentful data as a set of dynamically
 generated Ruby objects.  These objects are based on the content types in your
-Contentful space.  All these objects are generated by WCC::Contentful.init!
+Contentful space.  All these objects are generated by `WCC::Contentful.init!`
 
 The following examples show how to use this API to find entries of the `page`
 content type:
@@ -164,7 +176,8 @@ response.includes
 
 The client handles Paging automatically within the lazy iterator returned by #items.
 This lazy iterator does not respect the `limit` param - that param is only passed
-through to the API to set the page size.
+through to the API to set the page size.  If you truly want a limited subset of
+response items, use [`response.items.take(n)`](https://ruby-doc.org/core-2.5.3/Enumerable.html#method-i-take)
 
 Entries included via the `include` parameter are made available on the #includes
 field.  This is a hash of `<entry ID> => <raw entry>` and makes it easy to grab
@@ -211,6 +224,10 @@ class MyJob < ApplicationJob
 end
 ```
 
+## Architecture
+
+![wcc-contentful diagram](./doc/wcc-contentful.png)
+
 ## Test Helpers
 
 To use the test helpers, include the following in your rails_helper.rb:
@@ -239,7 +256,7 @@ instance.other_optional_field
 # => nil
 
 instance.not_a_field
-# NoMethodError: undefined method `not_a_field' for #<Menu:0x00007fbac81ee490>
+# NoMethodError: undefined method `not_a_field' for #<MyContentType:0x00007fbac81ee490>
 
 ##
 # Builds a rspec double of the Contentful model for the given content_type.
@@ -251,6 +268,9 @@ dbl = contentful_double('my-content-type', my_field: 'other-value')
 dbl.my_field
 # => "other-value"
 
+dbl.other_optional_field
+# => nil
+
 dbl.not_a_field
 # => #<Double (anonymous)> received unexpected message :not_a_field with (no args)
 
@@ -270,12 +290,81 @@ MyContentType.find_by(my_field: 'test') == stubbed
 # => true
 ```
 
+## Advanced Configuration Example
 
-## Development
+Here's an example containing all the configuration options, and a sample setup for
+automatic deployment to Heroku.  This is intended to make you aware of what is possible,
+and not as a general recommendation of what your setup should look like.
+
+```ruby
+# config/initializers/wcc_contentful.rb
+WCC::Contentful.configure do |config|
+  config.access_token = ENV['CONTENTFUL_ACCESS_TOKEN']
+  config.space = ENV['CONTENTFUL_SPACE_ID']
+  config.environment = ENV['CONTENTFUL_ENVIRONMENT']
+  config.preview_token = ENV['CONTENTFUL_PREVIEW_ACCESS_TOKEN']
+
+  # You may or may not want to provide this to your production server...
+  config.management_token = ENV['CONTENTFUL_MANAGEMENT_TOKEN'] unless Rails.env.production?
+
+  config.app_url = "https://#{ENV['HOSTNAME']}"
+  config.webhook_username = 'my-app-webhook'
+  config.webhook_password = Rails.application.secrets.webhook_password
+  config.webhook_jobs << MyOnWebhookJob
+
+  config.store = :lazy_sync, Rails.cache if Rails.env.production?
+  # config.store = MyCustomStore.new
+
+  # Use a custom Faraday connection
+  config.connection = Faraday.new do |builder|
+    f.request :retry
+    f.request MyFaradayRequestAdapter.new
+    ...
+  end
+  # OR implement some adapter like this to use another HTTP client
+  config.connection = MyNetHttpAdapter.new
+
+  config.update_schema_file = :never
+end
+
+WCC::Contentful.init!
+```
+
+For Heroku:
+
+```yaml
+# Procfile
+web: bundle exec rails s
+worker: bundle exec sidekiq
+release: bin/release
+```
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+```sh
+# bin/release
+#!/bin/sh
+
+set -e
+
+echo "Migrating database..."
+bin/rake db:migrate
+
+echo "Migrating contentful..."
+migrations_to_be_run=$( ... ) # somehow figure this out
+node_modules/.bin/contentful-migration \
+    -s $CONTENTFUL_SPACE_ID -a $CONTENTFUL_MANAGEMENT_TOKEN \
+    -y -p "$migrations_to_be_run"
+
+echo "Updating schema file..."
+rake wcc_contentful:download_schema
+```
+
+All configuration options can be found [in the rubydoc](https://www.rubydoc.info/gems/wcc-contentful/WCC/Contentful/Configuration) under
+{WCC::Contentful::Configuration}
+
+
+## Development
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
 ## Contributing
 

data/app/controllers/wcc/contentful/webhook_controller.rb

@@ -8,6 +8,7 @@ module WCC::Contentful
   # the jobs configured in {WCC::Contentful::Configuration WCC::Contentful::Configuration#webhook_jobs}
   class WebhookController < ApplicationController
     include WCC::Contentful::ServiceAccessors
+    include Wisper::Publisher
 
     before_action :authorize_contentful
     protect_from_forgery unless: -> { request.format.json? }
@@ -17,27 +18,17 @@ module WCC::Contentful
     end
 
     def receive
-      event = params.require('webhook').permit!
-      event.require('sys').require(%w[id type])
-      event = event.to_h
-
-      # Immediately update the store, we may update again later using DelayedSyncJob.
-      store.index(event) if store.respond_to?(:index)
-
-      jobs.each do |job|
-        begin
-          if job.respond_to?(:perform_later)
-            job.perform_later(event)
-          elsif job.respond_to?(:call)
-            job.call(event)
-          else
-            Rails.logger.error "Misconfigured webhook job: #{job} does not respond to " \
-              ':perform_later or :call'
-          end
-        rescue StandardError => e
-          Rails.logger.error "Error in job #{job}: #{e}"
-        end
-      end
+      params.require('sys').require(%w[id type])
+      params.permit('sys', 'fields')
+      event = params.slice('sys', 'fields').permit!.to_h
+
+      return unless check_environment(event)
+
+      # Immediately update the store, we may update again later using SyncEngine::Job.
+      store.index(event) if store.index?
+
+      event = WCC::Contentful::Event.from_raw(event, source: self)
+      emit_event(event)
     end
 
     private
@@ -62,9 +53,19 @@ module WCC::Contentful
       render json: { msg: 'This endpoint only responds to webhooks from Contentful' }, status: 406
     end
 
-    def jobs
-      jobs = [WCC::Contentful::DelayedSyncJob]
-      jobs.push(*WCC::Contentful.configuration.webhook_jobs)
+    def check_environment(event)
+      environment_id = event.dig('sys', 'environment', 'sys', 'id')
+      return true unless environment_id.present?
+
+      configured_environment = WCC::Contentful.configuration.environment.presence || 'master'
+      configured_environment.casecmp(environment_id) == 0
+    end
+
+    def emit_event(event)
+      type = event.dig('sys', 'type')
+      raise ArgumentError, "Unknown event type #{event}" unless type.present?
+
+      broadcast(type, event)
     end
   end
 end

data/app/jobs/wcc/contentful/webhook_enable_job.rb

@@ -7,7 +7,9 @@ module WCC::Contentful
     self.queue_adapter = :async
     queue_as :default
 
-    def perform(args)
+    def perform(args = {})
+      args = default_configuration.merge!(args)
+
       client = WCC::Contentful::SimpleClient::Management.new(
         args
       )
@@ -26,7 +28,8 @@ module WCC::Contentful
         'topics' => [
           '*.publish',
           '*.unpublish'
-        ]
+        ],
+        'filters' => webhook_filters
       }
       body['httpBasicUsername'] = webhook_username if webhook_username.present?
       body['httpBasicPassword'] = webhook_password if webhook_password.present?
@@ -39,5 +42,36 @@ module WCC::Contentful
         raise
       end
     end
+
+    private
+
+    def default_configuration
+      return {} unless config = WCC::Contentful&.configuration
+
+      {
+        management_token: config.management_token,
+        app_url: config.app_url,
+        space: config.space,
+        environment: config.environment,
+        default_locale: config.default_locale,
+        connection: config.connection,
+        webhook_username: config.webhook_username,
+        webhook_password: config.webhook_password
+      }
+    end
+
+    def webhook_filters
+      filters = []
+
+      if (environment_id = WCC::Contentful.configuration&.environment).present?
+        filters << {
+          'equals' => [
+            { 'doc' => 'sys.environment.sys.id' },
+            environment_id
+          ]
+        }
+      end
+      filters
+    end
   end
 end

data/config/routes.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 WCC::Contentful::Engine.routes.draw do
-  post 'webhook/receive', to: 'webhook#receive'
+  post 'webhook/receive', to: 'wcc/contentful/webhook#receive'
 end

data/lib/tasks/download_schema.rake

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'wcc/contentful'
+require 'wcc/contentful/downloads_schema'
+
+namespace :wcc_contentful do
+  desc 'Downloads the schema from the currently configured space and stores it in ' \
+    'db/contentful-schema.json'
+  task download_schema: :environment do
+    WCC::Contentful::DownloadsSchema.call
+  end
+end

data/lib/wcc/contentful.rb

@@ -5,9 +5,12 @@ require 'wcc/contentful/version'
 require 'active_support'
 require 'active_support/core_ext/object'
 
+require 'wcc/contentful/active_record_shim'
 require 'wcc/contentful/configuration'
+require 'wcc/contentful/downloads_schema'
 require 'wcc/contentful/exceptions'
 require 'wcc/contentful/helpers'
+require 'wcc/contentful/link_visitor'
 require 'wcc/contentful/services'
 require 'wcc/contentful/simple_client'
 require 'wcc/contentful/store'
@@ -16,6 +19,9 @@ require 'wcc/contentful/model'
 require 'wcc/contentful/model_methods'
 require 'wcc/contentful/model_singleton_methods'
 require 'wcc/contentful/model_builder'
+require 'wcc/contentful/sync_engine'
+require 'wcc/contentful/events'
+require 'wcc/contentful/middleware'
 
 # The root namespace of the wcc-contentful gem
 #
@@ -23,16 +29,33 @@ require 'wcc/contentful/model_builder'
 # initializer.
 module WCC::Contentful
   class << self
+    attr_reader :initialized
+
     # Gets the current configuration, after calling WCC::Contentful.configure
     attr_reader :configuration
 
     attr_reader :types
+
+    # Gets all queryable locales.
+    # Reserved for future use.
+    def locales
+      @locales ||= { 'en-US' => {} }.freeze
+    end
+
+    def logger
+      return Rails.logger if defined?(Rails)
+
+      @logger ||= Logger.new(STDERR)
+    end
   end
 
   # Configures the WCC::Contentful gem to talk to a Contentful space.
   # This must be called first in your initializer, before #init! or accessing the
   # client.
+  # See WCC::Contentful::Configuration for all configuration options.
   def self.configure
+    raise InitializationError, 'Cannot configure after initialization' if @initialized
+
     @configuration ||= Configuration.new
     yield(configuration)
 
@@ -51,29 +74,60 @@ module WCC::Contentful
   #   class Page < WCC::Contentful::Model::Page; end
   #   Page.find_by(slug: 'about-us')
   def self.init!
-    raise ArgumentError, 'Please first call WCC:Contentful.configure' if configuration.nil?
+    raise InitializationError, 'Please first call WCC:Contentful.configure' if configuration.nil?
+    raise InitializationError, 'Already Initialized' if @initialized
 
-    # we want as much as possible the raw JSON from the API so use the management
-    # client if possible
-    client = Services.instance.management_client ||
-      Services.instance.client
+    if configuration.update_schema_file == :always ||
+        (configuration.update_schema_file == :if_possible && Services.instance.management_client) ||
+        configuration.update_schema_file == :if_missing && !File.exist?(configuration.schema_file)
 
-    @content_types = client.content_types(limit: 1000).items
+      begin
+        downloader = WCC::Contentful::DownloadsSchema.new
+        downloader.update! if configuration.update_schema_file == :always || downloader.needs_update?
+      rescue WCC::Contentful::SimpleClient::ApiError => e
+        raise InitializationError, e if configuration.update_schema_file == :always
 
-    indexer =
-      ContentTypeIndexer.new.tap do |ixr|
-        @content_types.each { |type| ixr.index(type) }
+        WCC::Contentful.logger.warn("Unable to download schema from management API - #{e.message}")
       end
-    @types = indexer.types
+    end
 
-    store = Services.instance.store
-    if store.respond_to?(:index)
-      # Drop an initial sync
-      WCC::Contentful::DelayedSyncJob.perform_later
+    @content_types =
+      begin
+        if File.exist?(configuration.schema_file)
+          JSON.parse(File.read(configuration.schema_file))['contentTypes']
+        end
+      rescue JSON::ParserError
+        WCC::Contentful.logger.warn("Schema file invalid, ignoring it: #{configuration.schema_file}")
+        nil
+      end
+
+    if !@content_types && %i[if_possible never].include?(configuration.update_schema_file)
+      # Final fallback - try to grab content types from CDN.  We can't update the file
+      # because the CDN doesn't have all the field validation info, but we can at least
+      # build the WCC::Contentful::Model instances.
+      client = Services.instance.management_client ||
+        Services.instance.client
+      begin
+        @content_types = client.content_types(limit: 1000).items if client
+      rescue WCC::Contentful::SimpleClient::ApiError => e
+        # indicates bad credentials
+        WCC::Contentful.logger.warn("Unable to load content types from API - #{e.message}")
+      end
     end
 
-    WCC::Contentful::ModelBuilder.new(@types).build_models
+    unless @content_types
+      raise InitializationError, 'Unable to load content types from schema file or API!' \
+        ' Check your access token and space ID.'
+    end
 
-    require_relative 'contentful/client_ext' if defined?(::Contentful)
+    indexer = ContentTypeIndexer.from_json_schema(@content_types)
+    @types = indexer.types
+
+    # Drop an initial sync
+    WCC::Contentful::SyncEngine::Job.perform_later if defined?(WCC::Contentful::SyncEngine::Job)
+
+    WCC::Contentful::ModelBuilder.new(@types).build_models
+    @configuration = @configuration.freeze
+    @initialized = true
   end
 end