contentful_middleman 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 54e1723317b0e9be644aec1619d93323f4f22a13
-  data.tar.gz: d9fac5661d65115085e91f64761642d8aa8556c1
+  metadata.gz: 25dc16eb8eaa65244fc568404d2e3cf413c8dd34
+  data.tar.gz: e0ab36f581ddb8e6848e4aa3e9b0e4dd0611c70a
 SHA512:
-  metadata.gz: 8e8cf404b114d806e7bb760a9eeda39ae4979470573d58d8754690d341fac14faebc07c5d6bc50468f1aa5c8da895ead3f138d4b18f26fa83464a3aa818847ee
-  data.tar.gz: 26f43c14a068ef369a2d506cd2c3a3d4da7af7d32424f67bc425ff870bdff63b756f3ffdc180d61168bc059277e31a7ee3694d1544de547e4717d5e49d3478dc
+  metadata.gz: 8f46e858aa74ed09a603e7e16b27d18d8c62dae561c7952e75fc30f106a8768b0e1363b5dff5905301cbb98b9d37149fab6cb1382188c67112cd1af4f08ce5fe
+  data.tar.gz: 4a1948b0b9a1e0a40ca1afbf76413b4ad86275ee61590fa63ee1c71fc4bc04cc3360c5ee690a1bed9e7103506307fed28608c05eb97fbe237ea10ca40edbbf40

data/.travis.yml

@@ -1,8 +1,12 @@
 rvm:
-  - 1.9.3
   - 2.0.0
-  - jruby-19mode
+  - 2.1.2
+  - 2.2.3
+  - jruby
 
 script: "bundle exec rake test"
 
-env: TEST=true
+env: TEST=true JRUBY_OPTS='--2.0'
+notifications:
+  slack:
+    secure: MMWxqKMT4m2UhZ+Ix4wgCs1nLvu9hUFCbyV/qJWmIFif7V4GUKXi6h9krMqGqBAd3YV3pP/InPlw3QoKBUGIAIr5GzDpSnU5ACv7E61v548qViEvPBqsfhRHha2M164rUHn32tpejJnIizVUyND/hIzNviIrab+G8uOaZXUtB5I=

data/CHANGELOG.md

@@ -1,4 +1,14 @@
 # Change Log
+## Unreleased
+
+## 1.2.0
+### Added
+* Include `:all_entries` option to `activate` block for getting over 1000 entries[#45](https://github.com/contentful-labs/contentful_middleman/issues/45)
+* Update `middleman-core` required version
+* Removes `middleman-blog` dependency[#9](https://github.com/contentful-labs/contentful_middleman/issues/9)
+* Add Webhook Integration[#55](https://github.com/contentful-labs/contentful_middleman/pull/55)
+* Clarify that `:all_entries` may need an `:order` attribute in the `:cda_query` for avoiding entry skipping and provided a sane default[#60](https://github.com/contentful/contentful_middleman/issues/60)
+* Add Localize Helpers
 
 ## 1.1.1
 ### Changed

data/CONTRIBUTING.md

@@ -16,7 +16,7 @@ Here are some ways *you* can contribute:
 * by closing [issues][]
 * by reviewing patches
 
-[issues]: https://github.com/middleman/middleman-blog/issues
+[issues]: https://github.com/contentful-labs/contentful_middleman/issues
 
 ## Submitting an Issue
 We use the [GitHub issue tracker][issues] to track bugs and features. Before

data/Gemfile

@@ -2,7 +2,7 @@
 # the following line to use "https"
 source 'http://rubygems.org'
 
-gem "middleman-core", :github => "middleman/middleman", :branch => 'v3-stable'
+gem "middleman-core", '~> 3.4'
 
 # Specify your gem's dependencies in contentful_middleman.gemspec
 gemspec
@@ -15,4 +15,3 @@ gem "rspec"
 gem "simplecov"
 
 gem "cane", :platforms => [:mri_19, :mri_20], :require => false
-gem 'coveralls', :require => false

data/Guardfile

@@ -0,0 +1,5 @@
+guard :rspec, cmd: 'rspec --format documentation --color' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/README.md

@@ -1,6 +1,6 @@
 # Contentful Middleman
 
-[![Build Status](https://travis-ci.org/contentful-labs/contentful_middleman.svg?branch=master)](https://travis-ci.org/contentful-labs/contentful_middleman)
+[![Build Status](https://travis-ci.org/contentful/contentful_middleman.svg?branch=master)](https://travis-ci.org/contentful/contentful_middleman)
 
 Contentful Middleman is a [Middleman](http://middlemanapp.com/) extension to use the Middleman static site generator together with the API-driven [Contentful CMS](https://www.contentful.com). It is powered by the [Contentful Ruby Gem](https://github.com/contentful/contentful.rb).
 
@@ -44,15 +44,19 @@ activate :contentful do |f|
 end
 ```
 
-Parameter       | Description
-----------      | ------------
-space           | Hash with an user choosen name for the space as key and the space id as value
-access_token    | Contentful Delivery API access token
-cda_query       | Hash describing query configuration. See [contentful.rb](https://github.com/contentful/contentful.rb) for more info (look for filter options there). Note that by default only 100 entries will be fetched, this can be configured to up to 1000 entries using the `limit` option.
-content_types   | Hash describing the mapping applied to entries of the imported content types
-use_preview_api | Boolean to toogle the used API. Set it to `false` to use `cdn.contentful.com` (default value). Set it to `true` to use `preview.contentful.com`. More info in [the documentation](https://www.contentful.com/developers/documentation/content-delivery-api/#preview-api)
+Parameter           | Description
+----------          | ------------
+space               | Hash with an user choosen name for the space as key and the space id as value
+access_token        | Contentful Delivery API access token
+cda_query           | Hash describing query configuration. See [contentful.rb](https://github.com/contentful/contentful.rb) for more info (look for filter options there). Note that by default only 100 entries will be fetched, this can be configured to up to 1000 entries using the `limit` option. Example: `f.cda_query = { limit: 1000 }`
+content_types       | Hash describing the mapping applied to entries of the imported content types
+use_preview_api     | Boolean to toggle the used API. Set it to `false` to use `cdn.contentful.com` (default value). Set it to `true` to use `preview.contentful.com`. More info in [the documentation](https://www.contentful.com/developers/documentation/content-delivery-api/#preview-api)
+all_entries         | Boolean to toggle multiple requests to the API for getting over 1000 entries. This uses a naive approach and can get rate limited. When using this, have in mind adding an `order` in your `:cda_query` . Default order is `order: 'sys.createdAt'`
+rebuild_on_webhook  | Boolean to toggle Webhook server. Server will run in port 5678, and will be expecting to receive Contentful Webhook calls on `/receive`
+webhook_timeout     | Integer (in seconds) for wait time after Webhook received for rebuilding. Only used if `:rebuild_on_webhook` is true. Defaults to 300 seconds
 
 You can activate the extension multiple times to import entries from different spaces.
+
 ## Entry mapping
 
 The extension will transform every fetched entry before storing it as a yaml file in the local
@@ -94,6 +98,44 @@ end
 
 There's also an example back-reference mapper in the examples directory for adding back-references onto entries that are linked to by other entries.
 
+#### Multiple Mappers
+
+If you want to process a Content Type with multiple mappers, you can use the [Composite Design Pattern](https://en.wikipedia.org/wiki/Composite_pattern).
+The Mapper code should look something similar to the following.
+
+Then you can attach as many Custom Mappers as you want to that one.
+
+```ruby
+class CompositeMapper < ContentfulMiddleman::Mapper::Base
+  @@mappers = []
+  def self.mappers
+    @@mappers
+  end
+
+  def map(context, entry)
+    super
+    mappers.each do |m|
+      m.map(context, entry)
+    end
+  end
+end
+```
+
+Then in your config.rb file:
+
+```ruby
+CompositeMapper.mappers << YourMapper.new
+CompositeMapper.mappers << OtherMapper.new
+
+activate :contentful do |f|
+  '... your config here ...'
+  f.content_types = {content_type_name_you_want_to_map: {mapper: CompositeMapper, id: 'content_type_id'}}
+end
+```
+
+*NOTE*: This kind of Composite Mapper is static, therefore if you want to have multiple combinations of mappers
+ for multiple entries, you'd need to write code a bit differently.
+
 ## Configuration: examples
 
 ```ruby
@@ -104,6 +146,7 @@ activate :contentful do |f|
   f.content_types = { partner: 'content-type-id'}
 end
 ```
+
 The above configuration does the following:
 
   * Sets the alias `partners` to the space with id _some-id_
@@ -158,3 +201,74 @@ For Kramdown this would be:
 ```
 <%= Kramdown::Document.new(data).to_html %>
 ```
+
+### Locales
+
+If you have localized entries, and want to display content for multiple locales.
+You can now include `locale: '*'` in your CDA query.
+
+Then you have the following methods of accessing locales:
+
+* **Manual access**
+
+You can access your localized fields by fetching the locale directly from the data
+
+```html
+<h1>Partners</h1>
+<ol>
+  <% data.partners.partner.each do |id, partner| %>
+    <li><%= partner["name"]['en-US'] %></li>
+  <% end %>
+</ol>
+```
+
+* **Entry Helper**
+
+You can also map an specific locale for all entry fields using `localize_entry`
+
+```html
+<h1>Partners</h1>
+<ol>
+  <% data.partners.partner.each do |id, partner| %>
+    <% localized_partner = localize_entry(partner, 'es') %>
+    <li><%= localized_partner["name"] %></li>
+  <% end %>
+</ol>
+```
+
+* **Generic Field Helper**
+
+The `localize` helper will map an specific locale to a field of your entry
+
+```html
+<h1>Partners</h1>
+<ol>
+  <% data.partners.partner.each do |id, partner| %>
+    <li>Value Field: <%= localize(partner, 'name', 'en-US') %></li>
+    <li>Array Field: <%= localize(partner, 'phones', 'es') %></li>
+  <% end %>
+</ol>
+```
+
+* **Specific Field Type Helper**
+
+Or, you can use `localize_value` or `localize_array` if you want more granularity.
+
+> This method is discouraged, as `localize` achieves the same goal and is a field-type
+agnostic wrapper of these methods.
+
+```html
+<h1>Partners</h1>
+<ol>
+  <% data.partners.partner.each do |id, partner| %>
+    <li>Value Field: <%= localize_value(partner['name'], 'en-US') %></li>
+    <li>Array Field: <%= localize_array(partner['phones'], 'es') %></li>
+  <% end %>
+</ol>
+```
+
+If your fields are not localized, the value of the field will be returned.
+
+In case of the field being localized but no value being set for a given entry, it will use
+a fallback locale, by default is `en-US` but can be specified as an additional
+parameter in all the mentioned calls.

data/contentful_middleman.gemspec

@@ -21,10 +21,16 @@ Gem::Specification.new do |s|
   # The version of middleman-core your extension depends on
   s.add_dependency("middleman-core", ["~> 3.3"])
 
-  s.add_dependency("middleman-blog", ["~> 3.5"])
-
   # Additional dependencies
-  s.add_dependency("contentful")
+  s.add_dependency("contentful", '~> 0.8')
+  s.add_dependency("contentful-webhook-listener", '~> 0.1')
 
   s.add_development_dependency 'rubygems-tasks', '~> 0.2'
+  s.add_development_dependency "guard"
+  s.add_development_dependency "guard-rspec"
+  s.add_development_dependency "bundler", "~> 1.6"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "vcr"
+  s.add_development_dependency "webmock"
 end

data/lib/contentful_middleman/commands/contentful.rb

@@ -1,5 +1,4 @@
 require 'middleman-core/cli'
-require 'middleman-blog/uri_templates'
 require 'date'
 require 'digest'
 require 'contentful_middleman/commands/context'

data/lib/contentful_middleman/core.rb

@@ -1,8 +1,11 @@
+require 'logger'
 require 'middleman-core'
 require 'contentful'
+require 'contentful/webhook/listener'
 require_relative 'mappers/base'
 require_relative 'helpers'
 require_relative 'instance'
+require_relative 'webhook_handler'
 
 # The Contentful Middleman extensions allows to load managed content into Middleman projects through the Contentful Content Management Platform.
 module ContentfulMiddleman
@@ -28,6 +31,15 @@ module ContentfulMiddleman
     option :use_preview_api, false,
       'Use the Preview API when fetching content'
 
+    option :all_entries, false,
+      'Allow multiple requests to the API for getting over 1000 entries'
+
+    option :rebuild_on_webhook, false,
+      "Run `middleman contentful --rebuild` upon receiving a Webhook on http://0.0.0.0:5678/receive"
+
+    option :webhook_timeout, 300,
+      "Wait time before rebuild after receiving a Webhook call"
+
 
     helpers ContentfulMiddleman::Helpers
 
@@ -38,6 +50,8 @@ module ContentfulMiddleman
       massage_options
 
       ContentfulMiddleman.instances << (ContentfulMiddleman::Instance.new self)
+
+      webhook_options
     end
 
     private
@@ -70,5 +84,9 @@ module ContentfulMiddleman
 
       options.content_types = new_content_types_options
     end
+
+    def webhook_options
+      ::ContentfulMiddleman::WebhookHandler.start(options) if options.rebuild_on_webhook
+    end
   end
 end

data/lib/contentful_middleman/helpers.rb

@@ -3,5 +3,34 @@ module ContentfulMiddleman
     def contentful_instances
       ContentfulMiddleman.instances
     end
+
+    def localize_entry(entry, locale, fallback_locale='en-US')
+      localized_entry = {}
+      entry.each do |field, value|
+        localized_entry[field] = localize(entry, field, locale, fallback_locale)
+      end
+      localized_entry
+    end
+
+    def localize(entry, field, locale, fallback_locale='en-US')
+      value = entry.fetch(field)
+
+      return localize_array(value, locale, fallback_locale) if value.is_a? ::Array
+      localize_value(value, locale, fallback_locale)
+    end
+
+    def localize_array(value, locale, fallback_locale='en-US')
+      value.map do |v|
+        localize_value(v, locale, fallback_locale)
+      end
+    end
+
+    def localize_value(value, locale, fallback_locale='en-US')
+      if value.respond_to? :fetch
+        return value.fetch(locale) if value.key? locale
+        return value.fetch(fallback_locale)
+      end
+      value
+    end
   end
 end

data/lib/contentful_middleman/import_task.rb

@@ -25,15 +25,18 @@ module ContentfulMiddleman
     private
     def local_data_files
       entries.map do |entry|
-        content_type_mapper_class = @content_type_mappers.fetch(entry.content_type.id)
+        content_type_mapper_class = @content_type_mappers.fetch(entry.content_type.id, nil)
+
+        next unless content_type_mapper_class
+
         content_type_name         = @content_type_names.fetch(entry.content_type.id).to_s
         context                   = ContentfulMiddleman::Context.new
 
-        content_type_mapper = content_type_mapper_class.new(entries)
+        content_type_mapper = content_type_mapper_class.new(entries, @contentful.options)
         content_type_mapper.map(context, entry)
 
         LocalData::File.new(context.to_yaml, File.join(@space_name, content_type_name, entry.id))
-      end
+      end.compact
     end
 
     def entries

data/lib/contentful_middleman/instance.rb

@@ -7,7 +7,11 @@ module ContentfulMiddleman
     end
 
     def entries
-      client.entries(options.cda_query)
+      if options.all_entries
+        all_entries(options.cda_query)
+      else
+        client.entries(options.cda_query)
+      end
     end
 
     def space_name
@@ -30,7 +34,28 @@ module ContentfulMiddleman
       end
     end
 
+    def options
+      @extension.options
+    end
+
     private
+
+    def all_entries(cda_query)
+      all = []
+      query = cda_query.clone
+      query[:order] = 'sys.createdAt' unless query.key?(:order)
+      num_entries = client.entries(limit: 1).total
+
+      ((num_entries / 1000) + 1).times do |i|
+        query[:limit] = 1000
+        query[:skip] = i * 1000
+        page = client.entries(query)
+        page.each { |entry| all << entry }
+      end
+
+      all
+    end
+
     def client
       @client ||= Contentful::Client.new(client_options)
     end
@@ -46,9 +71,5 @@ module ContentfulMiddleman
       client_options[:api_url] = API_PREVIEW_URL if options.use_preview_api
       client_options
     end
-
-    def options
-      @extension.options
-    end
   end
 end

data/lib/contentful_middleman/mappers/base.rb

@@ -5,8 +5,9 @@ module ContentfulMiddleman
     class Base
       attr_reader :entries
 
-      def initialize(entries)
+      def initialize(entries, options)
         @entries = entries
+        @options = options
         @children = {}
       end
 
@@ -24,6 +25,10 @@ module ContentfulMiddleman
 
       private
 
+      def has_multiple_locales?
+        @options.cda_query.fetch(:locale, nil) == '*'
+      end
+
       def map_field(context, field_name, field_value)
         value_mapping = map_value(field_value)
         context.set(field_name, value_mapping)
@@ -56,7 +61,10 @@ module ContentfulMiddleman
 
       def map_entry_full(entry, context)
         context.id = entry.id
-        entry.fields.each {|k, v| map_field context, k, v}
+
+        fields = has_multiple_locales? ? entry.fields_with_locales : entry.fields
+
+        fields.each {|k, v| map_field context, k, v}
       end
 
       def map_entry(entry)

data/lib/contentful_middleman/version.rb

@@ -1,3 +1,3 @@
 module ContentfulMiddleman
-  VERSION = "1.1.1"
+  VERSION = "1.2.0"
 end

data/lib/contentful_middleman/webhook_handler.rb

@@ -0,0 +1,33 @@
+require 'contentful/webhook/listener'
+
+module ContentfulMiddleman
+  class WebhookHandler < Contentful::Webhook::Listener::Controllers::Wait
+    def perform(*)
+      logger.info 'Webhook Received - Waiting for rebuild'
+      super
+      logger.info 'Rebuilding'
+      system('bundle exec middleman contentful --rebuild')
+    end
+
+    def self.start(options)
+      Contentful::Webhook::Listener::Server.start do |config|
+        config[:endpoints] = [{
+          endpoint: '/receive',
+          timeout: options.webhook_timeout,
+          controller: ::ContentfulMiddleman::WebhookHandler
+        }]
+        logger = Logger.new(STDOUT)
+        logger.level = Logger::INFO
+        logger.formatter = proc do |severity, datetime, progname, msg|
+          date_format = datetime.strftime("%Y-%m-%d %H:%M:%S")
+          if severity == "INFO"
+            "Webhook Server: #{msg}\n"
+          else
+            "[#{date_format}] #{severity} (#{progname}): #{msg}\n"
+          end
+        end
+        config[:logger] = logger
+      end
+    end
+  end
+end

data/spec/contentful_middleman/commands/context_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe ContentfulMiddleman::Context do
+  describe 'instance methods' do
+    it '#set' do
+      subject.set('foo', 'bar')
+      expect(subject.foo).to eq 'bar'
+    end
+
+    it '#get' do
+      subject.set('foo', 'bar')
+      expect(subject.get('foo')).to eq 'bar'
+    end
+
+    describe '#method_missing' do
+      it 'when :something= is called it uses #set(something, value)' do
+        subject.foo = 'bar'
+        expect(subject.foo).to eq 'bar'
+      end
+    end
+  end
+end

data/spec/contentful_middleman/core_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+class MapperDouble
+end
+
+describe ContentfulMiddleman::Core do
+  subject { described_class.new Middleman::Application }
+  let(:options) { subject.options }
+
+  before(:each) do
+    ContentfulMiddleman.instance_variable_set(:@contentful_middleman_instances, [])
+  end
+
+  describe 'options' do
+    it 'defaults' do
+      expect(options.space).to eq(nil)
+      expect(options.access_token).to eq(nil)
+      expect(options.cda_query).to eq({})
+      expect(options.content_types).to eq({})
+      expect(options.use_preview_api).to eq(false)
+      expect(options.all_entries).to eq(false)
+      expect(options.rebuild_on_webhook).to eq(false)
+      expect(options.webhook_timeout).to eq(300)
+    end
+  end
+
+  describe 'hooks' do
+    describe '#after_configuration' do
+      before do
+        options.space = {some_name: 'some_id'}
+      end
+
+      it 'updates space data from original hash' do
+        subject.after_configuration
+
+        expect(options.space).to match(name: :some_name, id: 'some_id')
+      end
+
+      describe 'updates content type data' do
+        it 'uses base mapper when only id is set' do
+          options.content_types = {
+            some_content_type_name: 'some_id'
+          }
+
+          subject.after_configuration
+
+          expect(options.content_types).to match('some_id' => {name: :some_content_type_name, mapper: ContentfulMiddleman::Mapper::Base})
+        end
+
+        it 'uses custom mapper' do
+          options.content_types = {
+            some_content_type_name: {
+              id: 'some_id',
+              mapper: MapperDouble
+            }
+          }
+
+          subject.after_configuration
+
+          expect(options.content_types).to match('some_id' => {name: :some_content_type_name, mapper: MapperDouble})
+        end
+      end
+
+      it 'sets up instances' do
+        expect(ContentfulMiddleman.instances.size).to eq(0)
+
+        subject.after_configuration
+
+        expect(ContentfulMiddleman.instances.size).to eq(1)
+      end
+
+      describe 'webhook handler' do
+        it 'does not get called if rebuild_on_webhook is false' do
+          expect(ContentfulMiddleman::WebhookHandler).not_to receive(:start)
+
+          subject.after_configuration
+        end
+
+        it 'gets called if rebuild_on_webhook is true' do
+          options.rebuild_on_webhook = true
+
+          expect(ContentfulMiddleman::WebhookHandler).to receive(:start)
+
+          subject.after_configuration
+        end
+      end
+    end
+  end
+end