jekyll-contentful-data-import 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 16c03521fa21f10014b5e71459b53865e1da9aac
-  data.tar.gz: 53ef74f05c7123a7af50fb683f6f1c8db943740c
+  metadata.gz: 1babd5463d86525723186512d0a5ff1788b032c8
+  data.tar.gz: beedb4ade62bd19fa81a27afaf28070113018906
 SHA512:
-  metadata.gz: 27be80ec515ad6464e1ef00414a4a9e86c893ba1a535b5823980cce0eaadeb7572ee391125ee56c1f444e31701f351d3c625a029a69899d78aeaf77c663f28e2
-  data.tar.gz: 228eb4426ad9bd597dff4f064a290683621925ea928b3e1eaae74157743cc13552232991ec144924b4e5004212d1be6aa3c2bb75a95a1cde4fcdae0ca0964128
+  metadata.gz: 5757f7558659b94941ac13d42750943c428b0827eac1eb0d5c7465447676d84a8a3bcca25e14ee699d66c2ee620edd31162aeff9ac89551ac2f7b32c5de1aed1
+  data.tar.gz: 391ff1027438166c09188e906e334cb04ef99acdd0987e4417a54c5cb1b41d01e8de6511e0a27b351fc7dba7728d9e9fcc8b851025dfb5553dd4411cd5001a9a

data/CHANGELOG.md

@@ -1,6 +1,13 @@
 # Change Log
+
 ## Unreleased
 
+## v1.3.0
+### Added
+* Added the possibility to use `space` and `access_token` from environment variables [#14](https://github.com/contentful/jekyll-contentful-data-import/issues/14)
+* `all_entries` option to fetch entries over the 1000 limit of a single CDA request
+* `all_entries_page_size` option to customize the size of the request for particularly heavy entries
+
 ## v1.2.1
 ### Fixed
 * README showing incorrect configuration [#5](https://github.com/contentful/jekyll-contentful-data-import/issues/5)

data/README.md

@@ -42,27 +42,31 @@ To configure the extension, add the following configuration block to Jekyll's `_
 contentful:
   spaces:
     - example: # Jekyll _data folder identifier - Required
-        space: cfexampleapi        # Required
-        access_token: b4c0n73n7fu1 # Required
-        cda_query:                 # Optional
+        space: cfexampleapi         # Required
+        access_token: b4c0n73n7fu1  # Required
+        cda_query:                  # Optional
           include: 2
           limit: 100
-        content_types:             # Optional
+        all_entries: true           # Optional - Defaults to false, only grabbing the amount set on CDA Query
+        all_entries_page_size: 1000 # Optional - Defaults to 1000, maximum amount of entries per CDA Request for all_entries
+        content_types:              # Optional
           cat: MyCoolMapper
-        client_options:            # Optional
+        client_options:             # Optional
           api_url: 'preview.contentful.com' # Defaults to 'api.contentful.com' which is Production
-        base_path: app_path        # Optional - Defaults to Current directory
+        base_path: app_path         # Optional - Defaults to Current directory
 
 ```
 
-Parameter           | Description
-----------          | ------------
-space               | Contentful Space ID
-access_token        | Contentful Delivery API access token
-cda_query           | Hash describing query configuration. See [contentful.rb](https://github.com/contentful/contentful.rb) for more info (look for filter options there). Note that by default only 100 entries will be fetched, this can be configured to up to 1000 entries using the `limit` option.
-content_types       | Hash describing the mapping applied to entries of the imported content types
-client_options      | Hash describing Contentful::Client configuration. See [contentful.rb](https://github.com/contentful/contentful.rb) for more info.
-base_path           | String with path to your Jekyll Application, defaults to current directory. Path is relative to your current location.
+Parameter             | Description
+----------            | ------------
+space                 | Contentful Space ID
+access_token          | Contentful Delivery API access token
+cda_query             | Hash describing query configuration. See [contentful.rb](https://github.com/contentful/contentful.rb) for more info (look for filter options there). Note that by default only 100 entries will be fetched, this can be configured to up to 1000 entries using the `limit` option.
+all_entries           | Boolean, if true will run multiple queries to the API until it fetches all entries for the space
+all_entries_page_size | Integer, the amount of maximum entries per CDA Request when fetching :all_entries
+content_types         | Hash describing the mapping applied to entries of the imported content types
+client_options        | Hash describing Contentful::Client configuration. See [contentful.rb](https://github.com/contentful/contentful.rb) for more info.
+base_path             | String with path to your Jekyll Application, defaults to current directory. Path is relative to your current location.
 
 You can add multiple spaces to your configuration
 
@@ -95,6 +99,27 @@ class MySysMapper < ::Jekyll::Contentful::Mappers::Base
 end
 ```
 
+### Hiding Space and Access Token in Public Repositories
+
+In most cases you may want to avoid including your credentials in publicly available sites,
+therefore you can hide your `space` and `access_token` by replacing them with `ENV_` prepended variables.
+
+This will look for the values in your system.
+
+For example:
+
+```yaml
+contentful:
+  spaces:
+    - example:
+        space: ENV_CONTENTFUL_SPACE_ID
+        access_token: ENV_CONTENTFUL_ACCESS_TOKEN
+```
+
+With this setup, your Space ID will be looked upon on `ENV['CONTENTFUL_SPACE_ID']` and your Access Token
+on `ENV['CONTENTFUL_ACCESS_TOKEN']`. This way it is safe to share your code, without having to worry
+about your credentials.
+
 #### Caveats
 
 Jekyll itself only allows you to import code as plugins only for its recognized plugin entry points.

data/lib/jekyll-contentful-data-import/importer.rb

@@ -13,23 +13,49 @@ module Jekyll
       def run
         spaces.each do |name, options|
           space_client = client(
-            options['space'],
-            options['access_token'],
+            value_for(options, 'space'),
+            value_for(options, 'access_token'),
             client_options(options.fetch('client_options', {}))
           )
 
           Jekyll::Contentful::DataExporter.new(
             name,
-            space_client.entries(options.fetch('cda_query', {})),
+            get_entries(space_client, options),
             options
           ).run
         end
       end
 
+      def value_for(options, key)
+        potential_value = options[key]
+        return ENV[potential_value.gsub('ENV_', '')] if potential_value.start_with?('ENV_')
+        potential_value
+      end
+
       def spaces
         config['spaces'].map { |space_data| space_data.first }
       end
 
+      def get_entries(space_client, options)
+        cda_query = options.fetch('cda_query', {})
+        return space_client.entries(cda_query) unless options.fetch('all_entries', false)
+
+        all = []
+        query = cda_query.clone
+        query[:order] = 'sys.createdAt' unless query.key?(:order)
+        num_entries = space_client.entries(limit: 1).total
+
+        page_size = options.fetch('all_entries_page_size', 1000)
+        ((num_entries / page_size) + 1).times do |i|
+          query[:limit] = page_size
+          query[:skip] = i * page_size
+          page = space_client.entries(query)
+          page.each { |entry| all << entry }
+        end
+
+        all
+      end
+
       def client(space, access_token, options = {})
         options = {
           space: space,

data/lib/jekyll-contentful-data-import/version.rb

@@ -1,5 +1,5 @@
 module Jekyll
   module Contentful
-    VERSION = "1.2.1"
+    VERSION = "1.3.0"
   end
 end

data/lib/jekyll/commands/contentful.rb

@@ -8,7 +8,7 @@ module Jekyll
           c.syntax 'contentful [OPTIONS]'
           c.description 'Imports data from Contentful'
 
-          options.each {|opt| c.option *opt }
+          options.each {|opt| c.option(*opt) }
 
           c.action do |args, options|
             contentful_config = Jekyll.configuration['contentful']

data/spec/jekyll-contentful/importer_spec.rb

@@ -1,8 +1,18 @@
 require 'spec_helper'
 
+class EntryArrayDouble < ::Array
+  def total
+    size
+  end
+end
+
 class ClientDouble
+  def initialize(response = EntryArrayDouble.new)
+    @response = response
+  end
+
   def entries(options = {})
-    []
+    @response
   end
 end
 
@@ -44,9 +54,20 @@ describe Jekyll::Contentful::Importer do
       end
     end
 
+    describe '#value_for' do
+      it 'returns set value regularly' do
+        expect(subject.value_for({'foo' => 'bar'}, 'foo')).to eq 'bar'
+      end
+
+      it 'returns ENV value if prefixed with ENV_' do
+        ENV['bar'] = 'bar_from_env'
+        expect(subject.value_for({'foo' => 'ENV_bar'}, 'foo')).to eq 'bar_from_env'
+      end
+    end
+
     describe '#run' do
       it 'runs exporter for each space' do
-        allow(subject).to receive(:spaces).and_return([['foo', {}], ['bar', {}]])
+        allow(subject).to receive(:spaces).and_return([['foo', {'space' => 'foo', 'access_token' => 'bar'}], ['bar', {'space' => 'bar', 'access_token' => 'foo'}]])
         allow(subject).to receive(:client).and_return(ClientDouble.new)
 
         expect(Jekyll::Contentful::DataExporter).to receive(:new).and_return(ExporterDouble.new).twice
@@ -62,5 +83,28 @@ describe Jekyll::Contentful::Importer do
         subject.run
       end
     end
+
+    describe '#get_entries' do
+      it 'runs a single query by default' do
+        client = ClientDouble.new
+        expect(client).to receive(:entries).once
+
+        subject.get_entries(client, {})
+      end
+
+      it 'fetches all entries when all_entries is set' do
+        client = ClientDouble.new(EntryArrayDouble.new([1, 2, 3]))
+        expect(client).to receive(:entries).and_call_original.twice
+
+        subject.get_entries(client, {'all_entries' => true})
+      end
+
+      it 'can select page size when all_entries by using all_entries_page_size' do
+        client = ClientDouble.new(EntryArrayDouble.new([1, 2, 3]))
+        expect(client).to receive(:entries).and_call_original.exactly(3).times
+
+        subject.get_entries(client, {'all_entries' => true, 'all_entries_page_size' => 2})
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-contentful-data-import
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Contentful GmbH
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-02 00:00:00.000000000 Z
+date: 2016-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jekyll
@@ -223,7 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.0
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Include mangablable content from the Contentful CMS and API into your Jekyll